From d55b4194ab8aa2fbf4f3a75dc5b59a93c81700e6 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 9 Jul 2024 14:50:42 -0700 Subject: [PATCH 01/44] Add verification_mapping proto --- Cargo.lock | 46 +++++++++++++++++++++++++++++----------------- Cargo.toml | 2 +- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49984f909..20ef9addf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1621,7 +1621,7 @@ source = "git+https://github.com/helium/proto?branch=master#fdef2c6774416b1b361a dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "prost", "rand 0.8.5", "rand_chacha 0.3.0", @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "http-serde", "humantime-serde", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "notify", "serde", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hex-literal", "http 0.2.11", "lazy_static", @@ -3776,7 +3776,7 @@ dependencies = [ "h3o", "helium-anchor-gen", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", "hex", "itertools", "jsonrpc_client", @@ -3812,6 +3812,18 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "helium-proto" +version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" +dependencies = [ + "bytes", + "prost", + "prost-build", + "serde", + "serde_json", +] + [[package]] name = "helium-sub-daos" version = "0.1.5" @@ -3853,7 +3865,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4269,7 +4281,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "humantime-serde", "metrics", @@ -4338,7 +4350,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hextree", "http 0.2.11", "http-serde", @@ -4380,7 +4392,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "http-serde", "humantime-serde", @@ -4422,7 +4434,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http-serde", "humantime-serde", "iot-config", @@ -5010,7 +5022,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hextree", "http 0.2.11", "http-serde", @@ -5050,7 +5062,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "mobile-config", "prost", "rand 0.8.5", @@ -5086,7 +5098,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "http-serde", "humantime-serde", @@ -5130,7 +5142,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hex-assignments", "hextree", "http-serde", @@ -5813,7 +5825,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5896,7 +5908,7 @@ dependencies = [ "futures-util", "helium-anchor-gen", "helium-lib", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6503,7 +6515,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "humantime-serde", "lazy_static", "metrics", diff --git a/Cargo.toml b/Cargo.toml index a45355a99..c176e7b34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "macpie/verification_mapping", features = [ "services", ] } beacon = { git = "https://github.com/helium/proto", branch = "master" } From ef2eceab8df91c8d62ba14d3637f1d62b96700b0 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 11 Jul 2024 15:19:43 -0700 Subject: [PATCH 02/44] Add ingest for subscriber_mapping_event --- Cargo.lock | 46 +++++++++++------------------ Cargo.toml | 5 +++- file_store/src/file_info.rs | 5 ++++ file_store/src/traits/msg_verify.rs | 3 +- ingest/src/server_mobile.rs | 44 ++++++++++++++++++++++++++- 5 files changed, 71 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20ef9addf..49984f909 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1621,7 +1621,7 @@ source = "git+https://github.com/helium/proto?branch=master#fdef2c6774416b1b361a dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "prost", "rand 0.8.5", "rand_chacha 0.3.0", @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "notify", "serde", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hex-literal", "http 0.2.11", "lazy_static", @@ -3776,7 +3776,7 @@ dependencies = [ "h3o", "helium-anchor-gen", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", + "helium-proto", "hex", "itertools", "jsonrpc_client", @@ -3812,18 +3812,6 @@ dependencies = [ "tonic-build", ] -[[package]] -name = "helium-proto" -version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" -dependencies = [ - "bytes", - "prost", - "prost-build", - "serde", - "serde_json", -] - [[package]] name = "helium-sub-daos" version = "0.1.5" @@ -3865,7 +3853,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4281,7 +4269,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "humantime-serde", "metrics", @@ -4350,7 +4338,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -4392,7 +4380,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -4434,7 +4422,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http-serde", "humantime-serde", "iot-config", @@ -5022,7 +5010,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -5062,7 +5050,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "mobile-config", "prost", "rand 0.8.5", @@ -5098,7 +5086,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -5142,7 +5130,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hex-assignments", "hextree", "http-serde", @@ -5825,7 +5813,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5908,7 +5896,7 @@ dependencies = [ "futures-util", "helium-anchor-gen", "helium-lib", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6515,7 +6503,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "humantime-serde", "lazy_static", "metrics", diff --git a/Cargo.toml b/Cargo.toml index c176e7b34..81fde8a1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "macpie/verification_mapping", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ "services", ] } beacon = { git = "https://github.com/helium/proto", branch = "master" } @@ -133,3 +133,6 @@ sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb # [patch.'https://github.com/helium/proto'] # helium-proto = { path = "../proto" } # beacon = { path = "../proto/beacon" } +# Temporary patch +[patch."https://github.com/helium/proto"] +helium-proto = { path = "../proto" } diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index ece994502..2f1631c82 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -160,6 +160,7 @@ pub const LANDTYPE_DATA_SET: &str = "landtype"; pub const SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "service_provider_boosted_rewards_banned_radio"; pub const VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "verified_service_provider_boosted_rewards_banned_radio"; +pub const VERIFIED_SUBSCRIBER_MAPPING_EVENT: &str = "verified_subscriber_mapping_event"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -214,6 +215,7 @@ pub enum FileType { VerifiedInvalidatedRadioThresholdIngestReport, SPBoostedRewardsBannedRadioIngestReport, VerifiedSPBoostedRewardsBannedRadioIngestReport, + VerifiedSubscriberMappingEvent, } impl fmt::Display for FileType { @@ -279,6 +281,7 @@ impl fmt::Display for FileType { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } + Self::VerifiedSubscriberMappingEvent => VERIFIED_SUBSCRIBER_MAPPING_EVENT, }; f.write_str(s) } @@ -347,6 +350,7 @@ impl FileType { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } + Self::VerifiedSubscriberMappingEvent => VERIFIED_SUBSCRIBER_MAPPING_EVENT, } } } @@ -415,6 +419,7 @@ impl FromStr for FileType { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO => { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport } + VERIFIED_SUBSCRIBER_MAPPING_EVENT => Self::VerifiedSubscriberMappingEvent, _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 617de377b..e579787b8 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -3,7 +3,7 @@ use helium_crypto::{PublicKey, Verify}; use helium_proto::services::{ iot_config, mobile_config, poc_lora::{LoraBeaconReportReqV1, LoraStreamSessionInitV1, LoraWitnessReportReqV1}, - poc_mobile::ServiceProviderBoostedRewardsBannedRadioReqV1, + poc_mobile::{self, ServiceProviderBoostedRewardsBannedRadioReqV1}, }; use helium_proto::{ services::poc_mobile::{ @@ -94,6 +94,7 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamResV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); +impl_msg_verify!(poc_mobile::SubscriberMappingEventReqV1, signature); #[cfg(test)] mod test { diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 2172b7da6..9b2fde8ac 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -19,7 +19,8 @@ use helium_proto::services::poc_mobile::{ RadioThresholdReportRespV1, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, ServiceProviderBoostedRewardsBannedRadioReqV1, ServiceProviderBoostedRewardsBannedRadioRespV1, SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, SubscriberLocationIngestReportV1, - SubscriberLocationReqV1, SubscriberLocationRespV1, WifiHeartbeatIngestReportV1, + SubscriberLocationReqV1, SubscriberLocationRespV1, SubscriberMappingEventReqV1, + SubscriberMappingEventResV1, VerifiedSubscriberMappingEventV1, WifiHeartbeatIngestReportV1, WifiHeartbeatReqV1, WifiHeartbeatRespV1, }; use std::{net::SocketAddr, path::Path}; @@ -42,6 +43,7 @@ pub struct GrpcServer { invalidated_radio_threshold_report_sink: FileSinkClient, coverage_object_report_sink: FileSinkClient, sp_boosted_rewards_ban_sink: FileSinkClient, + subscriber_mapping_event_sink: FileSinkClient, required_network: Network, address: SocketAddr, api_token: MetadataValue, @@ -368,6 +370,33 @@ impl poc_mobile::PocMobile for GrpcServer { ServiceProviderBoostedRewardsBannedRadioRespV1 { id }, )) } + + async fn submit_subscriber_mapping_event( + &self, + request: Request, + ) -> GrpcResult { + let timestamp: u64 = Utc::now().timestamp_millis() as u64; + let event: SubscriberMappingEventReqV1 = request.into_inner(); + let subscriber_id = event.subscriber_id.clone(); + + custom_tracing::record("subscriber_id", bs58::encode(&subscriber_id).into_string()); + custom_tracing::record_b58("pub_key", &event.pub_key); + + let report = self + .verify_public_key(event.pub_key.as_ref()) + .and_then(|public_key| self.verify_network(public_key)) + .and_then(|public_key| self.verify_signature(public_key, event)) + .map(|(_, event)| VerifiedSubscriberMappingEventV1 { + subscriber_id: event.subscriber_id, + total_reward_points: event.total_reward_points, + timestamp, + })?; + + _ = self.subscriber_mapping_event_sink.write(report, []).await; + + let id = timestamp.to_string(); + Ok(Response::new(SubscriberMappingEventResV1 { id })) + } } pub async fn grpc_server(settings: &Settings) -> Result<()> { @@ -484,6 +513,17 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { .create() .await?; + let (subscriber_mapping_event_sink, subscriber_mapping_event_server) = + file_sink::FileSinkBuilder::new( + FileType::VerifiedSubscriberMappingEvent, + store_base_path, + file_upload.clone(), + concat!(env!("CARGO_PKG_NAME"), "_verified_subscriber_mapping_event"), + ) + .roll_time(settings.roll_time) + .create() + .await?; + let Some(api_token) = settings .token .as_ref() @@ -502,6 +542,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { invalidated_radio_threshold_report_sink, coverage_object_report_sink, sp_boosted_rewards_ban_sink, + subscriber_mapping_event_sink, required_network: settings.network, address: settings.listen_addr, api_token, @@ -524,6 +565,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { .add_task(invalidated_radio_threshold_report_sink_server) .add_task(coverage_object_report_sink_server) .add_task(sp_boosted_rewards_ban_sink_server) + .add_task(subscriber_mapping_event_server) .add_task(grpc_server) .build() .start() From 7b111932b7d1403ea938e1a94c9aabc0e7af51b1 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 12 Jul 2024 11:25:45 -0700 Subject: [PATCH 03/44] Add SubscriberMappingEventDeamon --- file_store/src/lib.rs | 1 + file_store/src/verified_mapping_event.rs | 53 +++++++++ .../migrations/35_verified_mapping_event.sql | 7 ++ mobile_verifier/src/cli/server.rs | 9 ++ mobile_verifier/src/lib.rs | 1 + .../src/subscriber_mapping_event.rs | 111 ++++++++++++++++++ 6 files changed, 182 insertions(+) create mode 100644 file_store/src/verified_mapping_event.rs create mode 100644 mobile_verifier/migrations/35_verified_mapping_event.sql create mode 100644 mobile_verifier/src/subscriber_mapping_event.rs diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index 66ab76973..bdf850866 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -24,6 +24,7 @@ pub mod reward_manifest; mod settings; pub mod speedtest; pub mod traits; +pub mod verified_mapping_event; pub mod wifi_heartbeat; pub use crate::file_store::FileStore; diff --git a/file_store/src/verified_mapping_event.rs b/file_store/src/verified_mapping_event.rs new file mode 100644 index 000000000..10c575972 --- /dev/null +++ b/file_store/src/verified_mapping_event.rs @@ -0,0 +1,53 @@ +use crate::{ + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventV1; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug)] +pub struct VerifiedSubscriberMappingEvent { + pub subscriber_id: String, + pub total_reward_points: u64, + pub timestamp: DateTime, +} + +impl MsgDecode for VerifiedSubscriberMappingEvent { + type Msg = VerifiedSubscriberMappingEventV1; +} + +impl MsgTimestamp>> for VerifiedSubscriberMappingEventV1 { + fn timestamp(&self) -> Result> { + self.timestamp.to_timestamp() + } +} + +impl MsgTimestamp for VerifiedSubscriberMappingEvent { + fn timestamp(&self) -> u64 { + self.timestamp.encode_timestamp() + } +} + +impl From for VerifiedSubscriberMappingEventV1 { + fn from(v: VerifiedSubscriberMappingEvent) -> Self { + let timestamp = v.timestamp(); + VerifiedSubscriberMappingEventV1 { + subscriber_id: v.subscriber_id, + total_reward_points: v.total_reward_points as u64, + timestamp, + } + } +} + +impl TryFrom for VerifiedSubscriberMappingEvent { + type Error = Error; + fn try_from(v: VerifiedSubscriberMappingEventV1) -> Result { + let timestamp = v.timestamp()?; + Ok(Self { + subscriber_id: v.subscriber_id, + total_reward_points: v.total_reward_points, + timestamp, + }) + } +} diff --git a/mobile_verifier/migrations/35_verified_mapping_event.sql b/mobile_verifier/migrations/35_verified_mapping_event.sql new file mode 100644 index 000000000..479a17de0 --- /dev/null +++ b/mobile_verifier/migrations/35_verified_mapping_event.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS verified_mapping_event ( + subscriber_id TEXT NOT NULL, + total_reward_points INTEGER NOT NULL, + timestamp TIMESTAMPTZ, + inserted_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (subscriber_id, timestamp) +); \ No newline at end of file diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 6f4effad2..52858dbe4 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -11,6 +11,7 @@ use crate::{ sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, + subscriber_mapping_event::SubscriberMappingEventDeamon, telemetry, Settings, }; use anyhow::Result; @@ -141,6 +142,14 @@ impl Cmd { ) .await?, ) + .add_task( + SubscriberMappingEventDeamon::create_managed_task( + pool.clone(), + settings, + report_ingest.clone(), + ) + .await?, + ) .add_task( CoverageDaemon::create_managed_task( pool.clone(), diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 697ce9d5a..8ff15f803 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -14,6 +14,7 @@ pub mod sp_boosted_rewards_bans; pub mod speedtests; pub mod speedtests_average; pub mod subscriber_location; +pub mod subscriber_mapping_event; pub mod telemetry; pub use settings::Settings; diff --git a/mobile_verifier/src/subscriber_mapping_event.rs b/mobile_verifier/src/subscriber_mapping_event.rs new file mode 100644 index 000000000..87c622248 --- /dev/null +++ b/mobile_verifier/src/subscriber_mapping_event.rs @@ -0,0 +1,111 @@ +use crate::Settings; +use file_store::{ + file_info_poller::{FileInfoStream, LookbackBehavior}, + file_source, + verified_mapping_event::VerifiedSubscriberMappingEvent, + FileStore, FileType, +}; +use futures::stream::StreamExt; +use sqlx::{Pool, Postgres, Transaction}; +use task_manager::{ManagedTask, TaskManager}; +use tokio::sync::mpsc::Receiver; + +pub struct SubscriberMappingEventDeamon { + pool: Pool, + events: Receiver>, +} + +impl SubscriberMappingEventDeamon { + pub fn new( + pool: Pool, + events: Receiver>, + ) -> Self { + Self { pool, events } + } + + pub async fn create_managed_task( + pool: Pool, + settings: &Settings, + file_store: FileStore, + ) -> anyhow::Result { + let (events, event_server) = + file_source::continuous_source::() + .state(pool.clone()) + .store(file_store) + .lookback(LookbackBehavior::StartAfter(settings.start_after)) + .prefix(FileType::VerifiedSubscriberMappingEvent.to_string()) + .create() + .await?; + + let task = Self::new(pool, events); + + Ok(TaskManager::builder() + .add_task(event_server) + .add_task(task) + .build()) + } + + pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + tracing::info!("Starting sme deamon"); + loop { + tokio::select! { + biased; + _ = shutdown.clone() => { + tracing::info!("sme deamon shutting down"); + break; + } + Some(file) = self.events.recv() => { + self.process_file(file).await?; + } + } + } + Ok(()) + } + + async fn process_file( + &self, + file: FileInfoStream, + ) -> anyhow::Result<()> { + tracing::info!( + "Processing Verified Mapping Event file {}", + file.file_info.key + ); + let mut transaction = self.pool.begin().await?; + let mut events = file.into_stream(&mut transaction).await?; + + while let Some(event) = events.next().await { + save_event(&event, &mut transaction).await?; + } + + transaction.commit().await?; + Ok(()) + } +} + +impl ManagedTask for SubscriberMappingEventDeamon { + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::future::LocalBoxFuture<'static, anyhow::Result<()>> { + Box::pin(self.run(shutdown)) + } +} + +pub async fn save_event( + event: &VerifiedSubscriberMappingEvent, + exec: &mut Transaction<'_, Postgres>, +) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + insert into verified_mapping_event (subscriber_id, total_reward_points, timestamp) + values ($1, $2, $3) + on conflict (subscriber_id, timestamp) do nothing + "#, + ) + .bind(&event.subscriber_id) + .bind(event.total_reward_points as i64) + .bind(event.timestamp) + .execute(exec) + .await?; + Ok(()) +} From d06dc850750a67a8d7230702876d36128e3a0e95 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 15 Jul 2024 16:07:25 -0700 Subject: [PATCH 04/44] - First pass at rewards for verifier mapping events - Make subscriber_id a Vec s Please enter the commit message for your changes. Lines starting --- .../migrations/35_verified_mapping_event.sql | 2 +- mobile_verifier/src/reward_shares.rs | 42 +++++++++++++++++-- mobile_verifier/src/rewarder.rs | 7 +++- .../src/subscriber_mapping_event.rs | 42 ++++++++++++++++++- 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/mobile_verifier/migrations/35_verified_mapping_event.sql b/mobile_verifier/migrations/35_verified_mapping_event.sql index 479a17de0..12eb4cedd 100644 --- a/mobile_verifier/migrations/35_verified_mapping_event.sql +++ b/mobile_verifier/migrations/35_verified_mapping_event.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS verified_mapping_event ( - subscriber_id TEXT NOT NULL, + subscriber_id BYTEA NOT NULL, total_reward_points INTEGER NOT NULL, timestamp TIMESTAMPTZ, inserted_at TIMESTAMPTZ DEFAULT now(), diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 04aeb1d88..56695074d 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -6,6 +6,7 @@ use crate::{ seniority::Seniority, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, + subscriber_mapping_event::VerifiedMappingEventShares, }; use chrono::{DateTime, Duration, Utc}; use coverage_point_calculator::SPBoostedRewardEligibility; @@ -187,12 +188,17 @@ impl TransferRewards { #[derive(Default)] pub struct MapperShares { pub discovery_mapping_shares: SubscriberValidatedLocations, + pub verified_mapping_event_shares: VerifiedMappingEventShares, } impl MapperShares { - pub fn new(discovery_mapping_shares: SubscriberValidatedLocations) -> Self { + pub fn new( + discovery_mapping_shares: SubscriberValidatedLocations, + verified_mapping_event_shares: VerifiedMappingEventShares, + ) -> Self { Self { discovery_mapping_shares, + verified_mapping_event_shares, } } @@ -204,11 +210,15 @@ impl MapperShares { // the number of subscribers eligible for discovery location rewards let discovery_mappers_count = Decimal::from(self.discovery_mapping_shares.len()); + let verified_mapping_event_count = Decimal::from(self.verified_mapping_event_shares.len()); + // calculate the total eligible mapping shares for the epoch // this could be simplified as every subscriber is awarded the same share // however the function is setup to allow the verification mapper shares to be easily // added without impacting code structure ( the per share value for those will be different ) - let total_mapper_shares = discovery_mappers_count * DISCOVERY_MAPPING_SHARES; + let total_mapper_shares = + (discovery_mappers_count + verified_mapping_event_count) * DISCOVERY_MAPPING_SHARES; + let res = total_mappers_pool .checked_div(total_mapper_shares) .unwrap_or(Decimal::ZERO); @@ -220,7 +230,8 @@ impl MapperShares { reward_period: &'_ Range>, reward_per_share: Decimal, ) -> impl Iterator + '_ { - self.discovery_mapping_shares + let discovery_rewards = self + .discovery_mapping_shares .into_iter() .map(move |subscriber_id| proto::SubscriberReward { subscriber_id, @@ -239,7 +250,32 @@ impl MapperShares { reward: Some(ProtoReward::SubscriberReward(subscriber_reward)), }, ) + }); + + let verified_rewards = self + .verified_mapping_event_shares + .into_iter() + .map(move |verified_share| proto::SubscriberReward { + subscriber_id: verified_share.subscriber_id, + discovery_location_amount: (Decimal::from(verified_share.total_reward_points) + * reward_per_share) + .round_dp_with_strategy(0, RoundingStrategy::ToZero) + .to_u64() + .unwrap_or_default(), }) + .filter(|subscriber_reward| subscriber_reward.discovery_location_amount > 0) + .map(|subscriber_reward| { + ( + subscriber_reward.discovery_location_amount, + proto::MobileRewardShare { + start_period: reward_period.start.encode_timestamp(), + end_period: reward_period.end.encode_timestamp(), + reward: Some(ProtoReward::SubscriberReward(subscriber_reward)), + }, + ) + }); + + discovery_rewards.chain(verified_rewards) } } diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 638ec8c06..0ecbf6309 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -9,7 +9,7 @@ use crate::{ }, sp_boosted_rewards_bans, speedtests, speedtests_average::SpeedtestAverages, - subscriber_location, telemetry, Settings, + subscriber_location, subscriber_mapping_event, telemetry, Settings, }; use anyhow::bail; use chrono::{DateTime, TimeZone, Utc}; @@ -512,8 +512,11 @@ pub async fn reward_mappers( let location_shares = subscriber_location::aggregate_location_shares(pool, reward_period).await?; + let vme_shares = + subscriber_mapping_event::aggregate_verified_mapping_events(pool, reward_period).await?; + // determine mapping shares based on location shares and data transferred - let mapping_shares = MapperShares::new(location_shares); + let mapping_shares = MapperShares::new(location_shares, vme_shares); let total_mappers_pool = reward_shares::get_scheduled_tokens_for_mappers(reward_period.end - reward_period.start); let rewards_per_share = mapping_shares.rewards_per_share(total_mappers_pool)?; diff --git a/mobile_verifier/src/subscriber_mapping_event.rs b/mobile_verifier/src/subscriber_mapping_event.rs index 87c622248..66dedb4b2 100644 --- a/mobile_verifier/src/subscriber_mapping_event.rs +++ b/mobile_verifier/src/subscriber_mapping_event.rs @@ -1,11 +1,14 @@ +use std::ops::Range; + use crate::Settings; +use chrono::{DateTime, Duration, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, file_source, verified_mapping_event::VerifiedSubscriberMappingEvent, FileStore, FileType, }; -use futures::stream::StreamExt; +use futures::{stream::StreamExt, TryStreamExt}; use sqlx::{Pool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; @@ -91,7 +94,7 @@ impl ManagedTask for SubscriberMappingEventDeamon { } } -pub async fn save_event( +async fn save_event( event: &VerifiedSubscriberMappingEvent, exec: &mut Transaction<'_, Postgres>, ) -> Result<(), sqlx::Error> { @@ -109,3 +112,38 @@ pub async fn save_event( .await?; Ok(()) } + +const SUBSCRIBER_REWARD_PERIOD_IN_DAYS: i64 = 1; +pub type VerifiedMappingEventShares = Vec; + +#[derive(sqlx::FromRow)] +pub struct VerifiedMappingEventShare { + pub subscriber_id: Vec, + pub total_reward_points: i64, +} + +pub async fn aggregate_verified_mapping_events( + db: impl sqlx::PgExecutor<'_> + Copy, + reward_period: &Range>, +) -> Result { + let mut rows = sqlx::query_as::<_, VerifiedMappingEventShare>( + "SELECT + subscriber_id, + SUM(total_reward_points) AS total_reward_points + FROM + verified_mapping_event + WHERE timestamp >= $1 AND timestamp < $2 + GROUP BY + subscriber_id;", + ) + .bind(reward_period.end - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) + .bind(reward_period.end) + .fetch(db); + + let mut vme_shares = VerifiedMappingEventShares::new(); + while let Some(share) = rows.try_next().await? { + vme_shares.push(share) + } + + Ok(vme_shares) +} From 1d46baef08405bf7c7a006cff4f22d906b23bd75 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 15 Jul 2024 16:21:04 -0700 Subject: [PATCH 05/44] String to Vec --- file_store/src/verified_mapping_event.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/file_store/src/verified_mapping_event.rs b/file_store/src/verified_mapping_event.rs index 10c575972..61ad7d78d 100644 --- a/file_store/src/verified_mapping_event.rs +++ b/file_store/src/verified_mapping_event.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Deserialize, Serialize, Debug)] pub struct VerifiedSubscriberMappingEvent { - pub subscriber_id: String, + pub subscriber_id: Vec, pub total_reward_points: u64, pub timestamp: DateTime, } From fcc9fb19d7c755d13e02e45b8faafe686e28f80e Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 16 Jul 2024 10:10:12 -0700 Subject: [PATCH 06/44] Fix reward test --- mobile_verifier/src/reward_shares.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 56695074d..1881dc638 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -877,6 +877,7 @@ mod test { speedtests::Speedtest, speedtests_average::SpeedtestAverage, subscriber_location::SubscriberValidatedLocations, + subscriber_mapping_event::VerifiedMappingEventShare, }; use chrono::{Duration, Utc}; use file_store::speedtest::CellSpeedtest; @@ -929,17 +930,24 @@ mod test { location_shares.push(n.encode_to_vec()); } + // simulate 10k vme shares + let mut vme_shares = VerifiedMappingEventShares::new(); + for n in 0..NUM_SUBSCRIBERS { + vme_shares.push(VerifiedMappingEventShare { + subscriber_id: n.encode_to_vec(), + total_reward_points: 30, + }); + } + // calculate discovery mapping rewards for a 24hr period let now = Utc::now(); let epoch = (now - Duration::hours(24))..now; - // translate location shares into discovery mapping shares - let mapping_shares = MapperShares::new(location_shares); + // translate location shares into shares + let shares = MapperShares::new(location_shares, vme_shares); let total_mappers_pool = reward_shares::get_scheduled_tokens_for_mappers(epoch.end - epoch.start); - let rewards_per_share = mapping_shares - .rewards_per_share(total_mappers_pool) - .unwrap(); + let rewards_per_share = shares.rewards_per_share(total_mappers_pool).unwrap(); // verify total rewards for the epoch let total_epoch_rewards = get_total_scheduled_tokens(epoch.end - epoch.start) @@ -955,12 +963,13 @@ mod test { .unwrap_or(0); assert_eq!(16_438_356_164_383, total_mapper_rewards); - let expected_reward_per_subscriber = total_mapper_rewards / NUM_SUBSCRIBERS; + // We have 2x more subscriber rewarded (for same amount as mappers 30) + let expected_reward_per_subscriber = total_mapper_rewards / NUM_SUBSCRIBERS / 2; // get the summed rewards allocated to subscribers for discovery location let mut allocated_mapper_rewards = 0_u64; for (reward_amount, subscriber_share) in - mapping_shares.into_subscriber_rewards(&epoch, rewards_per_share) + shares.into_subscriber_rewards(&epoch, rewards_per_share) { if let Some(MobileReward::SubscriberReward(r)) = subscriber_share.reward { assert_eq!(expected_reward_per_subscriber, r.discovery_location_amount); From e9ebcb86c88718d81f133e5d6fa369a26aad012a Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 16 Jul 2024 10:16:12 -0700 Subject: [PATCH 07/44] Clippy fix --- file_store/src/verified_mapping_event.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/file_store/src/verified_mapping_event.rs b/file_store/src/verified_mapping_event.rs index 61ad7d78d..9dd32d61a 100644 --- a/file_store/src/verified_mapping_event.rs +++ b/file_store/src/verified_mapping_event.rs @@ -34,7 +34,7 @@ impl From for VerifiedSubscriberMappingEventV1 { let timestamp = v.timestamp(); VerifiedSubscriberMappingEventV1 { subscriber_id: v.subscriber_id, - total_reward_points: v.total_reward_points as u64, + total_reward_points: v.total_reward_points, timestamp, } } From 36cebd561c91c9bab157ad41f46a1cb3cc2d9774 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 17 Jul 2024 10:07:24 -0700 Subject: [PATCH 08/44] Add ingestor test --- ingest/src/server_mobile.rs | 77 +++++++++++++++------ ingest/tests/common/mod.rs | 127 ++++++++++++++++++++++++++++++++++ ingest/tests/mobile_ingest.rs | 35 ++++++++++ 3 files changed, 218 insertions(+), 21 deletions(-) create mode 100644 ingest/tests/common/mod.rs create mode 100644 ingest/tests/mobile_ingest.rs diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 9b2fde8ac..8cd23167e 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -54,23 +54,7 @@ impl ManagedTask for GrpcServer { self: Box, shutdown: triggered::Listener, ) -> LocalBoxFuture<'static, anyhow::Result<()>> { - let api_token = self.api_token.clone(); - let address = self.address; - Box::pin(async move { - transport::Server::builder() - .layer(custom_tracing::grpc_layer::new_with_span(make_span)) - .layer(poc_metrics::request_layer!("ingest_server_grpc_connection")) - .add_service(poc_mobile::Server::with_interceptor( - *self, - move |req: Request<()>| match req.metadata().get("authorization") { - Some(t) if api_token == t => Ok(req), - _ => Err(Status::unauthenticated("No valid auth token")), - }, - )) - .serve_with_shutdown(address, shutdown) - .map_err(Error::from) - .await - }) + Box::pin(self.run(shutdown)) } } @@ -83,6 +67,57 @@ fn make_span(_request: &http::request::Request) -> } impl GrpcServer { + pub fn new( + heartbeat_report_sink: FileSinkClient, + wifi_heartbeat_report_sink: FileSinkClient, + speedtest_report_sink: FileSinkClient, + data_transfer_session_sink: FileSinkClient, + subscriber_location_report_sink: FileSinkClient, + radio_threshold_report_sink: FileSinkClient, + invalidated_radio_threshold_report_sink: FileSinkClient, + coverage_object_report_sink: FileSinkClient, + sp_boosted_rewards_ban_sink: FileSinkClient, + subscriber_mapping_event_sink: FileSinkClient, + required_network: Network, + address: SocketAddr, + api_token: MetadataValue, + ) -> Self { + GrpcServer { + heartbeat_report_sink, + wifi_heartbeat_report_sink, + speedtest_report_sink, + data_transfer_session_sink, + subscriber_location_report_sink, + radio_threshold_report_sink, + invalidated_radio_threshold_report_sink, + coverage_object_report_sink, + sp_boosted_rewards_ban_sink, + subscriber_mapping_event_sink, + required_network, + address, + api_token, + } + } + + pub async fn run(self, shutdown: triggered::Listener) -> anyhow::Result<()> { + let api_token = self.api_token.clone(); + let address = self.address; + + transport::Server::builder() + .layer(custom_tracing::grpc_layer::new_with_span(make_span)) + .layer(poc_metrics::request_layer!("ingest_server_grpc_connection")) + .add_service(poc_mobile::Server::with_interceptor( + self, + move |req: Request<()>| match req.metadata().get("authorization") { + Some(t) if api_token == t => Ok(req), + _ => Err(Status::unauthenticated("No valid auth token")), + }, + )) + .serve_with_shutdown(address, shutdown) + .map_err(Error::from) + .await + } + fn verify_network(&self, public_key: PublicKey) -> VerifyResult { if self.required_network == public_key.network { Ok(public_key) @@ -532,7 +567,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { bail!("expected valid api token in settings"); }; - let grpc_server = GrpcServer { + let grpc_server = GrpcServer::new( heartbeat_report_sink, wifi_heartbeat_report_sink, speedtest_report_sink, @@ -543,10 +578,10 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { coverage_object_report_sink, sp_boosted_rewards_ban_sink, subscriber_mapping_event_sink, - required_network: settings.network, - address: settings.listen_addr, + settings.network, + settings.listen_addr, api_token, - }; + ); tracing::info!( "grpc listening on {} and server mode {:?}", diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs new file mode 100644 index 000000000..aabd3ac0b --- /dev/null +++ b/ingest/tests/common/mod.rs @@ -0,0 +1,127 @@ +use anyhow::bail; +use backon::{ExponentialBuilder, Retryable}; +use file_store::file_sink::FileSinkClient; +use helium_crypto::{KeyTag, Keypair, Network, Sign}; +use helium_proto::services::poc_mobile::{ + Client as PocMobileClient, SubscriberMappingEventReqV1, SubscriberMappingEventResV1, +}; +use ingest::server_mobile::GrpcServer; +use prost::Message; +use rand::rngs::OsRng; +use std::{net::SocketAddr, sync::Arc}; +use tokio::{net::TcpListener, sync::mpsc::Receiver}; +use tonic::{ + metadata::{Ascii, MetadataValue}, + transport::Channel, + Request, +}; +use triggered::Trigger; + +pub async fn setup_mobile() -> anyhow::Result<( + TestClient, + Receiver, + Trigger, +)> { + let key_pair = generate_keypair(); + + let (file_sink_tx, file_sink_rx) = tokio::sync::mpsc::channel(10); + let file_sink = FileSinkClient::new(file_sink_tx, "test_file_sync"); + + let socket_addr = { + let tcp_listener = TcpListener::bind("127.0.0.1:0").await?; + tcp_listener.local_addr()? + }; + + let token = "api_token"; + let api_token = format!("Bearer {token}") + .parse::>() + .ok() + .unwrap(); + + let (trigger, listener) = triggered::trigger(); + + tokio::spawn(async move { + let grpc_server = GrpcServer::new( + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + file_sink.clone(), + Network::MainNet, + socket_addr, + api_token, + ); + + grpc_server.run(listener).await + }); + + let client = TestClient::new(socket_addr, key_pair, token.to_string()).await; + + Ok((client, file_sink_rx, trigger)) +} + +pub struct TestClient { + client: PocMobileClient, + key_pair: Arc, + authorization: MetadataValue, +} + +impl TestClient { + pub async fn new(socket_addr: SocketAddr, key_pair: Keypair, api_token: String) -> TestClient { + let client = (|| PocMobileClient::connect(format!("http://{socket_addr}"))) + .retry(&ExponentialBuilder::default()) + .await + .expect("client connect"); + + TestClient { + client, + key_pair: Arc::new(key_pair), + authorization: format!("Bearer {}", api_token).try_into().unwrap(), + } + } + + pub async fn submit_subscriber_mapping_event( + &mut self, + subscriber_id: Vec, + total_reward_points: u64, + ) -> anyhow::Result { + let mut req = SubscriberMappingEventReqV1 { + subscriber_id, + total_reward_points, + timestamp: 0, + pub_key: self.key_pair.public_key().to_vec(), + signature: vec![], + }; + + req.signature = self.key_pair.sign(&req.encode_to_vec()).expect("sign"); + + let mut request = Request::new(req); + let metadata = request.metadata_mut(); + + metadata.insert("authorization", self.authorization.clone()); + + let res = self.client.submit_subscriber_mapping_event(request).await?; + + Ok(res.into_inner()) + } +} + +pub fn generate_keypair() -> Keypair { + Keypair::generate(KeyTag::default(), &mut OsRng) +} + +pub async fn recv(mut rx: Receiver) -> anyhow::Result> { + match rx.recv().await { + Some(msg) => match msg { + file_store::file_sink::Message::Commit(_) => bail!("got Commit"), + file_store::file_sink::Message::Rollback(_) => bail!("got Rollback"), + file_store::file_sink::Message::Data(_, data) => Ok(data), + }, + None => bail!("got none"), + } +} diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs new file mode 100644 index 000000000..031d43b3a --- /dev/null +++ b/ingest/tests/mobile_ingest.rs @@ -0,0 +1,35 @@ +use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventV1; +use prost::Message; + +mod common; + +#[tokio::test] +async fn submit_subscriber_mapping_event() -> anyhow::Result<()> { + let (mut client, file_sink_rx, trigger) = common::setup_mobile().await?; + + let subscriber_id = vec![0]; + let total_reward_points = 100; + + let res = client + .submit_subscriber_mapping_event(subscriber_id.clone(), total_reward_points) + .await; + + assert!(res.is_ok()); + + let timestamp: String = res.unwrap().id; + + match common::recv(file_sink_rx).await { + Ok(data) => { + let event = VerifiedSubscriberMappingEventV1::decode(data.as_slice()) + .expect("unable to decode into VerifiedSubscriberMappingEventV1"); + + assert_eq!(subscriber_id, event.subscriber_id); + assert_eq!(total_reward_points, event.total_reward_points); + assert_eq!(timestamp, event.timestamp.to_string()) + } + Err(e) => panic!("got error {e}"), + } + + trigger.trigger(); + Ok(()) +} From dba08732bcdda4b91cc363080f58179228260c39 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 17 Jul 2024 11:44:31 -0700 Subject: [PATCH 09/44] Add mobile verifier test --- file_store/src/verified_mapping_event.rs | 12 ++- mobile_verifier/tests/integrations/main.rs | 1 + .../integrations/subscriber_mapping_event.rs | 95 +++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 mobile_verifier/tests/integrations/subscriber_mapping_event.rs diff --git a/file_store/src/verified_mapping_event.rs b/file_store/src/verified_mapping_event.rs index 9dd32d61a..46b6b635c 100644 --- a/file_store/src/verified_mapping_event.rs +++ b/file_store/src/verified_mapping_event.rs @@ -5,8 +5,9 @@ use crate::{ use chrono::{DateTime, Utc}; use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventV1; use serde::{Deserialize, Serialize}; +use sqlx::Row; -#[derive(Clone, Deserialize, Serialize, Debug)] +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] pub struct VerifiedSubscriberMappingEvent { pub subscriber_id: Vec, pub total_reward_points: u64, @@ -51,3 +52,12 @@ impl TryFrom for VerifiedSubscriberMappingEven }) } } +impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for VerifiedSubscriberMappingEvent { + fn from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { + Ok(Self { + subscriber_id: row.get::, &str>("subscriber_id"), + total_reward_points: row.get::("total_reward_points") as u64, + timestamp: row.get::, &str>("timestamp"), + }) + } +} diff --git a/mobile_verifier/tests/integrations/main.rs b/mobile_verifier/tests/integrations/main.rs index 6505c40ad..d13c50319 100644 --- a/mobile_verifier/tests/integrations/main.rs +++ b/mobile_verifier/tests/integrations/main.rs @@ -11,3 +11,4 @@ mod rewarder_poc_dc; mod rewarder_sp_rewards; mod seniority; mod speedtests; +mod subscriber_mapping_event; diff --git a/mobile_verifier/tests/integrations/subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_mapping_event.rs new file mode 100644 index 000000000..b28f99325 --- /dev/null +++ b/mobile_verifier/tests/integrations/subscriber_mapping_event.rs @@ -0,0 +1,95 @@ +use chrono::Utc; +use file_store::{ + file_info_poller::FileInfoStream, verified_mapping_event::VerifiedSubscriberMappingEvent, + FileInfo, +}; +use mobile_verifier::subscriber_mapping_event::SubscriberMappingEventDeamon; +use sqlx::{PgPool, Pool, Postgres}; + +#[sqlx::test] +async fn main_test(pool: PgPool) -> anyhow::Result<()> { + let (tx, rx) = tokio::sync::mpsc::channel(10); + let (trigger, listener) = triggered::trigger(); + let task_pool = pool.clone(); + + tokio::spawn(async move { + let deamon = SubscriberMappingEventDeamon::new(task_pool, rx); + deamon.run(listener).await.expect("failed to complete task"); + }); + + let (fis1, vsmes) = file_info_stream(); + tx.send(fis1).await?; + + let mut retry = 0; + const MAX_RETRIES: u32 = 10; + const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + while retry <= MAX_RETRIES { + let saved_vmes = select_events(&pool).await?; + if saved_vmes.len() == 2 { + assert_eq!(2, saved_vmes.len()); + assert!(vsmes.iter().all(|e| saved_vmes.contains(e))); + break; + } else { + tracing::debug!("wrong saved_vmes.len() {}", saved_vmes.len()); + retry += 1; + tokio::time::sleep(RETRY_WAIT).await; + } + } + + assert!( + retry <= MAX_RETRIES, + "Exceeded maximum retries: {}", + MAX_RETRIES + ); + + trigger.trigger(); + + Ok(()) +} + +fn file_info_stream() -> ( + FileInfoStream, + Vec, +) { + let file_info = FileInfo { + key: "test_file_info".to_string(), + prefix: "verified_mapping_event".to_string(), + timestamp: Utc::now(), + size: 0, + }; + + let vsmes = vec![ + VerifiedSubscriberMappingEvent { + subscriber_id: vec![0], + total_reward_points: 100, + timestamp: Utc::now(), + }, + VerifiedSubscriberMappingEvent { + subscriber_id: vec![1], + total_reward_points: 101, + timestamp: Utc::now(), + }, + ]; + ( + FileInfoStream::new("default".to_string(), file_info, vsmes.clone()), + vsmes, + ) +} + +pub async fn select_events( + pool: &Pool, +) -> anyhow::Result> { + Ok(sqlx::query_as::<_, VerifiedSubscriberMappingEvent>( + r#" + SELECT + subscriber_id, + total_reward_points, + timestamp + FROM verified_mapping_event + "#, + ) + .fetch_all(pool) + .await? + .into_iter() + .collect::>()) +} From b88cc655ee884a92156293677a3664c1df6d097f Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 17 Jul 2024 13:44:36 -0700 Subject: [PATCH 10/44] Add more testing --- .../src/subscriber_mapping_event.rs | 5 +- .../integrations/subscriber_mapping_event.rs | 53 ++++++++++++++++--- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/mobile_verifier/src/subscriber_mapping_event.rs b/mobile_verifier/src/subscriber_mapping_event.rs index 66dedb4b2..b791b5063 100644 --- a/mobile_verifier/src/subscriber_mapping_event.rs +++ b/mobile_verifier/src/subscriber_mapping_event.rs @@ -1,5 +1,3 @@ -use std::ops::Range; - use crate::Settings; use chrono::{DateTime, Duration, Utc}; use file_store::{ @@ -10,6 +8,7 @@ use file_store::{ }; use futures::{stream::StreamExt, TryStreamExt}; use sqlx::{Pool, Postgres, Transaction}; +use std::ops::Range; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; @@ -116,7 +115,7 @@ async fn save_event( const SUBSCRIBER_REWARD_PERIOD_IN_DAYS: i64 = 1; pub type VerifiedMappingEventShares = Vec; -#[derive(sqlx::FromRow)] +#[derive(sqlx::FromRow, PartialEq, Debug)] pub struct VerifiedMappingEventShare { pub subscriber_id: Vec, pub total_reward_points: i64, diff --git a/mobile_verifier/tests/integrations/subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_mapping_event.rs index b28f99325..c957bfabe 100644 --- a/mobile_verifier/tests/integrations/subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/subscriber_mapping_event.rs @@ -1,10 +1,14 @@ -use chrono::Utc; +use chrono::{Duration, Utc}; use file_store::{ file_info_poller::FileInfoStream, verified_mapping_event::VerifiedSubscriberMappingEvent, FileInfo, }; -use mobile_verifier::subscriber_mapping_event::SubscriberMappingEventDeamon; +use mobile_verifier::subscriber_mapping_event::{ + aggregate_verified_mapping_events, SubscriberMappingEventDeamon, VerifiedMappingEventShare, + VerifiedMappingEventShares, +}; use sqlx::{PgPool, Pool, Postgres}; +use std::{collections::HashMap, ops::Range}; #[sqlx::test] async fn main_test(pool: PgPool) -> anyhow::Result<()> { @@ -17,16 +21,15 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { deamon.run(listener).await.expect("failed to complete task"); }); - let (fis1, vsmes) = file_info_stream(); - tx.send(fis1).await?; + let (fis, vsmes) = file_info_stream(); + tx.send(fis).await?; let mut retry = 0; const MAX_RETRIES: u32 = 10; const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); while retry <= MAX_RETRIES { let saved_vmes = select_events(&pool).await?; - if saved_vmes.len() == 2 { - assert_eq!(2, saved_vmes.len()); + if vsmes.len() == saved_vmes.len() { assert!(vsmes.iter().all(|e| saved_vmes.contains(e))); break; } else { @@ -42,6 +45,18 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { MAX_RETRIES ); + // Testing aggregate_verified_mapping_events now + let reward_period = Range { + start: Utc::now() - Duration::days(1), + end: Utc::now(), + }; + let mut avmes = aggregate_verified_mapping_events(&pool, &reward_period).await?; + + assert_eq!( + vsmes_to_shares(vsmes).sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)), + avmes.sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)) + ); + trigger.trigger(); Ok(()) @@ -67,6 +82,11 @@ fn file_info_stream() -> ( VerifiedSubscriberMappingEvent { subscriber_id: vec![1], total_reward_points: 101, + timestamp: Utc::now() - Duration::seconds(10), + }, + VerifiedSubscriberMappingEvent { + subscriber_id: vec![1], + total_reward_points: 99, timestamp: Utc::now(), }, ]; @@ -76,7 +96,26 @@ fn file_info_stream() -> ( ) } -pub async fn select_events( +fn vsmes_to_shares(vsmes: Vec) -> VerifiedMappingEventShares { + let mut reward_map: HashMap, i64> = HashMap::new(); + + for vsme in vsmes { + let entry = reward_map.entry(vsme.subscriber_id).or_insert(0); + *entry += vsme.total_reward_points as i64; + } + + reward_map + .into_iter() + .map( + |(subscriber_id, total_reward_points)| VerifiedMappingEventShare { + subscriber_id, + total_reward_points, + }, + ) + .collect() +} + +async fn select_events( pool: &Pool, ) -> anyhow::Result> { Ok(sqlx::query_as::<_, VerifiedSubscriberMappingEvent>( From 581651b1b35e56135b3aa8af2a33bb304e7d8a7a Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 18 Jul 2024 11:15:56 -0700 Subject: [PATCH 11/44] Updgrade to support report --- file_store/src/file_info.rs | 17 +- file_store/src/lib.rs | 3 +- file_store/src/traits/msg_verify.rs | 2 +- ...s => verified_subscriber_mapping_event.rs} | 17 +- ..._subscriber_mapping_event_ingest_report.rs | 46 +++++ ingest/src/server_mobile.rs | 36 ++-- ingest/tests/common/mod.rs | 16 +- ingest/tests/mobile_ingest.rs | 24 ++- mobile_verifier/src/cli/server.rs | 7 +- mobile_verifier/src/lib.rs | 2 +- mobile_verifier/src/reward_shares.rs | 4 +- mobile_verifier/src/rewarder.rs | 5 +- ...s => verified_subscriber_mapping_event.rs} | 29 ++-- mobile_verifier/tests/integrations/main.rs | 2 +- .../integrations/subscriber_mapping_event.rs | 134 --------------- .../verified_subscriber_mapping_event.rs | 158 ++++++++++++++++++ 16 files changed, 302 insertions(+), 200 deletions(-) rename file_store/src/{verified_mapping_event.rs => verified_subscriber_mapping_event.rs} (81%) create mode 100644 file_store/src/verified_subscriber_mapping_event_ingest_report.rs rename mobile_verifier/src/{subscriber_mapping_event.rs => verified_subscriber_mapping_event.rs} (83%) delete mode 100644 mobile_verifier/tests/integrations/subscriber_mapping_event.rs create mode 100644 mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index 2f1631c82..af2954221 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -160,7 +160,8 @@ pub const LANDTYPE_DATA_SET: &str = "landtype"; pub const SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "service_provider_boosted_rewards_banned_radio"; pub const VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "verified_service_provider_boosted_rewards_banned_radio"; -pub const VERIFIED_SUBSCRIBER_MAPPING_EVENT: &str = "verified_subscriber_mapping_event"; +pub const VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT: &str = + "verified_subscriber_mapping_event_report"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -215,7 +216,7 @@ pub enum FileType { VerifiedInvalidatedRadioThresholdIngestReport, SPBoostedRewardsBannedRadioIngestReport, VerifiedSPBoostedRewardsBannedRadioIngestReport, - VerifiedSubscriberMappingEvent, + VerifiedSubscriberMappingEventIngestReportV1, } impl fmt::Display for FileType { @@ -281,7 +282,9 @@ impl fmt::Display for FileType { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } - Self::VerifiedSubscriberMappingEvent => VERIFIED_SUBSCRIBER_MAPPING_EVENT, + Self::VerifiedSubscriberMappingEventIngestReportV1 => { + VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT + } }; f.write_str(s) } @@ -350,7 +353,9 @@ impl FileType { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } - Self::VerifiedSubscriberMappingEvent => VERIFIED_SUBSCRIBER_MAPPING_EVENT, + Self::VerifiedSubscriberMappingEventIngestReportV1 => { + VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT + } } } } @@ -419,7 +424,9 @@ impl FromStr for FileType { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO => { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport } - VERIFIED_SUBSCRIBER_MAPPING_EVENT => Self::VerifiedSubscriberMappingEvent, + VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT => { + Self::VerifiedSubscriberMappingEventIngestReportV1 + } _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index bdf850866..30110f3d4 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -24,7 +24,8 @@ pub mod reward_manifest; mod settings; pub mod speedtest; pub mod traits; -pub mod verified_mapping_event; +pub mod verified_subscriber_mapping_event; +pub mod verified_subscriber_mapping_event_ingest_report; pub mod wifi_heartbeat; pub use crate::file_store::FileStore; diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index e579787b8..fb75dd78d 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -94,7 +94,7 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamResV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); -impl_msg_verify!(poc_mobile::SubscriberMappingEventReqV1, signature); +impl_msg_verify!(poc_mobile::VerifiedSubscriberMappingEventReqV1, signature); #[cfg(test)] mod test { diff --git a/file_store/src/verified_mapping_event.rs b/file_store/src/verified_subscriber_mapping_event.rs similarity index 81% rename from file_store/src/verified_mapping_event.rs rename to file_store/src/verified_subscriber_mapping_event.rs index 46b6b635c..1376f3066 100644 --- a/file_store/src/verified_mapping_event.rs +++ b/file_store/src/verified_subscriber_mapping_event.rs @@ -3,7 +3,7 @@ use crate::{ Error, Result, }; use chrono::{DateTime, Utc}; -use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventV1; +use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventReqV1; use serde::{Deserialize, Serialize}; use sqlx::Row; @@ -15,10 +15,10 @@ pub struct VerifiedSubscriberMappingEvent { } impl MsgDecode for VerifiedSubscriberMappingEvent { - type Msg = VerifiedSubscriberMappingEventV1; + type Msg = VerifiedSubscriberMappingEventReqV1; } -impl MsgTimestamp>> for VerifiedSubscriberMappingEventV1 { +impl MsgTimestamp>> for VerifiedSubscriberMappingEventReqV1 { fn timestamp(&self) -> Result> { self.timestamp.to_timestamp() } @@ -30,20 +30,22 @@ impl MsgTimestamp for VerifiedSubscriberMappingEvent { } } -impl From for VerifiedSubscriberMappingEventV1 { +impl From for VerifiedSubscriberMappingEventReqV1 { fn from(v: VerifiedSubscriberMappingEvent) -> Self { let timestamp = v.timestamp(); - VerifiedSubscriberMappingEventV1 { + VerifiedSubscriberMappingEventReqV1 { subscriber_id: v.subscriber_id, total_reward_points: v.total_reward_points, timestamp, + verification_mapping_pubkey: vec![], + signature: vec![], } } } -impl TryFrom for VerifiedSubscriberMappingEvent { +impl TryFrom for VerifiedSubscriberMappingEvent { type Error = Error; - fn try_from(v: VerifiedSubscriberMappingEventV1) -> Result { + fn try_from(v: VerifiedSubscriberMappingEventReqV1) -> Result { let timestamp = v.timestamp()?; Ok(Self { subscriber_id: v.subscriber_id, @@ -52,6 +54,7 @@ impl TryFrom for VerifiedSubscriberMappingEven }) } } + impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for VerifiedSubscriberMappingEvent { fn from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { Ok(Self { diff --git a/file_store/src/verified_subscriber_mapping_event_ingest_report.rs b/file_store/src/verified_subscriber_mapping_event_ingest_report.rs new file mode 100644 index 000000000..88d601e5e --- /dev/null +++ b/file_store/src/verified_subscriber_mapping_event_ingest_report.rs @@ -0,0 +1,46 @@ +use crate::{ + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventIngestReportV1; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct VerifiedSubscriberMappingEventIngestReport { + pub received_timestamp: DateTime, + pub report: Option, +} + +impl MsgDecode for VerifiedSubscriberMappingEventIngestReport { + type Msg = VerifiedSubscriberMappingEventIngestReportV1; +} + +impl MsgTimestamp>> for VerifiedSubscriberMappingEventIngestReportV1 { + fn timestamp(&self) -> Result> { + self.received_timestamp.to_timestamp() + } +} + +impl MsgTimestamp for VerifiedSubscriberMappingEventIngestReport { + fn timestamp(&self) -> u64 { + self.received_timestamp.encode_timestamp() + } +} + +impl TryFrom + for VerifiedSubscriberMappingEventIngestReport +{ + type Error = Error; + fn try_from(v: VerifiedSubscriberMappingEventIngestReportV1) -> Result { + let received_timestamp = v.timestamp()?; + Ok(Self { + received_timestamp, + report: match v.report { + None => None, + Some(event) => Some(event.try_into()?), + }, + }) + } +} diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 8cd23167e..97e537820 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -19,9 +19,10 @@ use helium_proto::services::poc_mobile::{ RadioThresholdReportRespV1, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, ServiceProviderBoostedRewardsBannedRadioReqV1, ServiceProviderBoostedRewardsBannedRadioRespV1, SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, SubscriberLocationIngestReportV1, - SubscriberLocationReqV1, SubscriberLocationRespV1, SubscriberMappingEventReqV1, - SubscriberMappingEventResV1, VerifiedSubscriberMappingEventV1, WifiHeartbeatIngestReportV1, - WifiHeartbeatReqV1, WifiHeartbeatRespV1, + SubscriberLocationReqV1, SubscriberLocationRespV1, + VerifiedSubscriberMappingEventIngestReportV1, VerifiedSubscriberMappingEventReqV1, + VerifiedSubscriberMappingEventResV1, WifiHeartbeatIngestReportV1, WifiHeartbeatReqV1, + WifiHeartbeatRespV1, }; use std::{net::SocketAddr, path::Path}; use task_manager::{ManagedTask, TaskManager}; @@ -67,6 +68,7 @@ fn make_span(_request: &http::request::Request) -> } impl GrpcServer { + #[allow(clippy::too_many_arguments)] pub fn new( heartbeat_report_sink: FileSinkClient, wifi_heartbeat_report_sink: FileSinkClient, @@ -406,31 +408,30 @@ impl poc_mobile::PocMobile for GrpcServer { )) } - async fn submit_subscriber_mapping_event( + async fn submit_verified_subscriber_mapping_event( &self, - request: Request, - ) -> GrpcResult { + request: Request, + ) -> GrpcResult { let timestamp: u64 = Utc::now().timestamp_millis() as u64; - let event: SubscriberMappingEventReqV1 = request.into_inner(); + let event: VerifiedSubscriberMappingEventReqV1 = request.into_inner(); let subscriber_id = event.subscriber_id.clone(); custom_tracing::record("subscriber_id", bs58::encode(&subscriber_id).into_string()); - custom_tracing::record_b58("pub_key", &event.pub_key); + custom_tracing::record_b58("pub_key", &event.verification_mapping_pubkey); let report = self - .verify_public_key(event.pub_key.as_ref()) + .verify_public_key(event.verification_mapping_pubkey.as_ref()) .and_then(|public_key| self.verify_network(public_key)) .and_then(|public_key| self.verify_signature(public_key, event)) - .map(|(_, event)| VerifiedSubscriberMappingEventV1 { - subscriber_id: event.subscriber_id, - total_reward_points: event.total_reward_points, - timestamp, + .map(|(_, event)| VerifiedSubscriberMappingEventIngestReportV1 { + received_timestamp: timestamp, + report: Some(event), })?; _ = self.subscriber_mapping_event_sink.write(report, []).await; let id = timestamp.to_string(); - Ok(Response::new(SubscriberMappingEventResV1 { id })) + Ok(Response::new(VerifiedSubscriberMappingEventResV1 { id })) } } @@ -550,10 +551,13 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { let (subscriber_mapping_event_sink, subscriber_mapping_event_server) = file_sink::FileSinkBuilder::new( - FileType::VerifiedSubscriberMappingEvent, + FileType::VerifiedSubscriberMappingEventIngestReportV1, store_base_path, file_upload.clone(), - concat!(env!("CARGO_PKG_NAME"), "_verified_subscriber_mapping_event"), + concat!( + env!("CARGO_PKG_NAME"), + "_verified_subscriber_mapping_event_ingest_report" + ), ) .roll_time(settings.roll_time) .create() diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index aabd3ac0b..0affcc342 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -3,7 +3,8 @@ use backon::{ExponentialBuilder, Retryable}; use file_store::file_sink::FileSinkClient; use helium_crypto::{KeyTag, Keypair, Network, Sign}; use helium_proto::services::poc_mobile::{ - Client as PocMobileClient, SubscriberMappingEventReqV1, SubscriberMappingEventResV1, + Client as PocMobileClient, VerifiedSubscriberMappingEventReqV1, + VerifiedSubscriberMappingEventResV1, }; use ingest::server_mobile::GrpcServer; use prost::Message; @@ -85,16 +86,16 @@ impl TestClient { } } - pub async fn submit_subscriber_mapping_event( + pub async fn submit_verified_subscriber_mapping_event( &mut self, subscriber_id: Vec, total_reward_points: u64, - ) -> anyhow::Result { - let mut req = SubscriberMappingEventReqV1 { + ) -> anyhow::Result { + let mut req = VerifiedSubscriberMappingEventReqV1 { subscriber_id, total_reward_points, timestamp: 0, - pub_key: self.key_pair.public_key().to_vec(), + verification_mapping_pubkey: self.key_pair.public_key().to_vec(), signature: vec![], }; @@ -105,7 +106,10 @@ impl TestClient { metadata.insert("authorization", self.authorization.clone()); - let res = self.client.submit_subscriber_mapping_event(request).await?; + let res = self + .client + .submit_verified_subscriber_mapping_event(request) + .await?; Ok(res.into_inner()) } diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index 031d43b3a..a169afa56 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -1,17 +1,17 @@ -use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventV1; +use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventIngestReportV1; use prost::Message; mod common; #[tokio::test] -async fn submit_subscriber_mapping_event() -> anyhow::Result<()> { +async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { let (mut client, file_sink_rx, trigger) = common::setup_mobile().await?; let subscriber_id = vec![0]; let total_reward_points = 100; let res = client - .submit_subscriber_mapping_event(subscriber_id.clone(), total_reward_points) + .submit_verified_subscriber_mapping_event(subscriber_id.clone(), total_reward_points) .await; assert!(res.is_ok()); @@ -20,12 +20,18 @@ async fn submit_subscriber_mapping_event() -> anyhow::Result<()> { match common::recv(file_sink_rx).await { Ok(data) => { - let event = VerifiedSubscriberMappingEventV1::decode(data.as_slice()) - .expect("unable to decode into VerifiedSubscriberMappingEventV1"); - - assert_eq!(subscriber_id, event.subscriber_id); - assert_eq!(total_reward_points, event.total_reward_points); - assert_eq!(timestamp, event.timestamp.to_string()) + let report = VerifiedSubscriberMappingEventIngestReportV1::decode(data.as_slice()) + .expect("unable to decode into VerifiedSubscriberMappingEventIngestReportV1"); + + assert_eq!(timestamp, report.received_timestamp.to_string()); + + match report.report { + None => panic!("No report found"), + Some(event) => { + assert_eq!(subscriber_id, event.subscriber_id); + assert_eq!(total_reward_points, event.total_reward_points); + } + } } Err(e) => panic!("got error {e}"), } diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 52858dbe4..b5d2b7f7b 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -11,8 +11,9 @@ use crate::{ sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, - subscriber_mapping_event::SubscriberMappingEventDeamon, - telemetry, Settings, + telemetry, + verified_subscriber_mapping_event::VerifiedSubscriberMappingEventDeamon, + Settings, }; use anyhow::Result; use file_store::{file_sink, file_upload, FileStore, FileType}; @@ -143,7 +144,7 @@ impl Cmd { .await?, ) .add_task( - SubscriberMappingEventDeamon::create_managed_task( + VerifiedSubscriberMappingEventDeamon::create_managed_task( pool.clone(), settings, report_ingest.clone(), diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 8ff15f803..f7b182503 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -14,8 +14,8 @@ pub mod sp_boosted_rewards_bans; pub mod speedtests; pub mod speedtests_average; pub mod subscriber_location; -pub mod subscriber_mapping_event; pub mod telemetry; +pub mod verified_subscriber_mapping_event; pub use settings::Settings; diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 1881dc638..e065224b1 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -6,7 +6,7 @@ use crate::{ seniority::Seniority, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, - subscriber_mapping_event::VerifiedMappingEventShares, + verified_subscriber_mapping_event::VerifiedMappingEventShares, }; use chrono::{DateTime, Duration, Utc}; use coverage_point_calculator::SPBoostedRewardEligibility; @@ -877,7 +877,7 @@ mod test { speedtests::Speedtest, speedtests_average::SpeedtestAverage, subscriber_location::SubscriberValidatedLocations, - subscriber_mapping_event::VerifiedMappingEventShare, + verified_subscriber_mapping_event::VerifiedMappingEventShare, }; use chrono::{Duration, Utc}; use file_store::speedtest::CellSpeedtest; diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 0ecbf6309..f152400d0 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -9,7 +9,7 @@ use crate::{ }, sp_boosted_rewards_bans, speedtests, speedtests_average::SpeedtestAverages, - subscriber_location, subscriber_mapping_event, telemetry, Settings, + subscriber_location, telemetry, verified_subscriber_mapping_event, Settings, }; use anyhow::bail; use chrono::{DateTime, TimeZone, Utc}; @@ -513,7 +513,8 @@ pub async fn reward_mappers( subscriber_location::aggregate_location_shares(pool, reward_period).await?; let vme_shares = - subscriber_mapping_event::aggregate_verified_mapping_events(pool, reward_period).await?; + verified_subscriber_mapping_event::aggregate_verified_mapping_events(pool, reward_period) + .await?; // determine mapping shares based on location shares and data transferred let mapping_shares = MapperShares::new(location_shares, vme_shares); diff --git a/mobile_verifier/src/subscriber_mapping_event.rs b/mobile_verifier/src/verified_subscriber_mapping_event.rs similarity index 83% rename from mobile_verifier/src/subscriber_mapping_event.rs rename to mobile_verifier/src/verified_subscriber_mapping_event.rs index b791b5063..426f0061b 100644 --- a/mobile_verifier/src/subscriber_mapping_event.rs +++ b/mobile_verifier/src/verified_subscriber_mapping_event.rs @@ -3,7 +3,8 @@ use chrono::{DateTime, Duration, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, file_source, - verified_mapping_event::VerifiedSubscriberMappingEvent, + verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, + verified_subscriber_mapping_event_ingest_report::VerifiedSubscriberMappingEventIngestReport, FileStore, FileType, }; use futures::{stream::StreamExt, TryStreamExt}; @@ -12,15 +13,15 @@ use std::ops::Range; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; -pub struct SubscriberMappingEventDeamon { +pub struct VerifiedSubscriberMappingEventDeamon { pool: Pool, - events: Receiver>, + events: Receiver>, } -impl SubscriberMappingEventDeamon { +impl VerifiedSubscriberMappingEventDeamon { pub fn new( pool: Pool, - events: Receiver>, + events: Receiver>, ) -> Self { Self { pool, events } } @@ -31,11 +32,11 @@ impl SubscriberMappingEventDeamon { file_store: FileStore, ) -> anyhow::Result { let (events, event_server) = - file_source::continuous_source::() + file_source::continuous_source::() .state(pool.clone()) .store(file_store) .lookback(LookbackBehavior::StartAfter(settings.start_after)) - .prefix(FileType::VerifiedSubscriberMappingEvent.to_string()) + .prefix(FileType::VerifiedSubscriberMappingEventIngestReportV1.to_string()) .create() .await?; @@ -66,17 +67,20 @@ impl SubscriberMappingEventDeamon { async fn process_file( &self, - file: FileInfoStream, + file: FileInfoStream, ) -> anyhow::Result<()> { tracing::info!( "Processing Verified Mapping Event file {}", file.file_info.key ); let mut transaction = self.pool.begin().await?; - let mut events = file.into_stream(&mut transaction).await?; + let mut reports = file.into_stream(&mut transaction).await?; - while let Some(event) = events.next().await { - save_event(&event, &mut transaction).await?; + while let Some(report) = reports.next().await { + match report.report { + Some(event) => save_event(&event, &mut transaction).await?, + None => tracing::debug!("no report"), + } } transaction.commit().await?; @@ -84,7 +88,7 @@ impl SubscriberMappingEventDeamon { } } -impl ManagedTask for SubscriberMappingEventDeamon { +impl ManagedTask for VerifiedSubscriberMappingEventDeamon { fn start_task( self: Box, shutdown: triggered::Listener, @@ -109,6 +113,7 @@ async fn save_event( .bind(event.timestamp) .execute(exec) .await?; + Ok(()) } diff --git a/mobile_verifier/tests/integrations/main.rs b/mobile_verifier/tests/integrations/main.rs index d13c50319..67ea58951 100644 --- a/mobile_verifier/tests/integrations/main.rs +++ b/mobile_verifier/tests/integrations/main.rs @@ -11,4 +11,4 @@ mod rewarder_poc_dc; mod rewarder_sp_rewards; mod seniority; mod speedtests; -mod subscriber_mapping_event; +mod verified_subscriber_mapping_event; diff --git a/mobile_verifier/tests/integrations/subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_mapping_event.rs deleted file mode 100644 index c957bfabe..000000000 --- a/mobile_verifier/tests/integrations/subscriber_mapping_event.rs +++ /dev/null @@ -1,134 +0,0 @@ -use chrono::{Duration, Utc}; -use file_store::{ - file_info_poller::FileInfoStream, verified_mapping_event::VerifiedSubscriberMappingEvent, - FileInfo, -}; -use mobile_verifier::subscriber_mapping_event::{ - aggregate_verified_mapping_events, SubscriberMappingEventDeamon, VerifiedMappingEventShare, - VerifiedMappingEventShares, -}; -use sqlx::{PgPool, Pool, Postgres}; -use std::{collections::HashMap, ops::Range}; - -#[sqlx::test] -async fn main_test(pool: PgPool) -> anyhow::Result<()> { - let (tx, rx) = tokio::sync::mpsc::channel(10); - let (trigger, listener) = triggered::trigger(); - let task_pool = pool.clone(); - - tokio::spawn(async move { - let deamon = SubscriberMappingEventDeamon::new(task_pool, rx); - deamon.run(listener).await.expect("failed to complete task"); - }); - - let (fis, vsmes) = file_info_stream(); - tx.send(fis).await?; - - let mut retry = 0; - const MAX_RETRIES: u32 = 10; - const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); - while retry <= MAX_RETRIES { - let saved_vmes = select_events(&pool).await?; - if vsmes.len() == saved_vmes.len() { - assert!(vsmes.iter().all(|e| saved_vmes.contains(e))); - break; - } else { - tracing::debug!("wrong saved_vmes.len() {}", saved_vmes.len()); - retry += 1; - tokio::time::sleep(RETRY_WAIT).await; - } - } - - assert!( - retry <= MAX_RETRIES, - "Exceeded maximum retries: {}", - MAX_RETRIES - ); - - // Testing aggregate_verified_mapping_events now - let reward_period = Range { - start: Utc::now() - Duration::days(1), - end: Utc::now(), - }; - let mut avmes = aggregate_verified_mapping_events(&pool, &reward_period).await?; - - assert_eq!( - vsmes_to_shares(vsmes).sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)), - avmes.sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)) - ); - - trigger.trigger(); - - Ok(()) -} - -fn file_info_stream() -> ( - FileInfoStream, - Vec, -) { - let file_info = FileInfo { - key: "test_file_info".to_string(), - prefix: "verified_mapping_event".to_string(), - timestamp: Utc::now(), - size: 0, - }; - - let vsmes = vec![ - VerifiedSubscriberMappingEvent { - subscriber_id: vec![0], - total_reward_points: 100, - timestamp: Utc::now(), - }, - VerifiedSubscriberMappingEvent { - subscriber_id: vec![1], - total_reward_points: 101, - timestamp: Utc::now() - Duration::seconds(10), - }, - VerifiedSubscriberMappingEvent { - subscriber_id: vec![1], - total_reward_points: 99, - timestamp: Utc::now(), - }, - ]; - ( - FileInfoStream::new("default".to_string(), file_info, vsmes.clone()), - vsmes, - ) -} - -fn vsmes_to_shares(vsmes: Vec) -> VerifiedMappingEventShares { - let mut reward_map: HashMap, i64> = HashMap::new(); - - for vsme in vsmes { - let entry = reward_map.entry(vsme.subscriber_id).or_insert(0); - *entry += vsme.total_reward_points as i64; - } - - reward_map - .into_iter() - .map( - |(subscriber_id, total_reward_points)| VerifiedMappingEventShare { - subscriber_id, - total_reward_points, - }, - ) - .collect() -} - -async fn select_events( - pool: &Pool, -) -> anyhow::Result> { - Ok(sqlx::query_as::<_, VerifiedSubscriberMappingEvent>( - r#" - SELECT - subscriber_id, - total_reward_points, - timestamp - FROM verified_mapping_event - "#, - ) - .fetch_all(pool) - .await? - .into_iter() - .collect::>()) -} diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs new file mode 100644 index 000000000..a8130ac0e --- /dev/null +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -0,0 +1,158 @@ +use chrono::{Duration, Utc}; +use file_store::{ + file_info_poller::FileInfoStream, + verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, + verified_subscriber_mapping_event_ingest_report::VerifiedSubscriberMappingEventIngestReport, + FileInfo, +}; +use mobile_verifier::verified_subscriber_mapping_event::{ + aggregate_verified_mapping_events, VerifiedMappingEventShare, VerifiedMappingEventShares, + VerifiedSubscriberMappingEventDeamon, +}; +use sqlx::{PgPool, Pool, Postgres}; +use std::{collections::HashMap, ops::Range}; + +#[sqlx::test] +async fn main_test(pool: PgPool) -> anyhow::Result<()> { + let (tx, rx) = tokio::sync::mpsc::channel(10); + let (trigger, listener) = triggered::trigger(); + let task_pool = pool.clone(); + + tokio::spawn(async move { + let deamon = VerifiedSubscriberMappingEventDeamon::new(task_pool, rx); + deamon.run(listener).await.expect("failed to complete task"); + }); + + let (fis, reports) = file_info_stream(); + tx.send(fis).await?; + + let mut retry = 0; + const MAX_RETRIES: u32 = 10; + const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + while retry <= MAX_RETRIES { + let saved_vmes = select_events(&pool).await?; + if reports.len() == saved_vmes.len() { + assert!(reports.iter().all(|r| { + match r.report.clone() { + None => false, + Some(event) => saved_vmes.contains(&event), + } + })); + break; + } else { + tracing::debug!("wrong saved_vmes.len() {}", saved_vmes.len()); + retry += 1; + tokio::time::sleep(RETRY_WAIT).await; + } + } + + assert!( + retry <= MAX_RETRIES, + "Exceeded maximum retries: {}", + MAX_RETRIES + ); + + // Testing aggregate_verified_mapping_events now + let reward_period = Range { + start: Utc::now() - Duration::days(1), + end: Utc::now(), + }; + let mut shares_from_reports = reports_to_shares(reports); + shares_from_reports.sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)); + + let mut shares = aggregate_verified_mapping_events(&pool, &reward_period).await?; + shares.sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)); + + assert_eq!(shares_from_reports, shares); + + trigger.trigger(); + + Ok(()) +} + +fn file_info_stream() -> ( + FileInfoStream, + Vec, +) { + let file_info = FileInfo { + key: "test_file_info".to_string(), + prefix: "verified_mapping_event".to_string(), + timestamp: Utc::now(), + size: 0, + }; + + let reports = vec![ + VerifiedSubscriberMappingEventIngestReport { + received_timestamp: Utc::now(), + report: Some(VerifiedSubscriberMappingEvent { + subscriber_id: vec![0], + total_reward_points: 100, + timestamp: Utc::now(), + }), + }, + VerifiedSubscriberMappingEventIngestReport { + received_timestamp: Utc::now() - Duration::seconds(10), + report: Some(VerifiedSubscriberMappingEvent { + subscriber_id: vec![1], + total_reward_points: 101, + timestamp: Utc::now() - Duration::seconds(10), + }), + }, + VerifiedSubscriberMappingEventIngestReport { + received_timestamp: Utc::now(), + report: Some(VerifiedSubscriberMappingEvent { + subscriber_id: vec![1], + total_reward_points: 99, + timestamp: Utc::now(), + }), + }, + ]; + ( + FileInfoStream::new("default".to_string(), file_info, reports.clone()), + reports, + ) +} + +fn reports_to_shares( + reports: Vec, +) -> VerifiedMappingEventShares { + let mut reward_map: HashMap, i64> = HashMap::new(); + + for report in reports { + match report.report { + None => (), + Some(event) => { + let entry = reward_map.entry(event.subscriber_id).or_insert(0); + *entry += event.total_reward_points as i64; + } + } + } + + reward_map + .into_iter() + .map( + |(subscriber_id, total_reward_points)| VerifiedMappingEventShare { + subscriber_id, + total_reward_points, + }, + ) + .collect() +} + +async fn select_events( + pool: &Pool, +) -> anyhow::Result> { + Ok(sqlx::query_as::<_, VerifiedSubscriberMappingEvent>( + r#" + SELECT + subscriber_id, + total_reward_points, + timestamp + FROM verified_mapping_event + "#, + ) + .fetch_all(pool) + .await? + .into_iter() + .collect::>()) +} From 20089df6a178b23ff7ea76345c5d70eaa8d72800 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 18 Jul 2024 13:59:55 -0700 Subject: [PATCH 12/44] - Do not make report an option - Add verification_mapping_pubkey to VerifiedSubscriberMappingEvent --- .../src/verified_subscriber_mapping_event.rs | 6 ++- ..._subscriber_mapping_event_ingest_report.rs | 13 +++--- .../src/verified_subscriber_mapping_event.rs | 5 +-- .../verified_subscriber_mapping_event.rs | 45 +++++++++++-------- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/file_store/src/verified_subscriber_mapping_event.rs b/file_store/src/verified_subscriber_mapping_event.rs index 1376f3066..973b313e4 100644 --- a/file_store/src/verified_subscriber_mapping_event.rs +++ b/file_store/src/verified_subscriber_mapping_event.rs @@ -3,6 +3,7 @@ use crate::{ Error, Result, }; use chrono::{DateTime, Utc}; +use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventReqV1; use serde::{Deserialize, Serialize}; use sqlx::Row; @@ -12,6 +13,7 @@ pub struct VerifiedSubscriberMappingEvent { pub subscriber_id: Vec, pub total_reward_points: u64, pub timestamp: DateTime, + pub verification_mapping_pubkey: PublicKeyBinary, } impl MsgDecode for VerifiedSubscriberMappingEvent { @@ -37,7 +39,7 @@ impl From for VerifiedSubscriberMappingEventReqV subscriber_id: v.subscriber_id, total_reward_points: v.total_reward_points, timestamp, - verification_mapping_pubkey: vec![], + verification_mapping_pubkey: v.verification_mapping_pubkey.into(), signature: vec![], } } @@ -51,6 +53,7 @@ impl TryFrom for VerifiedSubscriberMappingE subscriber_id: v.subscriber_id, total_reward_points: v.total_reward_points, timestamp, + verification_mapping_pubkey: v.verification_mapping_pubkey.into(), }) } } @@ -61,6 +64,7 @@ impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for VerifiedSubscriberMappingEvent subscriber_id: row.get::, &str>("subscriber_id"), total_reward_points: row.get::("total_reward_points") as u64, timestamp: row.get::, &str>("timestamp"), + verification_mapping_pubkey: vec![].into(), }) } } diff --git a/file_store/src/verified_subscriber_mapping_event_ingest_report.rs b/file_store/src/verified_subscriber_mapping_event_ingest_report.rs index 88d601e5e..51799afd0 100644 --- a/file_store/src/verified_subscriber_mapping_event_ingest_report.rs +++ b/file_store/src/verified_subscriber_mapping_event_ingest_report.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] pub struct VerifiedSubscriberMappingEventIngestReport { pub received_timestamp: DateTime, - pub report: Option, + pub report: VerifiedSubscriberMappingEvent, } impl MsgDecode for VerifiedSubscriberMappingEventIngestReport { @@ -34,13 +34,12 @@ impl TryFrom { type Error = Error; fn try_from(v: VerifiedSubscriberMappingEventIngestReportV1) -> Result { - let received_timestamp = v.timestamp()?; Ok(Self { - received_timestamp, - report: match v.report { - None => None, - Some(event) => Some(event.try_into()?), - }, + received_timestamp: v.timestamp()?, + report: v + .report + .ok_or_else(|| Error::not_found("ingest VerifiedSubscriberMappingEvent report"))? + .try_into()?, }) } } diff --git a/mobile_verifier/src/verified_subscriber_mapping_event.rs b/mobile_verifier/src/verified_subscriber_mapping_event.rs index 426f0061b..f381e130c 100644 --- a/mobile_verifier/src/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/src/verified_subscriber_mapping_event.rs @@ -77,10 +77,7 @@ impl VerifiedSubscriberMappingEventDeamon { let mut reports = file.into_stream(&mut transaction).await?; while let Some(report) = reports.next().await { - match report.report { - Some(event) => save_event(&event, &mut transaction).await?, - None => tracing::debug!("no report"), - } + save_event(&report.report, &mut transaction).await?; } transaction.commit().await?; diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index a8130ac0e..e919c11df 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -5,10 +5,12 @@ use file_store::{ verified_subscriber_mapping_event_ingest_report::VerifiedSubscriberMappingEventIngestReport, FileInfo, }; +use helium_crypto::{KeyTag, Keypair, PublicKeyBinary}; use mobile_verifier::verified_subscriber_mapping_event::{ aggregate_verified_mapping_events, VerifiedMappingEventShare, VerifiedMappingEventShares, VerifiedSubscriberMappingEventDeamon, }; +use rand::rngs::OsRng; use sqlx::{PgPool, Pool, Postgres}; use std::{collections::HashMap, ops::Range}; @@ -23,7 +25,7 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { deamon.run(listener).await.expect("failed to complete task"); }); - let (fis, reports) = file_info_stream(); + let (fis, mut reports) = file_info_stream(); tx.send(fis).await?; let mut retry = 0; @@ -32,11 +34,10 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { while retry <= MAX_RETRIES { let saved_vmes = select_events(&pool).await?; if reports.len() == saved_vmes.len() { - assert!(reports.iter().all(|r| { - match r.report.clone() { - None => false, - Some(event) => saved_vmes.contains(&event), - } + assert!(reports.iter_mut().all(|r| { + // We ahve to do this because we do not store verification_mapping_pubkey in DB + r.report.verification_mapping_pubkey = vec![].into(); + saved_vmes.contains(&r.report) })); break; } else { @@ -81,30 +82,36 @@ fn file_info_stream() -> ( size: 0, }; + let key_pair = generate_keypair(); + let public_key_binary: PublicKeyBinary = key_pair.public_key().to_owned().into(); + let reports = vec![ VerifiedSubscriberMappingEventIngestReport { received_timestamp: Utc::now(), - report: Some(VerifiedSubscriberMappingEvent { + report: VerifiedSubscriberMappingEvent { subscriber_id: vec![0], total_reward_points: 100, timestamp: Utc::now(), - }), + verification_mapping_pubkey: public_key_binary.clone(), + }, }, VerifiedSubscriberMappingEventIngestReport { received_timestamp: Utc::now() - Duration::seconds(10), - report: Some(VerifiedSubscriberMappingEvent { + report: VerifiedSubscriberMappingEvent { subscriber_id: vec![1], total_reward_points: 101, timestamp: Utc::now() - Duration::seconds(10), - }), + verification_mapping_pubkey: public_key_binary.clone(), + }, }, VerifiedSubscriberMappingEventIngestReport { received_timestamp: Utc::now(), - report: Some(VerifiedSubscriberMappingEvent { + report: VerifiedSubscriberMappingEvent { subscriber_id: vec![1], total_reward_points: 99, timestamp: Utc::now(), - }), + verification_mapping_pubkey: public_key_binary.clone(), + }, }, ]; ( @@ -119,13 +126,9 @@ fn reports_to_shares( let mut reward_map: HashMap, i64> = HashMap::new(); for report in reports { - match report.report { - None => (), - Some(event) => { - let entry = reward_map.entry(event.subscriber_id).or_insert(0); - *entry += event.total_reward_points as i64; - } - } + let event = report.report; + let entry = reward_map.entry(event.subscriber_id).or_insert(0); + *entry += event.total_reward_points as i64; } reward_map @@ -156,3 +159,7 @@ async fn select_events( .into_iter() .collect::>()) } + +fn generate_keypair() -> Keypair { + Keypair::generate(KeyTag::default(), &mut OsRng) +} From 6ce923eabeab62673cad92426ade624a5276a096 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 19 Jul 2024 10:50:33 -0700 Subject: [PATCH 13/44] Fix rewards_per_share --- mobile_verifier/src/reward_shares.rs | 16 +++++++++++----- mobile_verifier/src/rewarder.rs | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index e065224b1..743800e24 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -210,18 +210,24 @@ impl MapperShares { // the number of subscribers eligible for discovery location rewards let discovery_mappers_count = Decimal::from(self.discovery_mapping_shares.len()); - let verified_mapping_event_count = Decimal::from(self.verified_mapping_event_shares.len()); - // calculate the total eligible mapping shares for the epoch // this could be simplified as every subscriber is awarded the same share // however the function is setup to allow the verification mapper shares to be easily // added without impacting code structure ( the per share value for those will be different ) - let total_mapper_shares = - (discovery_mappers_count + verified_mapping_event_count) * DISCOVERY_MAPPING_SHARES; + let total_mapper_shares = discovery_mappers_count * DISCOVERY_MAPPING_SHARES; + + let total_verified_mapping_event_shares: Decimal = self + .verified_mapping_event_shares + .iter() + .map(|share| Decimal::from(share.total_reward_points)) + .sum(); + + let total_shares = total_mapper_shares + total_verified_mapping_event_shares; let res = total_mappers_pool - .checked_div(total_mapper_shares) + .checked_div(total_shares) .unwrap_or(Decimal::ZERO); + Ok(res) } diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index f152400d0..a31c8efb8 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -524,6 +524,7 @@ pub async fn reward_mappers( // translate discovery mapping shares into subscriber rewards let mut allocated_mapping_rewards = 0_u64; + for (reward_amount, mapping_share) in mapping_shares.into_subscriber_rewards(reward_period, rewards_per_share) { From b60c86231a1db4022136110e2481b96f52fbcd9b Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 29 Jul 2024 13:54:37 -0700 Subject: [PATCH 14/44] Update from proto --- file_store/src/verified_subscriber_mapping_event.rs | 8 ++++---- ingest/src/server_mobile.rs | 4 ++-- ingest/tests/common/mod.rs | 2 +- mobile_verifier/src/reward_shares.rs | 2 +- .../integrations/verified_subscriber_mapping_event.rs | 10 +++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/file_store/src/verified_subscriber_mapping_event.rs b/file_store/src/verified_subscriber_mapping_event.rs index 973b313e4..ad21c07a4 100644 --- a/file_store/src/verified_subscriber_mapping_event.rs +++ b/file_store/src/verified_subscriber_mapping_event.rs @@ -13,7 +13,7 @@ pub struct VerifiedSubscriberMappingEvent { pub subscriber_id: Vec, pub total_reward_points: u64, pub timestamp: DateTime, - pub verification_mapping_pubkey: PublicKeyBinary, + pub carrier_mapping_key: PublicKeyBinary, } impl MsgDecode for VerifiedSubscriberMappingEvent { @@ -39,7 +39,7 @@ impl From for VerifiedSubscriberMappingEventReqV subscriber_id: v.subscriber_id, total_reward_points: v.total_reward_points, timestamp, - verification_mapping_pubkey: v.verification_mapping_pubkey.into(), + carrier_mapping_key: v.carrier_mapping_key.into(), signature: vec![], } } @@ -53,7 +53,7 @@ impl TryFrom for VerifiedSubscriberMappingE subscriber_id: v.subscriber_id, total_reward_points: v.total_reward_points, timestamp, - verification_mapping_pubkey: v.verification_mapping_pubkey.into(), + carrier_mapping_key: v.carrier_mapping_key.into(), }) } } @@ -64,7 +64,7 @@ impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for VerifiedSubscriberMappingEvent subscriber_id: row.get::, &str>("subscriber_id"), total_reward_points: row.get::("total_reward_points") as u64, timestamp: row.get::, &str>("timestamp"), - verification_mapping_pubkey: vec![].into(), + carrier_mapping_key: vec![].into(), }) } } diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 97e537820..bead852ef 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -417,10 +417,10 @@ impl poc_mobile::PocMobile for GrpcServer { let subscriber_id = event.subscriber_id.clone(); custom_tracing::record("subscriber_id", bs58::encode(&subscriber_id).into_string()); - custom_tracing::record_b58("pub_key", &event.verification_mapping_pubkey); + custom_tracing::record_b58("pub_key", &event.carrier_mapping_key); let report = self - .verify_public_key(event.verification_mapping_pubkey.as_ref()) + .verify_public_key(event.carrier_mapping_key.as_ref()) .and_then(|public_key| self.verify_network(public_key)) .and_then(|public_key| self.verify_signature(public_key, event)) .map(|(_, event)| VerifiedSubscriberMappingEventIngestReportV1 { diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index 0affcc342..17f955654 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -95,7 +95,7 @@ impl TestClient { subscriber_id, total_reward_points, timestamp: 0, - verification_mapping_pubkey: self.key_pair.public_key().to_vec(), + carrier_mapping_key: self.key_pair.public_key().to_vec(), signature: vec![], }; diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 743800e24..b256bf446 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -222,7 +222,7 @@ impl MapperShares { .map(|share| Decimal::from(share.total_reward_points)) .sum(); - let total_shares = total_mapper_shares + total_verified_mapping_event_shares; + let total_shares = total_disco_mapper_shares + total_verified_mapping_event_shares; let res = total_mappers_pool .checked_div(total_shares) diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index e919c11df..3e9dc0d12 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -35,8 +35,8 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { let saved_vmes = select_events(&pool).await?; if reports.len() == saved_vmes.len() { assert!(reports.iter_mut().all(|r| { - // We ahve to do this because we do not store verification_mapping_pubkey in DB - r.report.verification_mapping_pubkey = vec![].into(); + // We have to do this because we do not store carrier_mapping_key in DB + r.report.carrier_mapping_key = vec![].into(); saved_vmes.contains(&r.report) })); break; @@ -92,7 +92,7 @@ fn file_info_stream() -> ( subscriber_id: vec![0], total_reward_points: 100, timestamp: Utc::now(), - verification_mapping_pubkey: public_key_binary.clone(), + carrier_mapping_key: public_key_binary.clone(), }, }, VerifiedSubscriberMappingEventIngestReport { @@ -101,7 +101,7 @@ fn file_info_stream() -> ( subscriber_id: vec![1], total_reward_points: 101, timestamp: Utc::now() - Duration::seconds(10), - verification_mapping_pubkey: public_key_binary.clone(), + carrier_mapping_key: public_key_binary.clone(), }, }, VerifiedSubscriberMappingEventIngestReport { @@ -110,7 +110,7 @@ fn file_info_stream() -> ( subscriber_id: vec![1], total_reward_points: 99, timestamp: Utc::now(), - verification_mapping_pubkey: public_key_binary.clone(), + carrier_mapping_key: public_key_binary.clone(), }, }, ]; From e6e98a3c724e6740fccc2867c9099ba0336250f0 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 29 Jul 2024 14:12:48 -0700 Subject: [PATCH 15/44] Do not impl from_row just for test --- .../src/verified_subscriber_mapping_event.rs | 12 ---------- .../verified_subscriber_mapping_event.rs | 22 ++++++++++++++----- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/file_store/src/verified_subscriber_mapping_event.rs b/file_store/src/verified_subscriber_mapping_event.rs index ad21c07a4..f76a014f2 100644 --- a/file_store/src/verified_subscriber_mapping_event.rs +++ b/file_store/src/verified_subscriber_mapping_event.rs @@ -6,7 +6,6 @@ use chrono::{DateTime, Utc}; use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventReqV1; use serde::{Deserialize, Serialize}; -use sqlx::Row; #[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] pub struct VerifiedSubscriberMappingEvent { @@ -57,14 +56,3 @@ impl TryFrom for VerifiedSubscriberMappingE }) } } - -impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for VerifiedSubscriberMappingEvent { - fn from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { - Ok(Self { - subscriber_id: row.get::, &str>("subscriber_id"), - total_reward_points: row.get::("total_reward_points") as u64, - timestamp: row.get::, &str>("timestamp"), - carrier_mapping_key: vec![].into(), - }) - } -} diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index 3e9dc0d12..c01540fe6 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -1,4 +1,4 @@ -use chrono::{Duration, Utc}; +use chrono::{DateTime, Duration, Utc}; use file_store::{ file_info_poller::FileInfoStream, verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, @@ -11,7 +11,7 @@ use mobile_verifier::verified_subscriber_mapping_event::{ VerifiedSubscriberMappingEventDeamon, }; use rand::rngs::OsRng; -use sqlx::{PgPool, Pool, Postgres}; +use sqlx::{PgPool, Pool, Postgres, Row}; use std::{collections::HashMap, ops::Range}; #[sqlx::test] @@ -145,7 +145,7 @@ fn reports_to_shares( async fn select_events( pool: &Pool, ) -> anyhow::Result> { - Ok(sqlx::query_as::<_, VerifiedSubscriberMappingEvent>( + let rows = sqlx::query( r#" SELECT subscriber_id, @@ -155,9 +155,19 @@ async fn select_events( "#, ) .fetch_all(pool) - .await? - .into_iter() - .collect::>()) + .await?; + + let events = rows + .into_iter() + .map(|row| VerifiedSubscriberMappingEvent { + subscriber_id: row.get::, _>("subscriber_id"), + total_reward_points: row.get::("total_reward_points") as u64, + timestamp: row.get::, _>("timestamp"), + carrier_mapping_key: vec![].into(), + }) + .collect::>(); + + Ok(events) } fn generate_keypair() -> Keypair { From 5b5c5f8706b9f8049395aa91ce19ff16541b7cbd Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 29 Jul 2024 14:21:44 -0700 Subject: [PATCH 16/44] Fix rebase --- mobile_verifier/src/reward_shares.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index b256bf446..743800e24 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -222,7 +222,7 @@ impl MapperShares { .map(|share| Decimal::from(share.total_reward_points)) .sum(); - let total_shares = total_disco_mapper_shares + total_verified_mapping_event_shares; + let total_shares = total_mapper_shares + total_verified_mapping_event_shares; let res = total_mappers_pool .checked_div(total_shares) From 5fbf7eace850bb031fd82de8e3de4d9e10100f5f Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 30 Jul 2024 12:24:01 -0700 Subject: [PATCH 17/44] Fix msg type name --- file_store/src/file_info.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index af2954221..4061bf7e8 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -160,8 +160,8 @@ pub const LANDTYPE_DATA_SET: &str = "landtype"; pub const SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "service_provider_boosted_rewards_banned_radio"; pub const VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "verified_service_provider_boosted_rewards_banned_radio"; -pub const VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT: &str = - "verified_subscriber_mapping_event_report"; +pub const VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT: &str = + "verified_subscriber_mapping_ingest_report"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -283,7 +283,7 @@ impl fmt::Display for FileType { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } Self::VerifiedSubscriberMappingEventIngestReportV1 => { - VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT + VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT } }; f.write_str(s) @@ -354,7 +354,7 @@ impl FileType { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } Self::VerifiedSubscriberMappingEventIngestReportV1 => { - VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT + VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT } } } @@ -424,7 +424,7 @@ impl FromStr for FileType { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO => { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport } - VERIFIED_SUBSCRIBER_MAPPING_EVENT_REPORT => { + VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT => { Self::VerifiedSubscriberMappingEventIngestReportV1 } _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), From f6ab8e099116dff6df80cafb37c486b37f2be123 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 1 Aug 2024 09:57:27 -0700 Subject: [PATCH 18/44] Simplify test further --- ingest/tests/common/mod.rs | 41 +++++++++++++++++++---------------- ingest/tests/mobile_ingest.rs | 4 ++-- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index 17f955654..476ea4e94 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -18,11 +18,7 @@ use tonic::{ }; use triggered::Trigger; -pub async fn setup_mobile() -> anyhow::Result<( - TestClient, - Receiver, - Trigger, -)> { +pub async fn setup_mobile() -> anyhow::Result<(TestClient, Trigger)> { let key_pair = generate_keypair(); let (file_sink_tx, file_sink_rx) = tokio::sync::mpsc::channel(10); @@ -61,19 +57,25 @@ pub async fn setup_mobile() -> anyhow::Result<( grpc_server.run(listener).await }); - let client = TestClient::new(socket_addr, key_pair, token.to_string()).await; + let client = TestClient::new(socket_addr, key_pair, token.to_string(), file_sink_rx).await; - Ok((client, file_sink_rx, trigger)) + Ok((client, trigger)) } pub struct TestClient { client: PocMobileClient, key_pair: Arc, authorization: MetadataValue, + file_sink_rx: Receiver, } impl TestClient { - pub async fn new(socket_addr: SocketAddr, key_pair: Keypair, api_token: String) -> TestClient { + pub async fn new( + socket_addr: SocketAddr, + key_pair: Keypair, + api_token: String, + file_sink_rx: Receiver, + ) -> TestClient { let client = (|| PocMobileClient::connect(format!("http://{socket_addr}"))) .retry(&ExponentialBuilder::default()) .await @@ -83,6 +85,18 @@ impl TestClient { client, key_pair: Arc::new(key_pair), authorization: format!("Bearer {}", api_token).try_into().unwrap(), + file_sink_rx, + } + } + + pub async fn recv(mut self) -> anyhow::Result> { + match self.file_sink_rx.recv().await { + Some(msg) => match msg { + file_store::file_sink::Message::Commit(_) => bail!("got Commit"), + file_store::file_sink::Message::Rollback(_) => bail!("got Rollback"), + file_store::file_sink::Message::Data(_, data) => Ok(data), + }, + None => bail!("got none"), } } @@ -118,14 +132,3 @@ impl TestClient { pub fn generate_keypair() -> Keypair { Keypair::generate(KeyTag::default(), &mut OsRng) } - -pub async fn recv(mut rx: Receiver) -> anyhow::Result> { - match rx.recv().await { - Some(msg) => match msg { - file_store::file_sink::Message::Commit(_) => bail!("got Commit"), - file_store::file_sink::Message::Rollback(_) => bail!("got Rollback"), - file_store::file_sink::Message::Data(_, data) => Ok(data), - }, - None => bail!("got none"), - } -} diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index a169afa56..50a770c1f 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -5,7 +5,7 @@ mod common; #[tokio::test] async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { - let (mut client, file_sink_rx, trigger) = common::setup_mobile().await?; + let (mut client, trigger) = common::setup_mobile().await?; let subscriber_id = vec![0]; let total_reward_points = 100; @@ -18,7 +18,7 @@ async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { let timestamp: String = res.unwrap().id; - match common::recv(file_sink_rx).await { + match client.recv().await { Ok(data) => { let report = VerifiedSubscriberMappingEventIngestReportV1::decode(data.as_slice()) .expect("unable to decode into VerifiedSubscriberMappingEventIngestReportV1"); From 68dd915f05b89fad8323ddcda301acce15656ac3 Mon Sep 17 00:00:00 2001 From: macpie Date: Thu, 1 Aug 2024 10:01:08 -0700 Subject: [PATCH 19/44] Update mobile_verifier/src/reward_shares.rs Co-authored-by: andymck --- mobile_verifier/src/reward_shares.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 743800e24..83fb2f70a 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -214,7 +214,7 @@ impl MapperShares { // this could be simplified as every subscriber is awarded the same share // however the function is setup to allow the verification mapper shares to be easily // added without impacting code structure ( the per share value for those will be different ) - let total_mapper_shares = discovery_mappers_count * DISCOVERY_MAPPING_SHARES; + let total_disco_mapper_shares = discovery_mappers_count * DISCOVERY_MAPPING_SHARES; let total_verified_mapping_event_shares: Decimal = self .verified_mapping_event_shares From 87713585c60d61bf1759de91b3d3bc0711536a79 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 1 Aug 2024 10:08:52 -0700 Subject: [PATCH 20/44] Fix var name --- mobile_verifier/src/reward_shares.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 83fb2f70a..743800e24 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -214,7 +214,7 @@ impl MapperShares { // this could be simplified as every subscriber is awarded the same share // however the function is setup to allow the verification mapper shares to be easily // added without impacting code structure ( the per share value for those will be different ) - let total_disco_mapper_shares = discovery_mappers_count * DISCOVERY_MAPPING_SHARES; + let total_mapper_shares = discovery_mappers_count * DISCOVERY_MAPPING_SHARES; let total_verified_mapping_event_shares: Decimal = self .verified_mapping_event_shares From adbd484af06c5e7e1e1c2ef74a590faffb0cee1a Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 1 Aug 2024 11:28:21 -0700 Subject: [PATCH 21/44] Add verification_mapping_amount to subscriber_reward --- file_store/src/cli/dump.rs | 1 + file_store/src/cli/dump_mobile_rewards.rs | 1 + mobile_verifier/src/reward_shares.rs | 105 +++++++++++++--------- reward_index/src/indexer.rs | 2 +- 4 files changed, 67 insertions(+), 42 deletions(-) diff --git a/file_store/src/cli/dump.rs b/file_store/src/cli/dump.rs index 1f348fdcd..5c0eba604 100644 --- a/file_store/src/cli/dump.rs +++ b/file_store/src/cli/dump.rs @@ -257,6 +257,7 @@ impl Cmd { Some(Reward::SubscriberReward(reward)) => print_json(&json!({ "subscriber_id": reward.subscriber_id, "discovery_location_amount": reward.discovery_location_amount, + "verification_mapping_amount": reward.verification_mapping_amount, }))?, Some(Reward::ServiceProviderReward(reward)) => print_json(&json!({ "service_provider": reward.service_provider_id, diff --git a/file_store/src/cli/dump_mobile_rewards.rs b/file_store/src/cli/dump_mobile_rewards.rs index 34e605002..aadad8792 100644 --- a/file_store/src/cli/dump_mobile_rewards.rs +++ b/file_store/src/cli/dump_mobile_rewards.rs @@ -50,6 +50,7 @@ impl Cmd { SubscriberReward(reward) => subscriber_reward.push(json!({ "subscriber_id": uuid::Uuid::from_slice(&reward.subscriber_id).unwrap(), "discovery_location_amount": reward.discovery_location_amount, + "verification_mapping_amount": reward.verification_mapping_amount, })), ServiceProviderReward(reward) => service_provider_reward.push(json!({ "service_provider": reward.service_provider_id, diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 743800e24..b37506de0 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -236,52 +236,70 @@ impl MapperShares { reward_period: &'_ Range>, reward_per_share: Decimal, ) -> impl Iterator + '_ { - let discovery_rewards = self - .discovery_mapping_shares - .into_iter() - .map(move |subscriber_id| proto::SubscriberReward { - subscriber_id, - discovery_location_amount: (DISCOVERY_MAPPING_SHARES * reward_per_share) - .round_dp_with_strategy(0, RoundingStrategy::ToZero) - .to_u64() - .unwrap_or_default(), - }) - .filter(|subscriber_reward| subscriber_reward.discovery_location_amount > 0) - .map(|subscriber_reward| { - ( - subscriber_reward.discovery_location_amount, - proto::MobileRewardShare { - start_period: reward_period.start.encode_timestamp(), - end_period: reward_period.end.encode_timestamp(), - reward: Some(ProtoReward::SubscriberReward(subscriber_reward)), - }, - ) - }); + let mut subscriber_rewards: HashMap, proto::SubscriberReward> = HashMap::new(); - let verified_rewards = self - .verified_mapping_event_shares + // Accumulate rewards from discovery_mapping_shares + for subscriber_id in self.discovery_mapping_shares { + let discovery_location_amount = (DISCOVERY_MAPPING_SHARES * reward_per_share) + .round_dp_with_strategy(0, RoundingStrategy::ToZero) + .to_u64() + .unwrap_or_default(); + + if discovery_location_amount > 0 { + subscriber_rewards + .entry(subscriber_id.clone()) + .and_modify(|reward| { + reward.discovery_location_amount += discovery_location_amount; + }) + .or_insert_with(|| proto::SubscriberReward { + subscriber_id: subscriber_id.clone(), + discovery_location_amount, + verification_mapping_amount: 0, + }); + } + } + + // Accumulate rewards from verified_mapping_event_shares + for verified_share in self.verified_mapping_event_shares { + let verification_mapping_amount = (Decimal::from(verified_share.total_reward_points) + * reward_per_share) + .round_dp_with_strategy(0, RoundingStrategy::ToZero) + .to_u64() + .unwrap_or_default(); + + if verification_mapping_amount > 0 { + subscriber_rewards + .entry(verified_share.subscriber_id.clone()) + .and_modify(|reward| { + reward.verification_mapping_amount += verification_mapping_amount; + }) + .or_insert_with(|| proto::SubscriberReward { + subscriber_id: verified_share.subscriber_id.clone(), + discovery_location_amount: 0, + verification_mapping_amount, + }); + } + } + + // Create the MobileRewardShare for each subscriber + subscriber_rewards .into_iter() - .map(move |verified_share| proto::SubscriberReward { - subscriber_id: verified_share.subscriber_id, - discovery_location_amount: (Decimal::from(verified_share.total_reward_points) - * reward_per_share) - .round_dp_with_strategy(0, RoundingStrategy::ToZero) - .to_u64() - .unwrap_or_default(), + .filter(|(_, reward)| { + reward.discovery_location_amount > 0 || reward.verification_mapping_amount > 0 }) - .filter(|subscriber_reward| subscriber_reward.discovery_location_amount > 0) - .map(|subscriber_reward| { + .map(move |(_, subscriber_reward)| { + let total_reward_amount = subscriber_reward.discovery_location_amount + + subscriber_reward.verification_mapping_amount; + ( - subscriber_reward.discovery_location_amount, + total_reward_amount, proto::MobileRewardShare { start_period: reward_period.start.encode_timestamp(), end_period: reward_period.end.encode_timestamp(), reward: Some(ProtoReward::SubscriberReward(subscriber_reward)), }, ) - }); - - discovery_rewards.chain(verified_rewards) + }) } } @@ -923,7 +941,7 @@ mod test { } #[tokio::test] - async fn discover_mapping_amount() { + async fn subscriber_rewards() { // test based on example defined at https://github.com/helium/oracles/issues/422 // NOTE: the example defined above lists values in mobile tokens, whereas // this test uses mobile bones @@ -969,8 +987,7 @@ mod test { .unwrap_or(0); assert_eq!(16_438_356_164_383, total_mapper_rewards); - // We have 2x more subscriber rewarded (for same amount as mappers 30) - let expected_reward_per_subscriber = total_mapper_rewards / NUM_SUBSCRIBERS / 2; + let expected_reward_per_subscriber = total_mapper_rewards / NUM_SUBSCRIBERS; // get the summed rewards allocated to subscribers for discovery location let mut allocated_mapper_rewards = 0_u64; @@ -978,8 +995,14 @@ mod test { shares.into_subscriber_rewards(&epoch, rewards_per_share) { if let Some(MobileReward::SubscriberReward(r)) = subscriber_share.reward { - assert_eq!(expected_reward_per_subscriber, r.discovery_location_amount); - assert_eq!(reward_amount, r.discovery_location_amount); + assert_eq!( + expected_reward_per_subscriber, + r.discovery_location_amount + r.verification_mapping_amount + ); + assert_eq!( + reward_amount, + r.discovery_location_amount + r.verification_mapping_amount + ); allocated_mapper_rewards += reward_amount; } } diff --git a/reward_index/src/indexer.rs b/reward_index/src/indexer.rs index e4f6fb847..f2823e778 100644 --- a/reward_index/src/indexer.rs +++ b/reward_index/src/indexer.rs @@ -163,7 +163,7 @@ impl Indexer { key: bs58::encode(&r.subscriber_id).into_string(), reward_type: RewardType::MobileSubscriber, }, - r.discovery_location_amount, + r.discovery_location_amount + r.verification_mapping_amount, ))), Some(MobileReward::ServiceProviderReward(r)) => { ServiceProvider::try_from(r.service_provider_id) From 303b52f58bdc5c90c6b8aa3323d400a902c47f15 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 09:53:23 -0700 Subject: [PATCH 22/44] Use git+https://github.com/helium/proto?branch=macpie%2Fverification_mapping#helium-proto@0.1.0 --- Cargo.lock | 47 +++++++++++++++++++++++++++++------------------ Cargo.toml | 6 ++---- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49984f909..9001e2a37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1621,7 +1621,7 @@ source = "git+https://github.com/helium/proto?branch=master#fdef2c6774416b1b361a dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "prost", "rand 0.8.5", "rand_chacha 0.3.0", @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "http-serde", "humantime-serde", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "notify", "serde", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hex-literal", "http 0.2.11", "lazy_static", @@ -3776,7 +3776,7 @@ dependencies = [ "h3o", "helium-anchor-gen", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", "hex", "itertools", "jsonrpc_client", @@ -3801,7 +3801,6 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdef2c6774416b1b361a22894078fb1cc5e89203" dependencies = [ "bytes", "prost", @@ -3812,6 +3811,18 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "helium-proto" +version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=master#62e22b2a4fb7de6f2ba7d67f933d77d4ad52a882" +dependencies = [ + "bytes", + "prost", + "prost-build", + "serde", + "serde_json", +] + [[package]] name = "helium-sub-daos" version = "0.1.5" @@ -3853,7 +3864,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4269,7 +4280,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "humantime-serde", "metrics", @@ -4338,7 +4349,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hextree", "http 0.2.11", "http-serde", @@ -4380,7 +4391,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "http-serde", "humantime-serde", @@ -4422,7 +4433,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http-serde", "humantime-serde", "iot-config", @@ -5010,7 +5021,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hextree", "http 0.2.11", "http-serde", @@ -5050,7 +5061,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "mobile-config", "prost", "rand 0.8.5", @@ -5086,7 +5097,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "http-serde", "humantime-serde", @@ -5130,7 +5141,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "hex-assignments", "hextree", "http-serde", @@ -5813,7 +5824,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5896,7 +5907,7 @@ dependencies = [ "futures-util", "helium-anchor-gen", "helium-lib", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6503,7 +6514,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", "humantime-serde", "lazy_static", "metrics", diff --git a/Cargo.toml b/Cargo.toml index 81fde8a1e..8f6db3618 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,10 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "macpie/verification_mapping", features = [ "services", ] } -beacon = { git = "https://github.com/helium/proto", branch = "master" } +beacon = { git = "https://github.com/helium/proto", branch = "macpie/verification_mapping" } solana-client = "1.18" solana-sdk = "1.18" solana-program = "1.18" @@ -134,5 +134,3 @@ sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb # helium-proto = { path = "../proto" } # beacon = { path = "../proto/beacon" } # Temporary patch -[patch."https://github.com/helium/proto"] -helium-proto = { path = "../proto" } From 2dfaf8e9af7002cfae6dc06f6d5bc9ab25ddcd97 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 10:04:09 -0700 Subject: [PATCH 23/44] Do not acc rewards --- mobile_verifier/src/reward_shares.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index b37506de0..b0e15515e 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -249,7 +249,7 @@ impl MapperShares { subscriber_rewards .entry(subscriber_id.clone()) .and_modify(|reward| { - reward.discovery_location_amount += discovery_location_amount; + reward.discovery_location_amount = discovery_location_amount; }) .or_insert_with(|| proto::SubscriberReward { subscriber_id: subscriber_id.clone(), @@ -271,7 +271,7 @@ impl MapperShares { subscriber_rewards .entry(verified_share.subscriber_id.clone()) .and_modify(|reward| { - reward.verification_mapping_amount += verification_mapping_amount; + reward.verification_mapping_amount = verification_mapping_amount; }) .or_insert_with(|| proto::SubscriberReward { subscriber_id: verified_share.subscriber_id.clone(), From 45afaf62e0f94aee900da99126aeb4095153cd43 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 10:17:43 -0700 Subject: [PATCH 24/44] Improve tests --- mobile_verifier/src/reward_shares.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index b0e15515e..17726ea69 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -1003,6 +1003,10 @@ mod test { reward_amount, r.discovery_location_amount + r.verification_mapping_amount ); + // These are the same because we gave `total_reward_points: 30,` for each + // VerifiedMappingEventShares which is the same amount as discovery mapping + assert_eq!(819_672_131, r.discovery_location_amount); + assert_eq!(819_672_131, r.verification_mapping_amount); allocated_mapper_rewards += reward_amount; } } From a0a7743ce7c3473e17df1144a7a20efcdbc6f878 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 11:35:51 -0700 Subject: [PATCH 25/44] Add timeout to test client --- ingest/tests/common/mod.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index 476ea4e94..8cc3edfee 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -9,8 +9,8 @@ use helium_proto::services::poc_mobile::{ use ingest::server_mobile::GrpcServer; use prost::Message; use rand::rngs::OsRng; -use std::{net::SocketAddr, sync::Arc}; -use tokio::{net::TcpListener, sync::mpsc::Receiver}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tokio::{net::TcpListener, sync::mpsc::Receiver, time::timeout}; use tonic::{ metadata::{Ascii, MetadataValue}, transport::Channel, @@ -90,13 +90,14 @@ impl TestClient { } pub async fn recv(mut self) -> anyhow::Result> { - match self.file_sink_rx.recv().await { - Some(msg) => match msg { + match timeout(Duration::from_secs(2), self.file_sink_rx.recv()).await { + Ok(Some(msg)) => match msg { file_store::file_sink::Message::Commit(_) => bail!("got Commit"), file_store::file_sink::Message::Rollback(_) => bail!("got Rollback"), file_store::file_sink::Message::Data(_, data) => Ok(data), }, - None => bail!("got none"), + Ok(None) => bail!("got none"), + Err(reason) => bail!("got error {reason}"), } } From b0384a172feece6f3be38adc6667ccc984402afe Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 11:56:34 -0700 Subject: [PATCH 26/44] The moogey commit --- mobile_verifier/src/reward_shares.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 17726ea69..e0bd986c0 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -203,11 +203,6 @@ impl MapperShares { } pub fn rewards_per_share(&self, total_mappers_pool: Decimal) -> anyhow::Result { - // note: currently rewards_per_share calculation only takes into - // consideration discovery mapping shares - // in the future it will also need to take into account - // verification mapping shares - // the number of subscribers eligible for discovery location rewards let discovery_mappers_count = Decimal::from(self.discovery_mapping_shares.len()); // calculate the total eligible mapping shares for the epoch @@ -238,14 +233,14 @@ impl MapperShares { ) -> impl Iterator + '_ { let mut subscriber_rewards: HashMap, proto::SubscriberReward> = HashMap::new(); - // Accumulate rewards from discovery_mapping_shares - for subscriber_id in self.discovery_mapping_shares { - let discovery_location_amount = (DISCOVERY_MAPPING_SHARES * reward_per_share) - .round_dp_with_strategy(0, RoundingStrategy::ToZero) - .to_u64() - .unwrap_or_default(); + let discovery_location_amount = (DISCOVERY_MAPPING_SHARES * reward_per_share) + .round_dp_with_strategy(0, RoundingStrategy::ToZero) + .to_u64() + .unwrap_or_default(); - if discovery_location_amount > 0 { + if discovery_location_amount > 0 { + // Collect rewards from discovery_mapping_shares + for subscriber_id in self.discovery_mapping_shares { subscriber_rewards .entry(subscriber_id.clone()) .and_modify(|reward| { @@ -259,7 +254,7 @@ impl MapperShares { } } - // Accumulate rewards from verified_mapping_event_shares + // Collect rewards from verified_mapping_event_shares for verified_share in self.verified_mapping_event_shares { let verification_mapping_amount = (Decimal::from(verified_share.total_reward_points) * reward_per_share) From 860bf3393a8ad4b794e85f39d34f943225bb37ae Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 12:01:44 -0700 Subject: [PATCH 27/44] Cleanup (txh moogey) --- mobile_verifier/src/reward_shares.rs | 6 +++--- .../src/verified_subscriber_mapping_event.rs | 12 ++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index e0bd986c0..edd8709c1 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -278,11 +278,11 @@ impl MapperShares { // Create the MobileRewardShare for each subscriber subscriber_rewards - .into_iter() - .filter(|(_, reward)| { + .into_values() + .filter(|reward| { reward.discovery_location_amount > 0 || reward.verification_mapping_amount > 0 }) - .map(move |(_, subscriber_reward)| { + .map(|subscriber_reward| { let total_reward_amount = subscriber_reward.discovery_location_amount + subscriber_reward.verification_mapping_amount; diff --git a/mobile_verifier/src/verified_subscriber_mapping_event.rs b/mobile_verifier/src/verified_subscriber_mapping_event.rs index f381e130c..0e02a6fdc 100644 --- a/mobile_verifier/src/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/src/verified_subscriber_mapping_event.rs @@ -7,7 +7,7 @@ use file_store::{ verified_subscriber_mapping_event_ingest_report::VerifiedSubscriberMappingEventIngestReport, FileStore, FileType, }; -use futures::{stream::StreamExt, TryStreamExt}; +use futures::stream::StreamExt; use sqlx::{Pool, Postgres, Transaction}; use std::ops::Range; use task_manager::{ManagedTask, TaskManager}; @@ -127,7 +127,7 @@ pub async fn aggregate_verified_mapping_events( db: impl sqlx::PgExecutor<'_> + Copy, reward_period: &Range>, ) -> Result { - let mut rows = sqlx::query_as::<_, VerifiedMappingEventShare>( + let vme_shares = sqlx::query_as::<_, VerifiedMappingEventShare>( "SELECT subscriber_id, SUM(total_reward_points) AS total_reward_points @@ -139,12 +139,8 @@ pub async fn aggregate_verified_mapping_events( ) .bind(reward_period.end - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) .bind(reward_period.end) - .fetch(db); - - let mut vme_shares = VerifiedMappingEventShares::new(); - while let Some(share) = rows.try_next().await? { - vme_shares.push(share) - } + .fetch_all(db) + .await?; Ok(vme_shares) } From 6ad41c9db59e747901a8c86976101c4fbd1a6873 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 2 Aug 2024 12:19:17 -0700 Subject: [PATCH 28/44] Fix tests --- mobile_verifier/src/reward_shares.rs | 4 ++-- mobile_verifier/tests/integrations/rewarder_mappers.rs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index edd8709c1..98c3ce4da 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -1000,8 +1000,8 @@ mod test { ); // These are the same because we gave `total_reward_points: 30,` for each // VerifiedMappingEventShares which is the same amount as discovery mapping - assert_eq!(819_672_131, r.discovery_location_amount); - assert_eq!(819_672_131, r.verification_mapping_amount); + assert_eq!(821_917_808, r.discovery_location_amount); + assert_eq!(821_917_808, r.verification_mapping_amount); allocated_mapper_rewards += reward_amount; } } diff --git a/mobile_verifier/tests/integrations/rewarder_mappers.rs b/mobile_verifier/tests/integrations/rewarder_mappers.rs index e438dbc52..fe98830fb 100644 --- a/mobile_verifier/tests/integrations/rewarder_mappers.rs +++ b/mobile_verifier/tests/integrations/rewarder_mappers.rs @@ -32,6 +32,7 @@ async fn test_mapper_rewards(pool: PgPool) -> anyhow::Result<()> { rewarder::reward_mappers(&pool, &mobile_rewards_client, &epoch), receive_expected_rewards(&mut mobile_rewards) ); + if let Ok((subscriber_rewards, unallocated_reward)) = rewards { // assert the mapper rewards // all 3 subscribers will have an equal share, @@ -101,7 +102,9 @@ async fn receive_expected_rewards( let subscriber_reward1 = mobile_rewards.receive_subscriber_reward().await; let subscriber_reward2 = mobile_rewards.receive_subscriber_reward().await; let subscriber_reward3 = mobile_rewards.receive_subscriber_reward().await; - let subscriber_rewards = vec![subscriber_reward1, subscriber_reward2, subscriber_reward3]; + let mut subscriber_rewards = vec![subscriber_reward1, subscriber_reward2, subscriber_reward3]; + + subscriber_rewards.sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)); // expect one unallocated reward let unallocated_reward = mobile_rewards.receive_unallocated_reward().await; From a5cb80f26b4c50a2cbd131a288a66226f33a4733 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 5 Aug 2024 10:36:31 -0700 Subject: [PATCH 29/44] Add some println --- .../verified_subscriber_mapping_event.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index c01540fe6..3d3ff2878 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -25,18 +25,25 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { deamon.run(listener).await.expect("failed to complete task"); }); - let (fis, mut reports) = file_info_stream(); + let (fis, mut reports, public_key_binary) = file_info_stream(); tx.send(fis).await?; let mut retry = 0; const MAX_RETRIES: u32 = 10; const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); while retry <= MAX_RETRIES { - let saved_vmes = select_events(&pool).await?; + let mut saved_vmes = select_events(&pool).await?; + if reports.len() == saved_vmes.len() { + // We have to do this because we do not store carrier_mapping_key in DB + for vme in &mut saved_vmes { + vme.carrier_mapping_key = public_key_binary.clone(); + println!("vme {:?}", vme); + } + assert!(reports.iter_mut().all(|r| { - // We have to do this because we do not store carrier_mapping_key in DB - r.report.carrier_mapping_key = vec![].into(); + println!("report {:?}", r.report); + saved_vmes.contains(&r.report) })); break; @@ -74,6 +81,7 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { fn file_info_stream() -> ( FileInfoStream, Vec, + PublicKeyBinary, ) { let file_info = FileInfo { key: "test_file_info".to_string(), @@ -117,6 +125,7 @@ fn file_info_stream() -> ( ( FileInfoStream::new("default".to_string(), file_info, reports.clone()), reports, + public_key_binary, ) } From e82fd9389af48cb763de8837cf261efa25f27e99 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 5 Aug 2024 11:31:24 -0700 Subject: [PATCH 30/44] Fix timestamp issue --- .../verified_subscriber_mapping_event.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index 3d3ff2878..33d256c92 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -35,15 +35,20 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { let mut saved_vmes = select_events(&pool).await?; if reports.len() == saved_vmes.len() { - // We have to do this because we do not store carrier_mapping_key in DB + let now = Utc::now(); + for vme in &mut saved_vmes { + // We have to do this because we do not store carrier_mapping_key in DB vme.carrier_mapping_key = public_key_binary.clone(); - println!("vme {:?}", vme); + // We also update timestamp because github action return a different timestamp + // just few nanoseconds later + vme.timestamp = now; + println!("vme {:?}", vme); } assert!(reports.iter_mut().all(|r| { + r.report.timestamp = now; println!("report {:?}", r.report); - saved_vmes.contains(&r.report) })); break; @@ -78,6 +83,11 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { Ok(()) } +fn are_timestamps_within_one_millisecond(t1: DateTime, t2: DateTime) -> bool { + let duration = t1.signed_duration_since(t2); + duration.num_milliseconds() < 1 +} + fn file_info_stream() -> ( FileInfoStream, Vec, From 36459f44c901d0db74f16dd3dffc4ac33190c415 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 5 Aug 2024 11:33:09 -0700 Subject: [PATCH 31/44] Remove unused fn --- .../tests/integrations/verified_subscriber_mapping_event.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index 33d256c92..01c000a7b 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -83,11 +83,6 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { Ok(()) } -fn are_timestamps_within_one_millisecond(t1: DateTime, t2: DateTime) -> bool { - let duration = t1.signed_duration_since(t2); - duration.num_milliseconds() < 1 -} - fn file_info_stream() -> ( FileInfoStream, Vec, From ae4e19296b918c994107bd6e0d55e9f9cb195c32 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 7 Aug 2024 09:29:33 -0700 Subject: [PATCH 32/44] Rename for consistency --- mobile_verifier/src/reward_shares.rs | 16 ++++++------- mobile_verifier/src/rewarder.rs | 4 ++-- .../src/verified_subscriber_mapping_event.rs | 10 ++++---- .../verified_subscriber_mapping_event.rs | 24 +++++++++---------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 98c3ce4da..727a25d21 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -6,7 +6,7 @@ use crate::{ seniority::Seniority, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, - verified_subscriber_mapping_event::VerifiedMappingEventShares, + verified_subscriber_mapping_event::VerifiedSubscriberMappingEventShares, }; use chrono::{DateTime, Duration, Utc}; use coverage_point_calculator::SPBoostedRewardEligibility; @@ -188,13 +188,13 @@ impl TransferRewards { #[derive(Default)] pub struct MapperShares { pub discovery_mapping_shares: SubscriberValidatedLocations, - pub verified_mapping_event_shares: VerifiedMappingEventShares, + pub verified_mapping_event_shares: VerifiedSubscriberMappingEventShares, } impl MapperShares { pub fn new( discovery_mapping_shares: SubscriberValidatedLocations, - verified_mapping_event_shares: VerifiedMappingEventShares, + verified_mapping_event_shares: VerifiedSubscriberMappingEventShares, ) -> Self { Self { discovery_mapping_shares, @@ -896,7 +896,7 @@ mod test { speedtests::Speedtest, speedtests_average::SpeedtestAverage, subscriber_location::SubscriberValidatedLocations, - verified_subscriber_mapping_event::VerifiedMappingEventShare, + verified_subscriber_mapping_event::VerifiedSubscriberMappingEventShare, }; use chrono::{Duration, Utc}; use file_store::speedtest::CellSpeedtest; @@ -949,10 +949,10 @@ mod test { location_shares.push(n.encode_to_vec()); } - // simulate 10k vme shares - let mut vme_shares = VerifiedMappingEventShares::new(); + // simulate 10k vsme shares + let mut vsme_shares = VerifiedSubscriberMappingEventShares::new(); for n in 0..NUM_SUBSCRIBERS { - vme_shares.push(VerifiedMappingEventShare { + vsme_shares.push(VerifiedSubscriberMappingEventShare { subscriber_id: n.encode_to_vec(), total_reward_points: 30, }); @@ -963,7 +963,7 @@ mod test { let epoch = (now - Duration::hours(24))..now; // translate location shares into shares - let shares = MapperShares::new(location_shares, vme_shares); + let shares = MapperShares::new(location_shares, vsme_shares); let total_mappers_pool = reward_shares::get_scheduled_tokens_for_mappers(epoch.end - epoch.start); let rewards_per_share = shares.rewards_per_share(total_mappers_pool).unwrap(); diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index a31c8efb8..e182b2a38 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -512,12 +512,12 @@ pub async fn reward_mappers( let location_shares = subscriber_location::aggregate_location_shares(pool, reward_period).await?; - let vme_shares = + let vsme_shares = verified_subscriber_mapping_event::aggregate_verified_mapping_events(pool, reward_period) .await?; // determine mapping shares based on location shares and data transferred - let mapping_shares = MapperShares::new(location_shares, vme_shares); + let mapping_shares = MapperShares::new(location_shares, vsme_shares); let total_mappers_pool = reward_shares::get_scheduled_tokens_for_mappers(reward_period.end - reward_period.start); let rewards_per_share = mapping_shares.rewards_per_share(total_mappers_pool)?; diff --git a/mobile_verifier/src/verified_subscriber_mapping_event.rs b/mobile_verifier/src/verified_subscriber_mapping_event.rs index 0e02a6fdc..f7dade654 100644 --- a/mobile_verifier/src/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/src/verified_subscriber_mapping_event.rs @@ -115,10 +115,10 @@ async fn save_event( } const SUBSCRIBER_REWARD_PERIOD_IN_DAYS: i64 = 1; -pub type VerifiedMappingEventShares = Vec; +pub type VerifiedSubscriberMappingEventShares = Vec; #[derive(sqlx::FromRow, PartialEq, Debug)] -pub struct VerifiedMappingEventShare { +pub struct VerifiedSubscriberMappingEventShare { pub subscriber_id: Vec, pub total_reward_points: i64, } @@ -126,8 +126,8 @@ pub struct VerifiedMappingEventShare { pub async fn aggregate_verified_mapping_events( db: impl sqlx::PgExecutor<'_> + Copy, reward_period: &Range>, -) -> Result { - let vme_shares = sqlx::query_as::<_, VerifiedMappingEventShare>( +) -> Result { + let vsme_shares = sqlx::query_as::<_, VerifiedSubscriberMappingEventShare>( "SELECT subscriber_id, SUM(total_reward_points) AS total_reward_points @@ -142,5 +142,5 @@ pub async fn aggregate_verified_mapping_events( .fetch_all(db) .await?; - Ok(vme_shares) + Ok(vsme_shares) } diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs index 01c000a7b..2107f8932 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs @@ -7,8 +7,8 @@ use file_store::{ }; use helium_crypto::{KeyTag, Keypair, PublicKeyBinary}; use mobile_verifier::verified_subscriber_mapping_event::{ - aggregate_verified_mapping_events, VerifiedMappingEventShare, VerifiedMappingEventShares, - VerifiedSubscriberMappingEventDeamon, + aggregate_verified_mapping_events, VerifiedSubscriberMappingEventDeamon, + VerifiedSubscriberMappingEventShare, VerifiedSubscriberMappingEventShares, }; use rand::rngs::OsRng; use sqlx::{PgPool, Pool, Postgres, Row}; @@ -32,28 +32,28 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { const MAX_RETRIES: u32 = 10; const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); while retry <= MAX_RETRIES { - let mut saved_vmes = select_events(&pool).await?; + let mut saved_vsmes = select_events(&pool).await?; - if reports.len() == saved_vmes.len() { + if reports.len() == saved_vsmes.len() { let now = Utc::now(); - for vme in &mut saved_vmes { + for vsme in &mut saved_vsmes { // We have to do this because we do not store carrier_mapping_key in DB - vme.carrier_mapping_key = public_key_binary.clone(); + vsme.carrier_mapping_key = public_key_binary.clone(); // We also update timestamp because github action return a different timestamp // just few nanoseconds later - vme.timestamp = now; - println!("vme {:?}", vme); + vsme.timestamp = now; + println!("vsme {:?}", vsme); } assert!(reports.iter_mut().all(|r| { r.report.timestamp = now; println!("report {:?}", r.report); - saved_vmes.contains(&r.report) + saved_vsmes.contains(&r.report) })); break; } else { - tracing::debug!("wrong saved_vmes.len() {}", saved_vmes.len()); + tracing::debug!("wrong saved_vsmes.len() {}", saved_vsmes.len()); retry += 1; tokio::time::sleep(RETRY_WAIT).await; } @@ -136,7 +136,7 @@ fn file_info_stream() -> ( fn reports_to_shares( reports: Vec, -) -> VerifiedMappingEventShares { +) -> VerifiedSubscriberMappingEventShares { let mut reward_map: HashMap, i64> = HashMap::new(); for report in reports { @@ -148,7 +148,7 @@ fn reports_to_shares( reward_map .into_iter() .map( - |(subscriber_id, total_reward_points)| VerifiedMappingEventShare { + |(subscriber_id, total_reward_points)| VerifiedSubscriberMappingEventShare { subscriber_id, total_reward_points, }, From 8a8c6539c3bc199760150d3a207c2e981b9baf89 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 7 Aug 2024 12:48:57 -0700 Subject: [PATCH 33/44] Add verified ingest report and rename --- file_store/src/file_info.rs | 30 +- file_store/src/lib.rs | 5 +- ...s => subscriber_verified_mapping_event.rs} | 22 +- ...er_verified_mapping_event_ingest_report.rs | 62 ++++ file_store/src/traits/msg_verify.rs | 2 +- ..._subscriber_mapping_event_ingest_report.rs | 45 --- ...er_verified_mapping_event_ingest_report.rs | 69 +++++ ingest/src/server_mobile.rs | 20 +- ingest/tests/common/mod.rs | 10 +- ingest/tests/mobile_ingest.rs | 6 +- .../migrations/35_verified_mapping_event.sql | 7 - ...fied_subscriber_verified_mapping_event.sql | 7 + mobile_verifier/src/cli/server.rs | 12 +- mobile_verifier/src/lib.rs | 2 +- mobile_verifier/src/reward_shares.rs | 12 +- mobile_verifier/src/rewarder.rs | 4 +- .../src/subscriber_verified_mapping_event.rs | 279 ++++++++++++++++++ .../src/verified_subscriber_mapping_event.rs | 146 --------- .../tests/integrations/common/mod.rs | 59 +++- mobile_verifier/tests/integrations/main.rs | 2 +- ...s => subscriber_verified_mapping_event.rs} | 115 ++++++-- 21 files changed, 630 insertions(+), 286 deletions(-) rename file_store/src/{verified_subscriber_mapping_event.rs => subscriber_verified_mapping_event.rs} (65%) create mode 100644 file_store/src/subscriber_verified_mapping_event_ingest_report.rs delete mode 100644 file_store/src/verified_subscriber_mapping_event_ingest_report.rs create mode 100644 file_store/src/verified_subscriber_verified_mapping_event_ingest_report.rs delete mode 100644 mobile_verifier/migrations/35_verified_mapping_event.sql create mode 100644 mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql create mode 100644 mobile_verifier/src/subscriber_verified_mapping_event.rs delete mode 100644 mobile_verifier/src/verified_subscriber_mapping_event.rs rename mobile_verifier/tests/integrations/{verified_subscriber_mapping_event.rs => subscriber_verified_mapping_event.rs} (52%) diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index 4061bf7e8..7c486d99c 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -160,8 +160,10 @@ pub const LANDTYPE_DATA_SET: &str = "landtype"; pub const SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "service_provider_boosted_rewards_banned_radio"; pub const VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "verified_service_provider_boosted_rewards_banned_radio"; -pub const VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT: &str = - "verified_subscriber_mapping_ingest_report"; +pub const SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT: &str = + "subscriber_verified_mapping_ingest_report"; +pub const VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT: &str = + "verified_subscriber_verified_mapping_ingest_report"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -216,7 +218,8 @@ pub enum FileType { VerifiedInvalidatedRadioThresholdIngestReport, SPBoostedRewardsBannedRadioIngestReport, VerifiedSPBoostedRewardsBannedRadioIngestReport, - VerifiedSubscriberMappingEventIngestReportV1, + SubscriberVerifiedMappingEventIngestReport, + VerifiedSubscriberVerifiedMappingEventIngestReport, } impl fmt::Display for FileType { @@ -282,8 +285,11 @@ impl fmt::Display for FileType { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } - Self::VerifiedSubscriberMappingEventIngestReportV1 => { - VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT + Self::SubscriberVerifiedMappingEventIngestReport => { + SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT + } + Self::VerifiedSubscriberVerifiedMappingEventIngestReport => { + VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT } }; f.write_str(s) @@ -353,8 +359,11 @@ impl FileType { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } - Self::VerifiedSubscriberMappingEventIngestReportV1 => { - VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT + Self::SubscriberVerifiedMappingEventIngestReport => { + SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT + } + Self::VerifiedSubscriberVerifiedMappingEventIngestReport => { + VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT } } } @@ -424,8 +433,11 @@ impl FromStr for FileType { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO => { Self::VerifiedSPBoostedRewardsBannedRadioIngestReport } - VERIFIED_SUBSCRIBER_MAPPING_INGEST_REPORT => { - Self::VerifiedSubscriberMappingEventIngestReportV1 + SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT => { + Self::SubscriberVerifiedMappingEventIngestReport + } + VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT => { + Self::VerifiedSubscriberVerifiedMappingEventIngestReport } _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index 30110f3d4..477c0dea9 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -23,9 +23,10 @@ pub mod mobile_transfer; pub mod reward_manifest; mod settings; pub mod speedtest; +pub mod subscriber_verified_mapping_event; +pub mod subscriber_verified_mapping_event_ingest_report; pub mod traits; -pub mod verified_subscriber_mapping_event; -pub mod verified_subscriber_mapping_event_ingest_report; +pub mod verified_subscriber_verified_mapping_event_ingest_report; pub mod wifi_heartbeat; pub use crate::file_store::FileStore; diff --git a/file_store/src/verified_subscriber_mapping_event.rs b/file_store/src/subscriber_verified_mapping_event.rs similarity index 65% rename from file_store/src/verified_subscriber_mapping_event.rs rename to file_store/src/subscriber_verified_mapping_event.rs index f76a014f2..f83630619 100644 --- a/file_store/src/verified_subscriber_mapping_event.rs +++ b/file_store/src/subscriber_verified_mapping_event.rs @@ -4,37 +4,37 @@ use crate::{ }; use chrono::{DateTime, Utc}; use helium_crypto::PublicKeyBinary; -use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventReqV1; +use helium_proto::services::poc_mobile::SubscriberVerifiedMappingEventReqV1; use serde::{Deserialize, Serialize}; #[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] -pub struct VerifiedSubscriberMappingEvent { +pub struct SubscriberVerifiedMappingEvent { pub subscriber_id: Vec, pub total_reward_points: u64, pub timestamp: DateTime, pub carrier_mapping_key: PublicKeyBinary, } -impl MsgDecode for VerifiedSubscriberMappingEvent { - type Msg = VerifiedSubscriberMappingEventReqV1; +impl MsgDecode for SubscriberVerifiedMappingEvent { + type Msg = SubscriberVerifiedMappingEventReqV1; } -impl MsgTimestamp>> for VerifiedSubscriberMappingEventReqV1 { +impl MsgTimestamp>> for SubscriberVerifiedMappingEventReqV1 { fn timestamp(&self) -> Result> { self.timestamp.to_timestamp() } } -impl MsgTimestamp for VerifiedSubscriberMappingEvent { +impl MsgTimestamp for SubscriberVerifiedMappingEvent { fn timestamp(&self) -> u64 { self.timestamp.encode_timestamp() } } -impl From for VerifiedSubscriberMappingEventReqV1 { - fn from(v: VerifiedSubscriberMappingEvent) -> Self { +impl From for SubscriberVerifiedMappingEventReqV1 { + fn from(v: SubscriberVerifiedMappingEvent) -> Self { let timestamp = v.timestamp(); - VerifiedSubscriberMappingEventReqV1 { + SubscriberVerifiedMappingEventReqV1 { subscriber_id: v.subscriber_id, total_reward_points: v.total_reward_points, timestamp, @@ -44,9 +44,9 @@ impl From for VerifiedSubscriberMappingEventReqV } } -impl TryFrom for VerifiedSubscriberMappingEvent { +impl TryFrom for SubscriberVerifiedMappingEvent { type Error = Error; - fn try_from(v: VerifiedSubscriberMappingEventReqV1) -> Result { + fn try_from(v: SubscriberVerifiedMappingEventReqV1) -> Result { let timestamp = v.timestamp()?; Ok(Self { subscriber_id: v.subscriber_id, diff --git a/file_store/src/subscriber_verified_mapping_event_ingest_report.rs b/file_store/src/subscriber_verified_mapping_event_ingest_report.rs new file mode 100644 index 000000000..1bb08a761 --- /dev/null +++ b/file_store/src/subscriber_verified_mapping_event_ingest_report.rs @@ -0,0 +1,62 @@ +use crate::{ + subscriber_verified_mapping_event::SubscriberVerifiedMappingEvent, + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_proto::services::poc_mobile::{ + SubscriberVerifiedMappingEventIngestReportV1, SubscriberVerifiedMappingEventReqV1, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct SubscriberVerifiedMappingEventIngestReport { + pub received_timestamp: DateTime, + pub report: SubscriberVerifiedMappingEvent, +} + +impl MsgDecode for SubscriberVerifiedMappingEventIngestReport { + type Msg = SubscriberVerifiedMappingEventIngestReportV1; +} + +impl MsgTimestamp>> for SubscriberVerifiedMappingEventIngestReportV1 { + fn timestamp(&self) -> Result> { + self.received_timestamp.to_timestamp() + } +} + +impl MsgTimestamp for SubscriberVerifiedMappingEventIngestReport { + fn timestamp(&self) -> u64 { + self.received_timestamp.encode_timestamp() + } +} + +impl From + for SubscriberVerifiedMappingEventIngestReportV1 +{ + fn from(v: SubscriberVerifiedMappingEventIngestReport) -> Self { + let received_timestamp = v.timestamp(); + let report: SubscriberVerifiedMappingEventReqV1 = v.report.into(); + Self { + received_timestamp, + report: Some(report), + } + } +} + +impl TryFrom + for SubscriberVerifiedMappingEventIngestReport +{ + type Error = Error; + fn try_from(v: SubscriberVerifiedMappingEventIngestReportV1) -> Result { + Ok(Self { + received_timestamp: v.timestamp()?, + report: v + .report + .ok_or_else(|| { + Error::not_found("ingest SubscriberVerifiedMappingEventIngestReport report") + })? + .try_into()?, + }) + } +} diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index fb75dd78d..47bb6cb40 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -94,7 +94,7 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamResV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); -impl_msg_verify!(poc_mobile::VerifiedSubscriberMappingEventReqV1, signature); +impl_msg_verify!(poc_mobile::SubscriberVerifiedMappingEventReqV1, signature); #[cfg(test)] mod test { diff --git a/file_store/src/verified_subscriber_mapping_event_ingest_report.rs b/file_store/src/verified_subscriber_mapping_event_ingest_report.rs deleted file mode 100644 index 51799afd0..000000000 --- a/file_store/src/verified_subscriber_mapping_event_ingest_report.rs +++ /dev/null @@ -1,45 +0,0 @@ -use crate::{ - traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, - verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, - Error, Result, -}; -use chrono::{DateTime, Utc}; -use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventIngestReportV1; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] -pub struct VerifiedSubscriberMappingEventIngestReport { - pub received_timestamp: DateTime, - pub report: VerifiedSubscriberMappingEvent, -} - -impl MsgDecode for VerifiedSubscriberMappingEventIngestReport { - type Msg = VerifiedSubscriberMappingEventIngestReportV1; -} - -impl MsgTimestamp>> for VerifiedSubscriberMappingEventIngestReportV1 { - fn timestamp(&self) -> Result> { - self.received_timestamp.to_timestamp() - } -} - -impl MsgTimestamp for VerifiedSubscriberMappingEventIngestReport { - fn timestamp(&self) -> u64 { - self.received_timestamp.encode_timestamp() - } -} - -impl TryFrom - for VerifiedSubscriberMappingEventIngestReport -{ - type Error = Error; - fn try_from(v: VerifiedSubscriberMappingEventIngestReportV1) -> Result { - Ok(Self { - received_timestamp: v.timestamp()?, - report: v - .report - .ok_or_else(|| Error::not_found("ingest VerifiedSubscriberMappingEvent report"))? - .try_into()?, - }) - } -} diff --git a/file_store/src/verified_subscriber_verified_mapping_event_ingest_report.rs b/file_store/src/verified_subscriber_verified_mapping_event_ingest_report.rs new file mode 100644 index 000000000..5fbbe04c4 --- /dev/null +++ b/file_store/src/verified_subscriber_verified_mapping_event_ingest_report.rs @@ -0,0 +1,69 @@ +use crate::{ + subscriber_verified_mapping_event_ingest_report::SubscriberVerifiedMappingEventIngestReport, + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_proto::services::poc_mobile::{ + SubscriberVerifiedMappingEventIngestReportV1, SubscriberVerifiedMappingEventVerificationStatus, + VerifiedSubscriberVerifiedMappingEventIngestReportV1, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct VerifiedSubscriberVerifiedMappingEventIngestReport { + pub report: SubscriberVerifiedMappingEventIngestReport, + pub status: SubscriberVerifiedMappingEventVerificationStatus, + pub timestamp: DateTime, +} + +impl MsgDecode for VerifiedSubscriberVerifiedMappingEventIngestReport { + type Msg = VerifiedSubscriberVerifiedMappingEventIngestReportV1; +} + +impl MsgTimestamp>> for VerifiedSubscriberVerifiedMappingEventIngestReportV1 { + fn timestamp(&self) -> Result> { + self.timestamp.to_timestamp() + } +} + +impl MsgTimestamp for VerifiedSubscriberVerifiedMappingEventIngestReport { + fn timestamp(&self) -> u64 { + self.timestamp.encode_timestamp() + } +} + +impl From + for VerifiedSubscriberVerifiedMappingEventIngestReportV1 +{ + fn from(v: VerifiedSubscriberVerifiedMappingEventIngestReport) -> Self { + let timestamp = v.timestamp(); + let report: SubscriberVerifiedMappingEventIngestReportV1 = v.report.into(); + Self { + report: Some(report), + status: v.status as i32, + timestamp, + } + } +} + +impl TryFrom + for VerifiedSubscriberVerifiedMappingEventIngestReport +{ + type Error = Error; + fn try_from(v: VerifiedSubscriberVerifiedMappingEventIngestReportV1) -> Result { + Ok(Self { + report: v + .clone() + .report + .ok_or_else(|| { + Error::not_found( + "ingest VerifiedSubscriberVerifiedMappingEventIngestReport report", + ) + })? + .try_into()?, + status: v.status.try_into()?, + timestamp: v.timestamp()?, + }) + } +} diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index bead852ef..db2008875 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -20,8 +20,8 @@ use helium_proto::services::poc_mobile::{ ServiceProviderBoostedRewardsBannedRadioReqV1, ServiceProviderBoostedRewardsBannedRadioRespV1, SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, SubscriberLocationIngestReportV1, SubscriberLocationReqV1, SubscriberLocationRespV1, - VerifiedSubscriberMappingEventIngestReportV1, VerifiedSubscriberMappingEventReqV1, - VerifiedSubscriberMappingEventResV1, WifiHeartbeatIngestReportV1, WifiHeartbeatReqV1, + SubscriberVerifiedMappingEventIngestReportV1, SubscriberVerifiedMappingEventReqV1, + SubscriberVerifiedMappingEventResV1, WifiHeartbeatIngestReportV1, WifiHeartbeatReqV1, WifiHeartbeatRespV1, }; use std::{net::SocketAddr, path::Path}; @@ -408,12 +408,12 @@ impl poc_mobile::PocMobile for GrpcServer { )) } - async fn submit_verified_subscriber_mapping_event( + async fn submit_subscriber_verified_mapping_event( &self, - request: Request, - ) -> GrpcResult { + request: Request, + ) -> GrpcResult { let timestamp: u64 = Utc::now().timestamp_millis() as u64; - let event: VerifiedSubscriberMappingEventReqV1 = request.into_inner(); + let event: SubscriberVerifiedMappingEventReqV1 = request.into_inner(); let subscriber_id = event.subscriber_id.clone(); custom_tracing::record("subscriber_id", bs58::encode(&subscriber_id).into_string()); @@ -423,7 +423,7 @@ impl poc_mobile::PocMobile for GrpcServer { .verify_public_key(event.carrier_mapping_key.as_ref()) .and_then(|public_key| self.verify_network(public_key)) .and_then(|public_key| self.verify_signature(public_key, event)) - .map(|(_, event)| VerifiedSubscriberMappingEventIngestReportV1 { + .map(|(_, event)| SubscriberVerifiedMappingEventIngestReportV1 { received_timestamp: timestamp, report: Some(event), })?; @@ -431,7 +431,7 @@ impl poc_mobile::PocMobile for GrpcServer { _ = self.subscriber_mapping_event_sink.write(report, []).await; let id = timestamp.to_string(); - Ok(Response::new(VerifiedSubscriberMappingEventResV1 { id })) + Ok(Response::new(SubscriberVerifiedMappingEventResV1 { id })) } } @@ -551,12 +551,12 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { let (subscriber_mapping_event_sink, subscriber_mapping_event_server) = file_sink::FileSinkBuilder::new( - FileType::VerifiedSubscriberMappingEventIngestReportV1, + FileType::SubscriberVerifiedMappingEventIngestReport, store_base_path, file_upload.clone(), concat!( env!("CARGO_PKG_NAME"), - "_verified_subscriber_mapping_event_ingest_report" + "_subscriber_verified_mapping_event_ingest_report" ), ) .roll_time(settings.roll_time) diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index 8cc3edfee..a42167357 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -3,8 +3,8 @@ use backon::{ExponentialBuilder, Retryable}; use file_store::file_sink::FileSinkClient; use helium_crypto::{KeyTag, Keypair, Network, Sign}; use helium_proto::services::poc_mobile::{ - Client as PocMobileClient, VerifiedSubscriberMappingEventReqV1, - VerifiedSubscriberMappingEventResV1, + Client as PocMobileClient, SubscriberVerifiedMappingEventReqV1, + SubscriberVerifiedMappingEventResV1, }; use ingest::server_mobile::GrpcServer; use prost::Message; @@ -105,8 +105,8 @@ impl TestClient { &mut self, subscriber_id: Vec, total_reward_points: u64, - ) -> anyhow::Result { - let mut req = VerifiedSubscriberMappingEventReqV1 { + ) -> anyhow::Result { + let mut req = SubscriberVerifiedMappingEventReqV1 { subscriber_id, total_reward_points, timestamp: 0, @@ -123,7 +123,7 @@ impl TestClient { let res = self .client - .submit_verified_subscriber_mapping_event(request) + .submit_subscriber_verified_mapping_event(request) .await?; Ok(res.into_inner()) diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index 50a770c1f..1f0f8da05 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -1,4 +1,4 @@ -use helium_proto::services::poc_mobile::VerifiedSubscriberMappingEventIngestReportV1; +use helium_proto::services::poc_mobile::SubscriberVerifiedMappingEventIngestReportV1; use prost::Message; mod common; @@ -20,8 +20,8 @@ async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { match client.recv().await { Ok(data) => { - let report = VerifiedSubscriberMappingEventIngestReportV1::decode(data.as_slice()) - .expect("unable to decode into VerifiedSubscriberMappingEventIngestReportV1"); + let report = SubscriberVerifiedMappingEventIngestReportV1::decode(data.as_slice()) + .expect("unable to decode into SubscriberVerifiedMappingEventIngestReportV1"); assert_eq!(timestamp, report.received_timestamp.to_string()); diff --git a/mobile_verifier/migrations/35_verified_mapping_event.sql b/mobile_verifier/migrations/35_verified_mapping_event.sql deleted file mode 100644 index 12eb4cedd..000000000 --- a/mobile_verifier/migrations/35_verified_mapping_event.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE IF NOT EXISTS verified_mapping_event ( - subscriber_id BYTEA NOT NULL, - total_reward_points INTEGER NOT NULL, - timestamp TIMESTAMPTZ, - inserted_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (subscriber_id, timestamp) -); \ No newline at end of file diff --git a/mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql b/mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql new file mode 100644 index 000000000..2359dfb03 --- /dev/null +++ b/mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS verified_subscriber_verified_mapping_event ( + subscriber_id BYTEA NOT NULL, + total_reward_points INTEGER NOT NULL, + received_timestamp TIMESTAMPTZ, + inserted_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (subscriber_id, received_timestamp) +); \ No newline at end of file diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index b5d2b7f7b..dbf5fd7d2 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -11,9 +11,8 @@ use crate::{ sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, - telemetry, - verified_subscriber_mapping_event::VerifiedSubscriberMappingEventDeamon, - Settings, + subscriber_verified_mapping_event::SubscriberVerifiedMappingEventDeamon, + telemetry, Settings, }; use anyhow::Result; use file_store::{file_sink, file_upload, FileStore, FileType}; @@ -144,10 +143,13 @@ impl Cmd { .await?, ) .add_task( - VerifiedSubscriberMappingEventDeamon::create_managed_task( + SubscriberVerifiedMappingEventDeamon::create_managed_task( pool.clone(), settings, + auth_client.clone(), + entity_client.clone(), report_ingest.clone(), + file_upload.clone(), ) .await?, ) @@ -178,7 +180,7 @@ impl Cmd { file_upload.clone(), report_ingest.clone(), auth_client.clone(), - entity_client, + entity_client.clone(), ) .await?, ) diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index f7b182503..4855f8a63 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -14,8 +14,8 @@ pub mod sp_boosted_rewards_bans; pub mod speedtests; pub mod speedtests_average; pub mod subscriber_location; +pub mod subscriber_verified_mapping_event; pub mod telemetry; -pub mod verified_subscriber_mapping_event; pub use settings::Settings; diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 727a25d21..ed6fa4556 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -6,7 +6,7 @@ use crate::{ seniority::Seniority, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, - verified_subscriber_mapping_event::VerifiedSubscriberMappingEventShares, + subscriber_verified_mapping_event::VerifiedSubscriberVerifiedMappingEventShares, }; use chrono::{DateTime, Duration, Utc}; use coverage_point_calculator::SPBoostedRewardEligibility; @@ -188,13 +188,13 @@ impl TransferRewards { #[derive(Default)] pub struct MapperShares { pub discovery_mapping_shares: SubscriberValidatedLocations, - pub verified_mapping_event_shares: VerifiedSubscriberMappingEventShares, + pub verified_mapping_event_shares: VerifiedSubscriberVerifiedMappingEventShares, } impl MapperShares { pub fn new( discovery_mapping_shares: SubscriberValidatedLocations, - verified_mapping_event_shares: VerifiedSubscriberMappingEventShares, + verified_mapping_event_shares: VerifiedSubscriberVerifiedMappingEventShares, ) -> Self { Self { discovery_mapping_shares, @@ -896,7 +896,7 @@ mod test { speedtests::Speedtest, speedtests_average::SpeedtestAverage, subscriber_location::SubscriberValidatedLocations, - verified_subscriber_mapping_event::VerifiedSubscriberMappingEventShare, + subscriber_verified_mapping_event::VerifiedSubscriberVerifiedMappingEventShare, }; use chrono::{Duration, Utc}; use file_store::speedtest::CellSpeedtest; @@ -950,9 +950,9 @@ mod test { } // simulate 10k vsme shares - let mut vsme_shares = VerifiedSubscriberMappingEventShares::new(); + let mut vsme_shares = VerifiedSubscriberVerifiedMappingEventShares::new(); for n in 0..NUM_SUBSCRIBERS { - vsme_shares.push(VerifiedSubscriberMappingEventShare { + vsme_shares.push(VerifiedSubscriberVerifiedMappingEventShare { subscriber_id: n.encode_to_vec(), total_reward_points: 30, }); diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index e182b2a38..d17ee903a 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -9,7 +9,7 @@ use crate::{ }, sp_boosted_rewards_bans, speedtests, speedtests_average::SpeedtestAverages, - subscriber_location, telemetry, verified_subscriber_mapping_event, Settings, + subscriber_location, subscriber_verified_mapping_event, telemetry, Settings, }; use anyhow::bail; use chrono::{DateTime, TimeZone, Utc}; @@ -513,7 +513,7 @@ pub async fn reward_mappers( subscriber_location::aggregate_location_shares(pool, reward_period).await?; let vsme_shares = - verified_subscriber_mapping_event::aggregate_verified_mapping_events(pool, reward_period) + subscriber_verified_mapping_event::aggregate_verified_mapping_events(pool, reward_period) .await?; // determine mapping shares based on location shares and data transferred diff --git a/mobile_verifier/src/subscriber_verified_mapping_event.rs b/mobile_verifier/src/subscriber_verified_mapping_event.rs new file mode 100644 index 000000000..a5e470a9b --- /dev/null +++ b/mobile_verifier/src/subscriber_verified_mapping_event.rs @@ -0,0 +1,279 @@ +use crate::Settings; +use chrono::{DateTime, Duration, Utc}; +use file_store::{ + file_info_poller::{FileInfoStream, LookbackBehavior}, + file_sink::{self, FileSinkClient}, + file_source, + file_upload::FileUpload, + subscriber_verified_mapping_event::SubscriberVerifiedMappingEvent, + subscriber_verified_mapping_event_ingest_report::SubscriberVerifiedMappingEventIngestReport, + verified_subscriber_verified_mapping_event_ingest_report::VerifiedSubscriberVerifiedMappingEventIngestReport, + FileStore, FileType, +}; +use futures::{stream::StreamExt, TryStreamExt}; +use helium_crypto::PublicKeyBinary; +use helium_proto::services::{ + mobile_config::NetworkKeyRole, + poc_mobile::{ + SubscriberVerifiedMappingEventVerificationStatus, + VerifiedSubscriberVerifiedMappingEventIngestReportV1, + }, +}; +use mobile_config::client::{ + authorization_client::AuthorizationVerifier, entity_client::EntityVerifier, +}; +use sqlx::{Pool, Postgres, Transaction}; +use std::ops::Range; +use task_manager::{ManagedTask, TaskManager}; +use tokio::sync::mpsc::Receiver; + +pub struct SubscriberVerifiedMappingEventDeamon { + pool: Pool, + authorization_verifier: AV, + entity_verifier: EV, + reports_receiver: Receiver>, + verified_report_sink: FileSinkClient, +} + +impl SubscriberVerifiedMappingEventDeamon +where + AV: AuthorizationVerifier + Send + Sync + 'static, + EV: EntityVerifier + Send + Sync + 'static, +{ + pub fn new( + pool: Pool, + authorization_verifier: AV, + entity_verifier: EV, + reports_receiver: Receiver>, + verified_report_sink: FileSinkClient, + ) -> Self { + Self { + pool, + authorization_verifier, + entity_verifier, + reports_receiver, + verified_report_sink, + } + } + + pub async fn create_managed_task( + pool: Pool, + settings: &Settings, + authorization_verifier: AV, + entity_verifier: EV, + file_store: FileStore, + file_upload: FileUpload, + ) -> anyhow::Result { + let (reports_receiver, reports_receiver_server) = + file_source::continuous_source::() + .state(pool.clone()) + .store(file_store) + .lookback(LookbackBehavior::StartAfter(settings.start_after)) + .prefix(FileType::SubscriberVerifiedMappingEventIngestReport.to_string()) + .create() + .await?; + + let (verified_report_sink, verified_report_sink_server) = file_sink::FileSinkBuilder::new( + FileType::VerifiedSubscriberVerifiedMappingEventIngestReport, + settings.store_base_path(), + file_upload.clone(), + concat!( + env!("CARGO_PKG_NAME"), + "_verified_subscriber_verified_mapping_event_ingest_report" + ), + ) + .auto_commit(false) + .create() + .await?; + + let task = Self::new( + pool, + authorization_verifier, + entity_verifier, + reports_receiver, + verified_report_sink, + ); + + Ok(TaskManager::builder() + .add_task(reports_receiver_server) + .add_task(verified_report_sink_server) + .add_task(task) + .build()) + } + + pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + tracing::info!("Starting sme deamon"); + loop { + tokio::select! { + biased; + _ = shutdown.clone() => { + tracing::info!("sme deamon shutting down"); + break; + } + Some(file) = self.reports_receiver.recv() => { + self.process_file(file).await?; + } + } + } + Ok(()) + } + + async fn process_file( + &self, + file_info_stream: FileInfoStream, + ) -> anyhow::Result<()> { + tracing::info!( + "Processing Verified Mapping Event file {}", + file_info_stream.file_info.key + ); + + let mut transaction = self.pool.begin().await?; + + file_info_stream + .into_stream(&mut transaction) + .await? + .map(anyhow::Ok) + .try_fold(transaction, |mut transaction, report: SubscriberVerifiedMappingEventIngestReport| async move { + // verifiy the report + let verified_report_status = self.verify_event(&report.report).await; + + // if the report is valid then save to the db + // and thus available to be rewarded + if verified_report_status + == SubscriberVerifiedMappingEventVerificationStatus::SvmeValid + { + save_to_db(&report, &mut transaction).await?; + } + + // write out paper trail of verified report, valid or invalid + let verified_report_proto: VerifiedSubscriberVerifiedMappingEventIngestReportV1 = + VerifiedSubscriberVerifiedMappingEventIngestReport { + report, + status: verified_report_status, + timestamp: Utc::now(), + } + .into(); + + self.verified_report_sink + .write( + verified_report_proto, + &[("report_status", verified_report_status.as_str_name())], + ) + .await?; + + Ok(transaction) + }) + .await? + .commit() + .await?; + + self.verified_report_sink.commit().await?; + + Ok(()) + } + + async fn verify_event( + &self, + event: &SubscriberVerifiedMappingEvent, + ) -> SubscriberVerifiedMappingEventVerificationStatus { + if !self + .verify_known_carrier_key(&event.carrier_mapping_key) + .await + { + return SubscriberVerifiedMappingEventVerificationStatus::SvmeInvalidCarrierKey; + } + + if !self.verify_subscriber_id(&event.subscriber_id).await { + return SubscriberVerifiedMappingEventVerificationStatus::SvmeInvalidSubscriberId; + } + + SubscriberVerifiedMappingEventVerificationStatus::SvmeValid + } + + async fn verify_known_carrier_key(&self, public_key: &PublicKeyBinary) -> bool { + match self + .authorization_verifier + .verify_authorized_key(public_key, NetworkKeyRole::MobileCarrier) + .await + { + Ok(res) => res, + Err(_err) => false, + } + } + + async fn verify_subscriber_id(&self, subscriber_id: &[u8]) -> bool { + match self + .entity_verifier + .verify_rewardable_entity(subscriber_id) + .await + { + Ok(res) => res, + Err(_err) => false, + } + } +} + +impl ManagedTask for SubscriberVerifiedMappingEventDeamon +where + AV: AuthorizationVerifier + Send + Sync + 'static, + EV: EntityVerifier + Send + Sync + 'static, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::future::LocalBoxFuture<'static, anyhow::Result<()>> { + Box::pin(self.run(shutdown)) + } +} + +async fn save_to_db( + report: &SubscriberVerifiedMappingEventIngestReport, + exec: &mut Transaction<'_, Postgres>, +) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + INSERT INTO verified_subscriber_verified_mapping_event (subscriber_id, total_reward_points, received_timestamp) + VALUES ($1, $2, $3) + ON CONFLICT (subscriber_id, received_timestamp) DO NOTHING + "#, + ) + .bind(&report.report.subscriber_id) + .bind(report.report.total_reward_points as i64) + .bind(report.received_timestamp) + .execute(exec) + .await?; + + Ok(()) +} + +const SUBSCRIBER_REWARD_PERIOD_IN_DAYS: i64 = 1; +pub type VerifiedSubscriberVerifiedMappingEventShares = + Vec; + +#[derive(sqlx::FromRow, PartialEq, Debug)] +pub struct VerifiedSubscriberVerifiedMappingEventShare { + pub subscriber_id: Vec, + pub total_reward_points: i64, +} + +pub async fn aggregate_verified_mapping_events( + db: impl sqlx::PgExecutor<'_> + Copy, + reward_period: &Range>, +) -> Result { + let vsme_shares = sqlx::query_as::<_, VerifiedSubscriberVerifiedMappingEventShare>( + "SELECT + subscriber_id, + SUM(total_reward_points) AS total_reward_points + FROM + verified_subscriber_verified_mapping_event + WHERE received_timestamp >= $1 AND received_timestamp < $2 + GROUP BY + subscriber_id;", + ) + .bind(reward_period.end - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) + .bind(reward_period.end) + .fetch_all(db) + .await?; + + Ok(vsme_shares) +} diff --git a/mobile_verifier/src/verified_subscriber_mapping_event.rs b/mobile_verifier/src/verified_subscriber_mapping_event.rs deleted file mode 100644 index f7dade654..000000000 --- a/mobile_verifier/src/verified_subscriber_mapping_event.rs +++ /dev/null @@ -1,146 +0,0 @@ -use crate::Settings; -use chrono::{DateTime, Duration, Utc}; -use file_store::{ - file_info_poller::{FileInfoStream, LookbackBehavior}, - file_source, - verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, - verified_subscriber_mapping_event_ingest_report::VerifiedSubscriberMappingEventIngestReport, - FileStore, FileType, -}; -use futures::stream::StreamExt; -use sqlx::{Pool, Postgres, Transaction}; -use std::ops::Range; -use task_manager::{ManagedTask, TaskManager}; -use tokio::sync::mpsc::Receiver; - -pub struct VerifiedSubscriberMappingEventDeamon { - pool: Pool, - events: Receiver>, -} - -impl VerifiedSubscriberMappingEventDeamon { - pub fn new( - pool: Pool, - events: Receiver>, - ) -> Self { - Self { pool, events } - } - - pub async fn create_managed_task( - pool: Pool, - settings: &Settings, - file_store: FileStore, - ) -> anyhow::Result { - let (events, event_server) = - file_source::continuous_source::() - .state(pool.clone()) - .store(file_store) - .lookback(LookbackBehavior::StartAfter(settings.start_after)) - .prefix(FileType::VerifiedSubscriberMappingEventIngestReportV1.to_string()) - .create() - .await?; - - let task = Self::new(pool, events); - - Ok(TaskManager::builder() - .add_task(event_server) - .add_task(task) - .build()) - } - - pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { - tracing::info!("Starting sme deamon"); - loop { - tokio::select! { - biased; - _ = shutdown.clone() => { - tracing::info!("sme deamon shutting down"); - break; - } - Some(file) = self.events.recv() => { - self.process_file(file).await?; - } - } - } - Ok(()) - } - - async fn process_file( - &self, - file: FileInfoStream, - ) -> anyhow::Result<()> { - tracing::info!( - "Processing Verified Mapping Event file {}", - file.file_info.key - ); - let mut transaction = self.pool.begin().await?; - let mut reports = file.into_stream(&mut transaction).await?; - - while let Some(report) = reports.next().await { - save_event(&report.report, &mut transaction).await?; - } - - transaction.commit().await?; - Ok(()) - } -} - -impl ManagedTask for VerifiedSubscriberMappingEventDeamon { - fn start_task( - self: Box, - shutdown: triggered::Listener, - ) -> futures::future::LocalBoxFuture<'static, anyhow::Result<()>> { - Box::pin(self.run(shutdown)) - } -} - -async fn save_event( - event: &VerifiedSubscriberMappingEvent, - exec: &mut Transaction<'_, Postgres>, -) -> Result<(), sqlx::Error> { - sqlx::query( - r#" - insert into verified_mapping_event (subscriber_id, total_reward_points, timestamp) - values ($1, $2, $3) - on conflict (subscriber_id, timestamp) do nothing - "#, - ) - .bind(&event.subscriber_id) - .bind(event.total_reward_points as i64) - .bind(event.timestamp) - .execute(exec) - .await?; - - Ok(()) -} - -const SUBSCRIBER_REWARD_PERIOD_IN_DAYS: i64 = 1; -pub type VerifiedSubscriberMappingEventShares = Vec; - -#[derive(sqlx::FromRow, PartialEq, Debug)] -pub struct VerifiedSubscriberMappingEventShare { - pub subscriber_id: Vec, - pub total_reward_points: i64, -} - -pub async fn aggregate_verified_mapping_events( - db: impl sqlx::PgExecutor<'_> + Copy, - reward_period: &Range>, -) -> Result { - let vsme_shares = sqlx::query_as::<_, VerifiedSubscriberMappingEventShare>( - "SELECT - subscriber_id, - SUM(total_reward_points) AS total_reward_points - FROM - verified_mapping_event - WHERE timestamp >= $1 AND timestamp < $2 - GROUP BY - subscriber_id;", - ) - .bind(reward_period.end - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) - .bind(reward_period.end) - .fetch_all(db) - .await?; - - Ok(vsme_shares) -} diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 7fb27d442..5ba8ab557 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -4,18 +4,26 @@ use file_store::{ traits::TimestampEncode, }; use futures::{stream, StreamExt}; +use helium_crypto::PublicKeyBinary; use helium_proto::{ - services::poc_mobile::{ - mobile_reward_share::Reward as MobileReward, radio_reward_v2, GatewayReward, - MobileRewardShare, OracleBoostingHexAssignment, OracleBoostingReportV1, RadioReward, - RadioRewardV2, ServiceProviderReward, SpeedtestAvg, SubscriberReward, UnallocatedReward, + services::{ + mobile_config::NetworkKeyRole, + poc_mobile::{ + mobile_reward_share::Reward as MobileReward, radio_reward_v2, GatewayReward, + MobileRewardShare, OracleBoostingHexAssignment, OracleBoostingReportV1, RadioReward, + RadioRewardV2, ServiceProviderReward, SpeedtestAvg, SubscriberReward, + UnallocatedReward, + }, }, Message, }; use hex_assignments::{Assignment, HexAssignment, HexBoostData}; use mobile_config::{ boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, - client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, + client::{ + authorization_client::AuthorizationVerifier, entity_client::EntityVerifier, + hex_boosting_client::HexBoostingInfoResolver, ClientError, + }, }; use mobile_verifier::boosting_oracles::AssignedCoverageObjects; @@ -24,6 +32,7 @@ use rust_decimal_macros::dec; use sqlx::PgPool; use std::{collections::HashMap, str::FromStr}; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; +use tonic::async_trait; #[derive(Debug, Clone)] #[allow(dead_code)] @@ -322,3 +331,43 @@ pub async fn set_unassigned_oracle_boosting_assignments( assigned_coverage_objs.save(pool).await?; Ok(output) } + +#[derive(Clone)] +pub struct MockAuthorizationClient {} + +impl MockAuthorizationClient { + pub fn new() -> MockAuthorizationClient { + Self {} + } +} + +#[async_trait] +impl AuthorizationVerifier for MockAuthorizationClient { + type Error = ClientError; + + async fn verify_authorized_key( + &self, + _pubkey: &PublicKeyBinary, + _role: NetworkKeyRole, + ) -> Result { + Ok(true) + } +} + +#[derive(Clone)] +pub struct MockEntityClient {} + +impl MockEntityClient { + pub fn new() -> MockEntityClient { + Self {} + } +} + +#[async_trait] +impl EntityVerifier for MockEntityClient { + type Error = ClientError; + + async fn verify_rewardable_entity(&self, _entity_id: &[u8]) -> Result { + Ok(true) + } +} diff --git a/mobile_verifier/tests/integrations/main.rs b/mobile_verifier/tests/integrations/main.rs index 67ea58951..0980b0065 100644 --- a/mobile_verifier/tests/integrations/main.rs +++ b/mobile_verifier/tests/integrations/main.rs @@ -11,4 +11,4 @@ mod rewarder_poc_dc; mod rewarder_sp_rewards; mod seniority; mod speedtests; -mod verified_subscriber_mapping_event; +mod subscriber_verified_mapping_event; diff --git a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs similarity index 52% rename from mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs rename to mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs index 2107f8932..e723e97fb 100644 --- a/mobile_verifier/tests/integrations/verified_subscriber_mapping_event.rs +++ b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs @@ -1,33 +1,48 @@ use chrono::{DateTime, Duration, Utc}; use file_store::{ - file_info_poller::FileInfoStream, - verified_subscriber_mapping_event::VerifiedSubscriberMappingEvent, - verified_subscriber_mapping_event_ingest_report::VerifiedSubscriberMappingEventIngestReport, + file_info_poller::FileInfoStream, file_sink::FileSinkClient, + subscriber_verified_mapping_event::SubscriberVerifiedMappingEvent, + subscriber_verified_mapping_event_ingest_report::SubscriberVerifiedMappingEventIngestReport, FileInfo, }; use helium_crypto::{KeyTag, Keypair, PublicKeyBinary}; -use mobile_verifier::verified_subscriber_mapping_event::{ - aggregate_verified_mapping_events, VerifiedSubscriberMappingEventDeamon, - VerifiedSubscriberMappingEventShare, VerifiedSubscriberMappingEventShares, +use helium_proto::services::poc_mobile::VerifiedSubscriberVerifiedMappingEventIngestReportV1; +use mobile_verifier::subscriber_verified_mapping_event::{ + aggregate_verified_mapping_events, SubscriberVerifiedMappingEventDeamon, + VerifiedSubscriberVerifiedMappingEventShare, VerifiedSubscriberVerifiedMappingEventShares, }; +use prost::Message; use rand::rngs::OsRng; use sqlx::{PgPool, Pool, Postgres, Row}; use std::{collections::HashMap, ops::Range}; +use tokio::time::timeout; + +use crate::common::{MockAuthorizationClient, MockEntityClient}; #[sqlx::test] async fn main_test(pool: PgPool) -> anyhow::Result<()> { - let (tx, rx) = tokio::sync::mpsc::channel(10); + let (reports_tx, reports_rx) = tokio::sync::mpsc::channel(10); + let (sink_tx, mut sink_rx) = tokio::sync::mpsc::channel(10); let (trigger, listener) = triggered::trigger(); let task_pool = pool.clone(); tokio::spawn(async move { - let deamon = VerifiedSubscriberMappingEventDeamon::new(task_pool, rx); + let deamon = SubscriberVerifiedMappingEventDeamon::new( + task_pool, + MockAuthorizationClient::new(), + MockEntityClient::new(), + reports_rx, + FileSinkClient::new(sink_tx, "metric"), + ); + deamon.run(listener).await.expect("failed to complete task"); }); + // Sending reports as if they are coming from ingestor let (fis, mut reports, public_key_binary) = file_info_stream(); - tx.send(fis).await?; + reports_tx.send(fis).await?; + // Testing that each report we sent made it into DB let mut retry = 0; const MAX_RETRIES: u32 = 10; const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); @@ -65,12 +80,53 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { MAX_RETRIES ); + // Testing that each report we sent made it on verified report FileSink + for expected_report in reports.clone() { + match timeout(std::time::Duration::from_secs(2), sink_rx.recv()).await { + Ok(Some(msg)) => match msg { + file_store::file_sink::Message::Commit(_) => panic!("got Commit"), + file_store::file_sink::Message::Rollback(_) => panic!("got Rollback"), + file_store::file_sink::Message::Data(_, data) => { + let proto_verified_report = VerifiedSubscriberVerifiedMappingEventIngestReportV1::decode(data.as_slice()) + .expect("unable to decode into VerifiedSubscriberVerifiedMappingEventIngestReportV1"); + + let rcv_report: SubscriberVerifiedMappingEventIngestReport = + proto_verified_report.report.unwrap().try_into()?; + + assert!(timestamp_match( + expected_report.received_timestamp, + rcv_report.received_timestamp + )); + assert_eq!( + expected_report.report.subscriber_id, + rcv_report.report.subscriber_id + ); + assert_eq!( + expected_report.report.total_reward_points, + rcv_report.report.total_reward_points + ); + assert_eq!( + expected_report.report.carrier_mapping_key, + rcv_report.report.carrier_mapping_key + ); + + assert!(timestamp_match( + expected_report.received_timestamp, + rcv_report.report.timestamp + )); + } + }, + Ok(None) => panic!("got none"), + Err(reason) => panic!("got error {reason}"), + } + } + // Testing aggregate_verified_mapping_events now let reward_period = Range { start: Utc::now() - Duration::days(1), end: Utc::now(), }; - let mut shares_from_reports = reports_to_shares(reports); + let mut shares_from_reports = reports_to_shares(reports.clone()); shares_from_reports.sort_by(|a, b| a.subscriber_id.cmp(&b.subscriber_id)); let mut shares = aggregate_verified_mapping_events(&pool, &reward_period).await?; @@ -83,9 +139,14 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { Ok(()) } +fn timestamp_match(dt1: DateTime, dt2: DateTime) -> bool { + let difference = dt1.signed_duration_since(dt2); + difference.num_seconds().abs() < 1 +} + fn file_info_stream() -> ( - FileInfoStream, - Vec, + FileInfoStream, + Vec, PublicKeyBinary, ) { let file_info = FileInfo { @@ -99,27 +160,27 @@ fn file_info_stream() -> ( let public_key_binary: PublicKeyBinary = key_pair.public_key().to_owned().into(); let reports = vec![ - VerifiedSubscriberMappingEventIngestReport { + SubscriberVerifiedMappingEventIngestReport { received_timestamp: Utc::now(), - report: VerifiedSubscriberMappingEvent { + report: SubscriberVerifiedMappingEvent { subscriber_id: vec![0], total_reward_points: 100, timestamp: Utc::now(), carrier_mapping_key: public_key_binary.clone(), }, }, - VerifiedSubscriberMappingEventIngestReport { + SubscriberVerifiedMappingEventIngestReport { received_timestamp: Utc::now() - Duration::seconds(10), - report: VerifiedSubscriberMappingEvent { + report: SubscriberVerifiedMappingEvent { subscriber_id: vec![1], total_reward_points: 101, timestamp: Utc::now() - Duration::seconds(10), carrier_mapping_key: public_key_binary.clone(), }, }, - VerifiedSubscriberMappingEventIngestReport { + SubscriberVerifiedMappingEventIngestReport { received_timestamp: Utc::now(), - report: VerifiedSubscriberMappingEvent { + report: SubscriberVerifiedMappingEvent { subscriber_id: vec![1], total_reward_points: 99, timestamp: Utc::now(), @@ -135,8 +196,8 @@ fn file_info_stream() -> ( } fn reports_to_shares( - reports: Vec, -) -> VerifiedSubscriberMappingEventShares { + reports: Vec, +) -> VerifiedSubscriberVerifiedMappingEventShares { let mut reward_map: HashMap, i64> = HashMap::new(); for report in reports { @@ -148,7 +209,7 @@ fn reports_to_shares( reward_map .into_iter() .map( - |(subscriber_id, total_reward_points)| VerifiedSubscriberMappingEventShare { + |(subscriber_id, total_reward_points)| VerifiedSubscriberVerifiedMappingEventShare { subscriber_id, total_reward_points, }, @@ -158,14 +219,14 @@ fn reports_to_shares( async fn select_events( pool: &Pool, -) -> anyhow::Result> { +) -> anyhow::Result> { let rows = sqlx::query( r#" SELECT subscriber_id, total_reward_points, - timestamp - FROM verified_mapping_event + received_timestamp + FROM verified_subscriber_verified_mapping_event "#, ) .fetch_all(pool) @@ -173,13 +234,13 @@ async fn select_events( let events = rows .into_iter() - .map(|row| VerifiedSubscriberMappingEvent { + .map(|row| SubscriberVerifiedMappingEvent { subscriber_id: row.get::, _>("subscriber_id"), total_reward_points: row.get::("total_reward_points") as u64, - timestamp: row.get::, _>("timestamp"), + timestamp: row.get::, _>("received_timestamp"), carrier_mapping_key: vec![].into(), }) - .collect::>(); + .collect::>(); Ok(events) } From 4234177785525307275e37d01abda85958f537f4 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 7 Aug 2024 13:29:30 -0700 Subject: [PATCH 34/44] Fix proto --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9001e2a37..3084edc6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", ] @@ -9862,7 +9862,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", "twox-hash", "xorf", From 83bee393289b75aa1314d812b23cbdc61ec01580 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 7 Aug 2024 14:28:44 -0700 Subject: [PATCH 35/44] Clear older events --- mobile_verifier/src/rewarder.rs | 1 + .../src/subscriber_verified_mapping_event.rs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index d17ee903a..7f22868a7 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -297,6 +297,7 @@ where data_session::clear_hotspot_data_sessions(&mut transaction, &reward_period.start).await?; coverage::clear_coverage_objects(&mut transaction, &reward_period.start).await?; sp_boosted_rewards_bans::clear_bans(&mut transaction, reward_period.start).await?; + subscriber_verified_mapping_event::clear(&mut transaction, &reward_period.end).await?; // subscriber_location::clear_location_shares(&mut transaction, &reward_period.end).await?; let next_reward_period = scheduler.next_reward_period(); diff --git a/mobile_verifier/src/subscriber_verified_mapping_event.rs b/mobile_verifier/src/subscriber_verified_mapping_event.rs index a5e470a9b..a5f0464b0 100644 --- a/mobile_verifier/src/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/src/subscriber_verified_mapping_event.rs @@ -265,7 +265,7 @@ pub async fn aggregate_verified_mapping_events( subscriber_id, SUM(total_reward_points) AS total_reward_points FROM - verified_subscriber_verified_mapping_event + verified_subscriber_verified_mapping_event WHERE received_timestamp >= $1 AND received_timestamp < $2 GROUP BY subscriber_id;", @@ -277,3 +277,16 @@ pub async fn aggregate_verified_mapping_events( Ok(vsme_shares) } + +pub async fn clear( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + timestamp: &DateTime, +) -> Result<(), sqlx::Error> { + sqlx::query( + "DELETE FROM verified_subscriber_verified_mapping_event WHERE received_timestamp < $1", + ) + .bind(timestamp) + .execute(&mut *tx) + .await?; + Ok(()) +} From 2dc6162eced8a5f851081e29ca3598f3e46245e0 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 9 Aug 2024 09:19:42 -0700 Subject: [PATCH 36/44] Update table name and use `reward_period.start` --- ... => 35_subscriber_verified_mapping_event.sql} | 2 +- .../src/subscriber_verified_mapping_event.rs | 16 +++++++--------- .../subscriber_verified_mapping_event.rs | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) rename mobile_verifier/migrations/{35_verified_subscriber_verified_mapping_event.sql => 35_subscriber_verified_mapping_event.sql} (74%) diff --git a/mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql b/mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql similarity index 74% rename from mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql rename to mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql index 2359dfb03..bde97f235 100644 --- a/mobile_verifier/migrations/35_verified_subscriber_verified_mapping_event.sql +++ b/mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS verified_subscriber_verified_mapping_event ( +CREATE TABLE IF NOT EXISTS subscriber_verified_mapping_event ( subscriber_id BYTEA NOT NULL, total_reward_points INTEGER NOT NULL, received_timestamp TIMESTAMPTZ, diff --git a/mobile_verifier/src/subscriber_verified_mapping_event.rs b/mobile_verifier/src/subscriber_verified_mapping_event.rs index a5f0464b0..48eb122a4 100644 --- a/mobile_verifier/src/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/src/subscriber_verified_mapping_event.rs @@ -232,7 +232,7 @@ async fn save_to_db( ) -> Result<(), sqlx::Error> { sqlx::query( r#" - INSERT INTO verified_subscriber_verified_mapping_event (subscriber_id, total_reward_points, received_timestamp) + INSERT INTO subscriber_verified_mapping_event (subscriber_id, total_reward_points, received_timestamp) VALUES ($1, $2, $3) ON CONFLICT (subscriber_id, received_timestamp) DO NOTHING "#, @@ -265,12 +265,12 @@ pub async fn aggregate_verified_mapping_events( subscriber_id, SUM(total_reward_points) AS total_reward_points FROM - verified_subscriber_verified_mapping_event + subscriber_verified_mapping_event WHERE received_timestamp >= $1 AND received_timestamp < $2 GROUP BY subscriber_id;", ) - .bind(reward_period.end - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) + .bind(reward_period.start - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) .bind(reward_period.end) .fetch_all(db) .await?; @@ -282,11 +282,9 @@ pub async fn clear( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, timestamp: &DateTime, ) -> Result<(), sqlx::Error> { - sqlx::query( - "DELETE FROM verified_subscriber_verified_mapping_event WHERE received_timestamp < $1", - ) - .bind(timestamp) - .execute(&mut *tx) - .await?; + sqlx::query("DELETE FROM subscriber_verified_mapping_event WHERE received_timestamp < $1") + .bind(timestamp) + .execute(&mut *tx) + .await?; Ok(()) } diff --git a/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs index e723e97fb..b0925f8c2 100644 --- a/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs @@ -226,7 +226,7 @@ async fn select_events( subscriber_id, total_reward_points, received_timestamp - FROM verified_subscriber_verified_mapping_event + FROM subscriber_verified_mapping_event "#, ) .fetch_all(pool) From 8f02efaebd2907712dda1e505f6ca8b1240caaea Mon Sep 17 00:00:00 2001 From: Andrew McKenzie Date: Fri, 2 Aug 2024 16:49:25 +0100 Subject: [PATCH 37/44] requirements originating from verification mapping verifier --- coverage_point_calculator/src/lib.rs | 2 +- coverage_point_calculator/src/speedtest.rs | 12 +++-- file_store/src/speedtest.rs | 53 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/coverage_point_calculator/src/lib.rs b/coverage_point_calculator/src/lib.rs index 8e891d737..1e6ddf475 100644 --- a/coverage_point_calculator/src/lib.rs +++ b/coverage_point_calculator/src/lib.rs @@ -75,7 +75,7 @@ use service_provider_boosting::{MAX_AVERAGE_DISTANCE, MIN_WIFI_TRUST_MULTIPLIER} mod hexes; mod location; mod service_provider_boosting; -mod speedtest; +pub mod speedtest; pub type Result = std::result::Result; diff --git a/coverage_point_calculator/src/speedtest.rs b/coverage_point_calculator/src/speedtest.rs index dea741595..7c2dccea5 100644 --- a/coverage_point_calculator/src/speedtest.rs +++ b/coverage_point_calculator/src/speedtest.rs @@ -2,8 +2,8 @@ use chrono::{DateTime, Utc}; use rust_decimal::Decimal; use rust_decimal_macros::dec; -const MIN_REQUIRED_SPEEDTEST_SAMPLES: usize = 2; -const MAX_ALLOWED_SPEEDTEST_SAMPLES: usize = 6; +pub const MIN_REQUIRED_SPEEDTEST_SAMPLES: usize = 2; +pub const MAX_ALLOWED_SPEEDTEST_SAMPLES: usize = 6; type Millis = u32; @@ -86,12 +86,14 @@ impl Speedtest { } pub fn multiplier(&self) -> Decimal { + self.tier().multiplier() + } + + pub fn tier(&self) -> SpeedtestTier { let upload = SpeedtestTier::from_upload(self.upload_speed); let download = SpeedtestTier::from_download(self.download_speed); let latency = SpeedtestTier::from_latency(self.latency_millis); - - let tier = upload.min(download).min(latency); - tier.multiplier() + upload.min(download).min(latency) } pub fn avg(speedtests: &[Self]) -> Self { diff --git a/file_store/src/speedtest.rs b/file_store/src/speedtest.rs index 2c8fce4a6..3142a0945 100644 --- a/file_store/src/speedtest.rs +++ b/file_store/src/speedtest.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, TimeZone, Utc}; use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::{ Speedtest, SpeedtestAvg, SpeedtestAvgValidity, SpeedtestIngestReportV1, SpeedtestReqV1, + SpeedtestVerificationResult, VerifiedSpeedtest as VerifiedSpeedtestProto, }; use serde::{Deserialize, Serialize}; @@ -34,6 +35,10 @@ impl MsgDecode for CellSpeedtestIngestReport { type Msg = SpeedtestIngestReportV1; } +impl MsgDecode for VerifiedSpeedtest { + type Msg = VerifiedSpeedtestProto; +} + impl MsgTimestamp>> for SpeedtestReqV1 { fn timestamp(&self) -> Result> { self.timestamp.to_timestamp() @@ -58,6 +63,18 @@ impl MsgTimestamp for CellSpeedtestIngestReport { } } +impl MsgTimestamp for VerifiedSpeedtest { + fn timestamp(&self) -> u64 { + self.timestamp.encode_timestamp_millis() + } +} + +impl MsgTimestamp>> for VerifiedSpeedtestProto { + fn timestamp(&self) -> Result> { + self.timestamp.to_timestamp_millis() + } +} + impl From for SpeedtestReqV1 { fn from(v: CellSpeedtest) -> Self { let timestamp = v.timestamp(); @@ -112,6 +129,42 @@ impl From for SpeedtestIngestReportV1 { } } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct VerifiedSpeedtest { + pub timestamp: DateTime, + pub report: CellSpeedtestIngestReport, + pub result: SpeedtestVerificationResult, +} + +impl TryFrom for VerifiedSpeedtest { + type Error = Error; + fn try_from(v: VerifiedSpeedtestProto) -> Result { + let result = SpeedtestVerificationResult::try_from(v.result).map_err(|_| { + DecodeError::unsupported_status_reason("verified_speedtest_proto", v.result) + })?; + Ok(Self { + timestamp: v.timestamp()?, + report: v + .report + .ok_or_else(|| Error::not_found("ingest verified speedtest report"))? + .try_into()?, + result, + }) + } +} + +impl From for VerifiedSpeedtestProto { + fn from(v: VerifiedSpeedtest) -> Self { + let timestamp = v.timestamp(); + let report: SpeedtestIngestReportV1 = v.report.into(); + Self { + timestamp, + report: Some(report), + result: v.result as i32, + } + } +} + pub mod cli { use super::*; From d91967b4a9a2ce3ded9be9c2b438ab87a6b8fa32 Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 14:15:34 -0400 Subject: [PATCH 38/44] revert proto/beacon to master branch --- Cargo.lock | 48 ++++++++++++++++++------------------------------ Cargo.toml | 5 ++--- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3084edc6f..2487342b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,11 +1617,11 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdef2c6774416b1b361a22894078fb1cc5e89203" +source = "git+https://github.com/helium/proto?branch=master#2916bcd01d740171ac8d49a29d1ea892419820f0" dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "prost", "rand 0.8.5", "rand_chacha 0.3.0", @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "notify", "serde", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hex-literal", "http 0.2.11", "lazy_static", @@ -3776,7 +3776,7 @@ dependencies = [ "h3o", "helium-anchor-gen", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", + "helium-proto", "hex", "itertools", "jsonrpc_client", @@ -3811,18 +3811,6 @@ dependencies = [ "tonic-build", ] -[[package]] -name = "helium-proto" -version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#62e22b2a4fb7de6f2ba7d67f933d77d4ad52a882" -dependencies = [ - "bytes", - "prost", - "prost-build", - "serde", - "serde_json", -] - [[package]] name = "helium-sub-daos" version = "0.1.5" @@ -3864,7 +3852,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4280,7 +4268,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "humantime-serde", "metrics", @@ -4349,7 +4337,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -4391,7 +4379,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -4433,7 +4421,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http-serde", "humantime-serde", "iot-config", @@ -5021,7 +5009,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -5061,7 +5049,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "mobile-config", "prost", "rand 0.8.5", @@ -5097,7 +5085,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -5141,7 +5129,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "hex-assignments", "hextree", "http-serde", @@ -5824,7 +5812,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5907,7 +5895,7 @@ dependencies = [ "futures-util", "helium-anchor-gen", "helium-lib", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6514,7 +6502,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/verification_mapping)", + "helium-proto", "humantime-serde", "lazy_static", "metrics", diff --git a/Cargo.toml b/Cargo.toml index 8f6db3618..a45355a99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,10 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "macpie/verification_mapping", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ "services", ] } -beacon = { git = "https://github.com/helium/proto", branch = "macpie/verification_mapping" } +beacon = { git = "https://github.com/helium/proto", branch = "master" } solana-client = "1.18" solana-sdk = "1.18" solana-program = "1.18" @@ -133,4 +133,3 @@ sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb # [patch.'https://github.com/helium/proto'] # helium-proto = { path = "../proto" } # beacon = { path = "../proto/beacon" } -# Temporary patch From 3c6b24a967dac3fbf76613ff072b04de076aa708 Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 14:19:50 -0400 Subject: [PATCH 39/44] fix proto master branch ref --- Cargo.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2487342b2..41ab8393a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", ] @@ -3801,6 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=master#2916bcd01d740171ac8d49a29d1ea892419820f0" dependencies = [ "bytes", "prost", @@ -9850,7 +9851,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", "twox-hash", "xorf", From 461caed0a191609ddbae63534c2045767abdf65e Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 15:15:57 -0400 Subject: [PATCH 40/44] review tweaks --- ingest/src/server_mobile.rs | 3 +-- .../migrations/35_subscriber_verified_mapping_event.sql | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index db2008875..57bc19c21 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -414,9 +414,8 @@ impl poc_mobile::PocMobile for GrpcServer { ) -> GrpcResult { let timestamp: u64 = Utc::now().timestamp_millis() as u64; let event: SubscriberVerifiedMappingEventReqV1 = request.into_inner(); - let subscriber_id = event.subscriber_id.clone(); - custom_tracing::record("subscriber_id", bs58::encode(&subscriber_id).into_string()); + custom_tracing::record_b58("subscriber_id", &event.subscriber_id); custom_tracing::record_b58("pub_key", &event.carrier_mapping_key); let report = self diff --git a/mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql b/mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql index bde97f235..59e718e7b 100644 --- a/mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql +++ b/mobile_verifier/migrations/35_subscriber_verified_mapping_event.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS subscriber_verified_mapping_event ( subscriber_id BYTEA NOT NULL, - total_reward_points INTEGER NOT NULL, - received_timestamp TIMESTAMPTZ, + total_reward_points BIGINT NOT NULL, + received_timestamp TIMESTAMPTZ NOT NULL, inserted_at TIMESTAMPTZ DEFAULT now(), PRIMARY KEY (subscriber_id, received_timestamp) -); \ No newline at end of file +); From 867dbfbd5b5f20f25d52e9f21c8fef9ce3f195e2 Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 15:34:27 -0400 Subject: [PATCH 41/44] typo nit --- mobile_verifier/src/cli/server.rs | 4 ++-- mobile_verifier/src/subscriber_verified_mapping_event.rs | 6 +++--- .../tests/integrations/subscriber_verified_mapping_event.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index dbf5fd7d2..7e1768d21 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -11,7 +11,7 @@ use crate::{ sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, - subscriber_verified_mapping_event::SubscriberVerifiedMappingEventDeamon, + subscriber_verified_mapping_event::SubscriberVerifiedMappingEventDaemon, telemetry, Settings, }; use anyhow::Result; @@ -143,7 +143,7 @@ impl Cmd { .await?, ) .add_task( - SubscriberVerifiedMappingEventDeamon::create_managed_task( + SubscriberVerifiedMappingEventDaemon::create_managed_task( pool.clone(), settings, auth_client.clone(), diff --git a/mobile_verifier/src/subscriber_verified_mapping_event.rs b/mobile_verifier/src/subscriber_verified_mapping_event.rs index 48eb122a4..e4c38178e 100644 --- a/mobile_verifier/src/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/src/subscriber_verified_mapping_event.rs @@ -27,7 +27,7 @@ use std::ops::Range; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; -pub struct SubscriberVerifiedMappingEventDeamon { +pub struct SubscriberVerifiedMappingEventDaemon { pool: Pool, authorization_verifier: AV, entity_verifier: EV, @@ -35,7 +35,7 @@ pub struct SubscriberVerifiedMappingEventDeamon { verified_report_sink: FileSinkClient, } -impl SubscriberVerifiedMappingEventDeamon +impl SubscriberVerifiedMappingEventDaemon where AV: AuthorizationVerifier + Send + Sync + 'static, EV: EntityVerifier + Send + Sync + 'static, @@ -213,7 +213,7 @@ where } } -impl ManagedTask for SubscriberVerifiedMappingEventDeamon +impl ManagedTask for SubscriberVerifiedMappingEventDaemon where AV: AuthorizationVerifier + Send + Sync + 'static, EV: EntityVerifier + Send + Sync + 'static, diff --git a/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs index b0925f8c2..70e69a7f1 100644 --- a/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs @@ -8,7 +8,7 @@ use file_store::{ use helium_crypto::{KeyTag, Keypair, PublicKeyBinary}; use helium_proto::services::poc_mobile::VerifiedSubscriberVerifiedMappingEventIngestReportV1; use mobile_verifier::subscriber_verified_mapping_event::{ - aggregate_verified_mapping_events, SubscriberVerifiedMappingEventDeamon, + aggregate_verified_mapping_events, SubscriberVerifiedMappingEventDaemon, VerifiedSubscriberVerifiedMappingEventShare, VerifiedSubscriberVerifiedMappingEventShares, }; use prost::Message; @@ -27,7 +27,7 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { let task_pool = pool.clone(); tokio::spawn(async move { - let deamon = SubscriberVerifiedMappingEventDeamon::new( + let deamon = SubscriberVerifiedMappingEventDaemon::new( task_pool, MockAuthorizationClient::new(), MockEntityClient::new(), From b2540a91b6eeaf48ff2caf1a821228a6e8f5a421 Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 15:43:30 -0400 Subject: [PATCH 42/44] fix reward aggregation to single period --- mobile_verifier/src/subscriber_verified_mapping_event.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/src/subscriber_verified_mapping_event.rs b/mobile_verifier/src/subscriber_verified_mapping_event.rs index e4c38178e..cdb122733 100644 --- a/mobile_verifier/src/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/src/subscriber_verified_mapping_event.rs @@ -1,5 +1,5 @@ use crate::Settings; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, file_sink::{self, FileSinkClient}, @@ -246,7 +246,6 @@ async fn save_to_db( Ok(()) } -const SUBSCRIBER_REWARD_PERIOD_IN_DAYS: i64 = 1; pub type VerifiedSubscriberVerifiedMappingEventShares = Vec; @@ -270,7 +269,7 @@ pub async fn aggregate_verified_mapping_events( GROUP BY subscriber_id;", ) - .bind(reward_period.start - Duration::days(SUBSCRIBER_REWARD_PERIOD_IN_DAYS)) + .bind(reward_period.start) .bind(reward_period.end) .fetch_all(db) .await?; From 59837049b9a338d864000396f9d4b1287e33378a Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 16:01:40 -0400 Subject: [PATCH 43/44] fix test --- .../tests/integrations/subscriber_verified_mapping_event.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs index 70e69a7f1..14674d911 100644 --- a/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/tests/integrations/subscriber_verified_mapping_event.rs @@ -236,7 +236,7 @@ async fn select_events( .into_iter() .map(|row| SubscriberVerifiedMappingEvent { subscriber_id: row.get::, _>("subscriber_id"), - total_reward_points: row.get::("total_reward_points") as u64, + total_reward_points: row.get::("total_reward_points") as u64, timestamp: row.get::, _>("received_timestamp"), carrier_mapping_key: vec![].into(), }) From 14258aed3480ce16a542ea1556b83273ba85752b Mon Sep 17 00:00:00 2001 From: jeffgrunewald Date: Tue, 13 Aug 2024 16:41:19 -0400 Subject: [PATCH 44/44] fix sum type for output decoding --- mobile_verifier/src/subscriber_verified_mapping_event.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/subscriber_verified_mapping_event.rs b/mobile_verifier/src/subscriber_verified_mapping_event.rs index cdb122733..5f67ff28b 100644 --- a/mobile_verifier/src/subscriber_verified_mapping_event.rs +++ b/mobile_verifier/src/subscriber_verified_mapping_event.rs @@ -262,7 +262,7 @@ pub async fn aggregate_verified_mapping_events( let vsme_shares = sqlx::query_as::<_, VerifiedSubscriberVerifiedMappingEventShare>( "SELECT subscriber_id, - SUM(total_reward_points) AS total_reward_points + SUM(total_reward_points)::BIGINT AS total_reward_points FROM subscriber_verified_mapping_event WHERE received_timestamp >= $1 AND received_timestamp < $2