From f46c818b4087202a682dd7eb241bc824d277ea97 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 16 May 2024 11:23:46 -0400 Subject: [PATCH] Fix up tests, address comments --- .../src/boosting_oracles/data_sets.rs | 92 +++++++++---------- .../src/boosting_oracles/footfall.rs | 19 ++-- .../src/boosting_oracles/landtype.rs | 19 ++-- mobile_verifier/src/boosting_oracles/mod.rs | 31 +------ .../src/boosting_oracles/urbanization.rs | 19 ++-- .../tests/integrations/boosting_oracles.rs | 12 +-- .../tests/integrations/common/mod.rs | 55 ++++++++++- .../tests/integrations/hex_boosting.rs | 7 +- .../tests/integrations/modeled_coverage.rs | 10 +- .../tests/integrations/rewarder_poc_dc.rs | 7 +- 10 files changed, 132 insertions(+), 139 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index c3074b9c5..7ef92a371 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -13,7 +13,6 @@ use file_store::{ }; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; -use hextree::disktree::DiskTreeMap; use lazy_static::lazy_static; use regex::Regex; use rust_decimal::prelude::ToPrimitive; @@ -24,7 +23,9 @@ use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::Receiver}; use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel, Settings}; -use super::{footfall::Footfall, landtype::Landtype, HexAssignment, HexBoostData, Urbanization}; +use super::{ + footfall::Footfall, landtype::Landtype, urbanization::Urbanization, HexAssignment, HexBoostData, +}; #[async_trait::async_trait] pub trait DataSet: HexAssignment + Send + Sync + 'static { @@ -169,9 +170,7 @@ where } } -impl - DataSetDownloaderDaemon, Landtype, Urbanization> -{ +impl DataSetDownloaderDaemon { pub async fn create_managed_task( pool: PgPool, settings: &Settings, @@ -190,9 +189,9 @@ impl .create() .await?; - let urbanization: Urbanization = Urbanization::new(); - let footfall: Footfall = Footfall::new(); - let landtype: Landtype = Landtype::new(); + let urbanization = Urbanization::new(); + let footfall = Footfall::new(); + let landtype = Landtype::new(); let hex_boost_data = HexBoostData::builder() .footfall(footfall) .landtype(landtype) @@ -548,14 +547,34 @@ pub mod db { .fetch_one(pool) .await?) } + + pub fn fetch_all_hexes(pool: &PgPool) -> impl Stream> + '_ { + sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool) + } + + pub fn fetch_hexes_with_null_assignments( + pool: &PgPool, + ) -> impl Stream> + '_ { + sqlx::query_as( + "SELECT + uuid, hex, signal_level, signal_power + FROM + hexes + WHERE + urbanized IS NULL + OR footfall IS NULL + OR landtype IS NULL", + ) + .fetch(pool) + } } pub struct AssignedCoverageObjects { - coverage_objs: HashMap>, + pub coverage_objs: HashMap>, } impl AssignedCoverageObjects { - async fn from_stream( + pub async fn assign_hex_stream( stream: impl Stream>, data_sets: &HexBoostData, ) -> anyhow::Result { @@ -568,38 +587,6 @@ impl AssignedCoverageObjects { Ok(Self { coverage_objs }) } - async fn fetch_all( - pool: &PgPool, - data_sets: &HexBoostData, - ) -> anyhow::Result { - Self::from_stream( - sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool), - data_sets, - ) - .await - } - - async fn fetch_unassigned( - pool: &PgPool, - data_sets: &HexBoostData, - ) -> anyhow::Result { - Self::from_stream( - sqlx::query_as( - "SELECT - uuid, hex, signal_level, signal_power - FROM - hexes - WHERE - urbanized IS NULL - OR footfall IS NULL - OR landtype IS NULL", - ) - .fetch(pool), - data_sets, - ) - .await - } - async fn write(&self, boosting_reports: &FileSinkClient) -> file_store::Result { let timestamp = Utc::now().encode_timestamp(); for (uuid, hexes) in self.coverage_objs.iter() { @@ -635,7 +622,7 @@ impl AssignedCoverageObjects { Ok(()) } - async fn save(self, pool: &PgPool) -> anyhow::Result<()> { + pub async fn save(self, pool: &PgPool) -> anyhow::Result<()> { const NUMBER_OF_FIELDS_IN_QUERY: u16 = 7; const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; @@ -702,11 +689,11 @@ impl UnassignedHex { } pub struct AssignedHex { - uuid: uuid::Uuid, - hex: u64, - signal_level: SignalLevel, - signal_power: i32, - assignments: HexAssignments, + pub uuid: uuid::Uuid, + pub hex: u64, + pub signal_level: SignalLevel, + pub signal_power: i32, + pub assignments: HexAssignments, } pub async fn set_all_oracle_boosting_assignments( @@ -714,7 +701,8 @@ pub async fn set_all_oracle_boosting_assignments( data_sets: &HexBoostData, file_sink: &FileSinkClient, ) -> anyhow::Result<()> { - let assigned_coverage_objs = AssignedCoverageObjects::fetch_all(pool, data_sets).await?; + let assigned_coverage_objs = + AssignedCoverageObjects::assign_hex_stream(db::fetch_all_hexes(pool), data_sets).await?; assigned_coverage_objs.write(file_sink).await?; assigned_coverage_objs.save(pool).await?; Ok(()) @@ -725,7 +713,11 @@ pub async fn set_unassigned_oracle_boosting_assignments( data_sets: &HexBoostData, file_sink: &FileSinkClient, ) -> anyhow::Result<()> { - let assigned_coverage_objs = AssignedCoverageObjects::fetch_unassigned(pool, data_sets).await?; + let assigned_coverage_objs = AssignedCoverageObjects::assign_hex_stream( + db::fetch_hexes_with_null_assignments(pool), + data_sets, + ) + .await?; assigned_coverage_objs.write(file_sink).await?; assigned_coverage_objs.save(pool).await?; Ok(()) diff --git a/mobile_verifier/src/boosting_oracles/footfall.rs b/mobile_verifier/src/boosting_oracles/footfall.rs index 1b134bca7..6f9db1e51 100644 --- a/mobile_verifier/src/boosting_oracles/footfall.rs +++ b/mobile_verifier/src/boosting_oracles/footfall.rs @@ -3,14 +3,14 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; +use super::{Assignment, DataSet, DataSetType, HexAssignment}; -pub struct Footfall { - footfall: Option, +pub struct Footfall { + footfall: Option, timestamp: Option>, } -impl Footfall { +impl Footfall { pub fn new() -> Self { Self { footfall: None, @@ -18,7 +18,7 @@ impl Footfall { } } - pub fn new_mock(footfall: F) -> Self { + pub fn new_mock(footfall: DiskTreeMap) -> Self { Self { footfall: Some(footfall), timestamp: None, @@ -26,14 +26,14 @@ impl Footfall { } } -impl Default for Footfall { +impl Default for Footfall { fn default() -> Self { Self::new() } } #[async_trait::async_trait] -impl DataSet for Footfall { +impl DataSet for Footfall { const TYPE: DataSetType = DataSetType::Footfall; fn timestamp(&self) -> Option> { @@ -51,10 +51,7 @@ impl DataSet for Footfall { } } -impl HexAssignment for Footfall -where - Foot: DiskTreeLike + Send + Sync + 'static, -{ +impl HexAssignment for Footfall { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { let Some(ref footfall) = self.footfall else { anyhow::bail!("No footfall data set has been loaded"); diff --git a/mobile_verifier/src/boosting_oracles/landtype.rs b/mobile_verifier/src/boosting_oracles/landtype.rs index 6493bde3c..b6cda7aef 100644 --- a/mobile_verifier/src/boosting_oracles/landtype.rs +++ b/mobile_verifier/src/boosting_oracles/landtype.rs @@ -3,14 +3,14 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; +use super::{Assignment, DataSet, DataSetType, HexAssignment}; -pub struct Landtype { - landtype: Option, +pub struct Landtype { + landtype: Option, timestamp: Option>, } -impl Landtype { +impl Landtype { pub fn new() -> Self { Self { landtype: None, @@ -18,7 +18,7 @@ impl Landtype { } } - pub fn new_mock(landtype: L) -> Self { + pub fn new_mock(landtype: DiskTreeMap) -> Self { Self { landtype: Some(landtype), timestamp: None, @@ -26,14 +26,14 @@ impl Landtype { } } -impl Default for Landtype { +impl Default for Landtype { fn default() -> Self { Self::new() } } #[async_trait::async_trait] -impl DataSet for Landtype { +impl DataSet for Landtype { const TYPE: DataSetType = DataSetType::Landtype; fn timestamp(&self) -> Option> { @@ -131,10 +131,7 @@ impl From for Assignment { } } -impl HexAssignment for Landtype -where - Land: DiskTreeLike + Send + Sync + 'static, -{ +impl HexAssignment for Landtype { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { let Some(ref landtype) = self.landtype else { anyhow::bail!("No landtype data set has been loaded"); diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 773c86cf7..0f63845cb 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -10,9 +10,6 @@ use crate::boosting_oracles::assignment::HexAssignments; pub use assignment::Assignment; pub use data_sets::*; -use hextree::disktree::DiskTreeMap; -pub use urbanization::Urbanization; - pub trait HexAssignment: Send + Sync + 'static { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result; } @@ -68,38 +65,14 @@ where } } -trait DiskTreeLike: Send + Sync { - fn get(&self, cell: hextree::Cell) -> hextree::Result>; -} - -impl DiskTreeLike for DiskTreeMap { - fn get(&self, cell: hextree::Cell) -> hextree::Result> { - self.get(cell) - } -} - -impl DiskTreeLike for std::collections::HashSet { - fn get(&self, cell: hextree::Cell) -> hextree::Result> { - Ok(self.contains(&cell).then_some((cell, &[]))) - } -} - -pub struct MockDiskTree; - -impl DiskTreeLike for MockDiskTree { - fn get(&self, cell: hextree::Cell) -> hextree::Result> { - Ok(Some((cell, &[]))) - } -} - #[cfg(test)] mod tests { use std::io::Cursor; - use hextree::HexTreeMap; + use hextree::{disktree::DiskTreeMap, HexTreeMap}; - use self::{footfall::Footfall, landtype::Landtype}; + use self::{footfall::Footfall, landtype::Landtype, urbanization::Urbanization}; use super::*; diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs index 506598943..8070c445c 100644 --- a/mobile_verifier/src/boosting_oracles/urbanization.rs +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -3,14 +3,14 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; +use super::{Assignment, DataSet, DataSetType, HexAssignment}; -pub struct Urbanization
{ - urbanized: Option
, +pub struct Urbanization { + urbanized: Option, timestamp: Option>, } -impl
Urbanization
{ +impl Urbanization { pub fn new() -> Self { Self { urbanized: None, @@ -18,7 +18,7 @@ impl
Urbanization
{ } } - pub fn new_mock(urbanized: DT) -> Self { + pub fn new_mock(urbanized: DiskTreeMap) -> Self { Self { urbanized: Some(urbanized), timestamp: None, @@ -26,14 +26,14 @@ impl
Urbanization
{ } } -impl
Default for Urbanization
{ +impl Default for Urbanization { fn default() -> Self { Self::new() } } #[async_trait::async_trait] -impl DataSet for Urbanization { +impl DataSet for Urbanization { const TYPE: DataSetType = DataSetType::Urbanization; fn timestamp(&self) -> Option> { @@ -51,10 +51,7 @@ impl DataSet for Urbanization { } } -impl HexAssignment for Urbanization -where - Urban: DiskTreeLike + Send + Sync + 'static, -{ +impl HexAssignment for Urbanization { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { let Some(ref urbanized) = self.urbanized else { anyhow::bail!("No urbanization data set has been loaded"); diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index bc0126fa6..880407c37 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -13,7 +13,7 @@ use helium_proto::services::poc_mobile::{ }; use mobile_config::boosted_hex_info::BoostedHexes; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, Assignment, HexBoostData, UnassignedHex}, + boosting_oracles::{Assignment, HexBoostData}, coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{Heartbeat, HeartbeatReward, LocationCache, SeniorityUpdate, ValidatedHeartbeat}, @@ -29,6 +29,8 @@ use sqlx::PgPool; use std::{collections::HashMap, pin::pin}; use uuid::Uuid; +use crate::common; + #[derive(Clone)] struct MockGeofence; @@ -209,15 +211,12 @@ async fn test_footfall_and_urbanization_report(pool: PgPool) -> anyhow::Result<( .await?; transaction.commit().await?; - let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); let hex_boost_data = HexBoostData::builder() .footfall(footfall) .landtype(landtype) .urbanization(urbanized) .build()?; - let oba = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool) - .await? - .collect::>(); + let oba = common::set_unassigned_oracle_boosting_assignments(&pool, &hex_boost_data).await?; assert_eq!(oba.len(), 1); assert_eq!(oba[0].assignments, hexes); @@ -340,13 +339,12 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re .await?; transaction.commit().await?; - let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); let hex_boost_data = HexBoostData::builder() .footfall(footfall) .landtype(landtype) .urbanization(urbanized) .build()?; - let _ = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool).await?; + let _ = common::set_unassigned_oracle_boosting_assignments(&pool, &hex_boost_data).await?; let heartbeats = heartbeats(12, start, &owner, &cbsd_id, 0.0, 0.0, uuid); diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 1ec3a0a94..5722f6a7a 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -1,10 +1,14 @@ use chrono::{DateTime, Utc}; -use file_store::file_sink::{FileSinkClient, Message as SinkMessage}; +use file_store::{ + file_sink::{FileSinkClient, Message as SinkMessage}, + traits::TimestampEncode, +}; use futures::{stream, StreamExt}; use helium_proto::{ services::poc_mobile::{ - mobile_reward_share::Reward as MobileReward, GatewayReward, MobileRewardShare, RadioReward, - ServiceProviderReward, SpeedtestAvg, SubscriberReward, UnallocatedReward, + mobile_reward_share::Reward as MobileReward, GatewayReward, MobileRewardShare, + OracleBoostingHexAssignment, OracleBoostingReportV1, RadioReward, ServiceProviderReward, + SpeedtestAvg, SubscriberReward, UnallocatedReward, }, Message, }; @@ -12,7 +16,12 @@ use mobile_config::{ boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; -use mobile_verifier::boosting_oracles::{Assignment, HexBoostData}; +use mobile_verifier::boosting_oracles::{ + AssignedCoverageObjects, Assignment, HexAssignment, HexBoostData, +}; +use rust_decimal::prelude::ToPrimitive; +use rust_decimal_macros::dec; +use sqlx::PgPool; use std::collections::HashMap; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; @@ -204,3 +213,41 @@ pub fn mock_hex_boost_data( .build() .unwrap() } + +pub async fn set_unassigned_oracle_boosting_assignments( + pool: &PgPool, + data_sets: &HexBoostData, +) -> anyhow::Result> { + let assigned_coverage_objs = AssignedCoverageObjects::assign_hex_stream( + mobile_verifier::boosting_oracles::data_sets::db::fetch_hexes_with_null_assignments(pool), + data_sets, + ) + .await?; + let timestamp = Utc::now().encode_timestamp(); + let mut output = Vec::new(); + for (uuid, hexes) in assigned_coverage_objs.coverage_objs.iter() { + let assignments: Vec<_> = hexes + .iter() + .map(|hex| { + let location = format!("{:x}", hex.hex); + let assignment_multiplier = (hex.assignments.boosting_multiplier() * dec!(1000)) + .to_u32() + .unwrap_or(0); + OracleBoostingHexAssignment { + location, + urbanized: hex.assignments.urbanized.into(), + footfall: hex.assignments.footfall.into(), + landtype: hex.assignments.landtype.into(), + assignment_multiplier, + } + }) + .collect(); + output.push(OracleBoostingReportV1 { + coverage_object: Vec::from(uuid.into_bytes()), + assignments, + timestamp, + }); + } + assigned_coverage_objs.save(pool).await?; + Ok(output) +} diff --git a/mobile_verifier/tests/integrations/hex_boosting.rs b/mobile_verifier/tests/integrations/hex_boosting.rs index 8dcbb823d..b438cd19c 100644 --- a/mobile_verifier/tests/integrations/hex_boosting.rs +++ b/mobile_verifier/tests/integrations/hex_boosting.rs @@ -13,7 +13,6 @@ use helium_proto::services::poc_mobile::{ use hextree::Cell; use mobile_config::boosted_hex_info::BoostedHexInfo; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, cell_type::CellType, coverage::CoverageObject, heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, @@ -35,11 +34,9 @@ const BOOST_HEX_PUBKEY: &str = "J9JiLTpjaShxL8eMvUs8txVw6TZ36E38SiJ89NxnMbLU"; const BOOST_CONFIG_PUBKEY: &str = "BZM1QTud72B2cpTW7PhEnFmRX7ZWzvY7DpPpNJJuDrWG"; async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { - let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments( - unassigned_hexes, - &common::mock_hex_boost_data_default(), + let _ = common::set_unassigned_oracle_boosting_assignments( pool, + &common::mock_hex_boost_data_default(), ) .await?; Ok(()) diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 392801766..fee26b159 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -1,4 +1,3 @@ -use crate::common; use chrono::{DateTime, Duration, Utc}; use file_store::{ coverage::{CoverageObjectIngestReport, RadioHexSignalLevel}, @@ -16,7 +15,6 @@ use hextree::Cell; use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{ @@ -34,6 +32,8 @@ use sqlx::PgPool; use std::{collections::HashMap, num::NonZeroU32, ops::Range, pin::pin, str::FromStr}; use uuid::Uuid; +use crate::common; + #[derive(Clone)] struct MockGeofence; @@ -405,11 +405,9 @@ async fn process_input( } transaction.commit().await?; - let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments( - unassigned_hexes, - &common::mock_hex_boost_data_default(), + let _ = common::set_unassigned_oracle_boosting_assignments( pool, + &common::mock_hex_boost_data_default(), ) .await?; diff --git a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs index 2812a06be..f9287ed5e 100644 --- a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs @@ -10,7 +10,6 @@ use helium_proto::services::poc_mobile::{ SignalLevel, UnallocatedReward, UnallocatedRewardType, }; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, cell_type::CellType, coverage::CoverageObject, data_session, @@ -266,11 +265,9 @@ async fn seed_heartbeats( } async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { - let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments( - unassigned_hexes, - &common::mock_hex_boost_data_default(), + let _ = common::set_unassigned_oracle_boosting_assignments( pool, + &common::mock_hex_boost_data_default(), ) .await?; Ok(())