Mj/footfall boosting (#761)

* add footfall data to boosting flow

footfall and urbanization are considered together for determining the
multiplier applied to a hex.

* require passing a valid hex cell when asking about a cell

* Fix hex_assignment

* Fix tests

* Fix formatting

* Fix footfall Assignment to A for coverage tests

* Provide mocked hex assignments struct

Test that don't care about the specific values or underlying structures
of HexAssignments can use this mock. The only option available is to
return the best possible multiplier for a hex, hopefully behaving in a
way that removes hex boosting from the equation.

* restructure HexBoosting

The top level HexBoosting struct is the wrapper for asking internal
DiskTree about Assignments. For debugability, the information inside a
DiskTree is turned into an enum mirroring the verbage in HIP-103, then
get's turned into an Assignment.

* use helper to make hex boost data with settings path

* keep naming consistent between structs

* no longer only urbanization hexes, all unassigned hexes

* Update mobile_verifier/src/boosting_oracles/mod.rs

Co-authored-by: Matthew Plant <matty@nova-labs.com>

* Implement HexAssignment for both of the data sets independently

* add file-store dump for coverage object

* add error context for keypair deserializing

* add dump mobile rewards command

groups mobile rewards from a file into a valid json object and prints

* Pull calculations out of code into structs

Co-authored-by: macpie <macpie@users.noreply.github.com>

* more nonzero use

* format

* update helium-proto

* Change proto back to master

---------

Co-authored-by: Michael Jeffrey <michaeldjeffrey@gmail.com>
Co-authored-by: Matthew Plant <matty@nova-labs.com>
Co-authored-by: Brian Balser <bbalser@nova-labs.com>

file_store/src/cli/dump.rs

@@ -1,5 +1,6 @@
 use crate::{
     cli::print_json,
+    coverage::CoverageObject,
     file_source,
     heartbeat::{CbrsHeartbeat, CbrsHeartbeatIngestReport},
     iot_packet::IotValidPacket,
@@ -23,7 +24,7 @@ use helium_proto::{
         },
         poc_mobile::{
             mobile_reward_share::Reward, CellHeartbeatIngestReportV1, CellHeartbeatReqV1,
-            Heartbeat, InvalidDataTransferIngestReportV1, MobileRewardShare,
+            CoverageObjectV1, Heartbeat, InvalidDataTransferIngestReportV1, MobileRewardShare,
             OracleBoostingReportV1, RadioRewardShare, SpeedtestAvg, SpeedtestIngestReportV1,
             SpeedtestReqV1,
         },
@@ -355,7 +356,17 @@ impl Cmd {
                     "timestamp": report.timestamp.to_timestamp()?,
                     }))?
                 }
-                _ => (),
+                FileType::CoverageObject => {
+                    let coverage = CoverageObjectV1::decode(msg)?;
+                    let coverage = CoverageObject::try_from(coverage.coverage_object.unwrap())?;
+                    print_json(&json!({
+                        "pub_key": coverage.pub_key,
+                        "uuid": coverage.uuid,
+                        "coverage_claim_time": coverage.coverage_claim_time,
+                        "coverage": coverage.coverage,
+                    }))?;
+                }
+                missing_filetype => println!("No dump for {missing_filetype}"),
             }
         }
 

file_store/src/cli/dump_mobile_rewards.rs

@@ -0,0 +1,68 @@
+use crate::cli::print_json;
+use crate::{file_source, Result, Settings};
+use futures::stream::StreamExt;
+use helium_crypto::PublicKey;
+use helium_proto::services::poc_mobile::mobile_reward_share::Reward::*;
+use helium_proto::services::poc_mobile::MobileRewardShare;
+use prost::Message;
+use serde_json::json;
+use std::path::PathBuf;
+
+#[derive(Debug, clap::Args)]
+pub struct Cmd {
+    path: PathBuf,
+}
+
+impl Cmd {
+    pub async fn run(&self, _settings: &Settings) -> Result {
+        let mut file_stream = file_source::source([&self.path]);
+
+        let mut radio_reward = vec![];
+        let mut gateway_reward = vec![];
+        let mut subscriber_reward = vec![];
+        let mut service_provider_reward = vec![];
+        let mut unallocated_reward = vec![];
+
+        while let Some(result) = file_stream.next().await {
+            let msg = result?;
+            let reward = MobileRewardShare::decode(msg)?;
+            match reward.reward {
+                Some(r) => match r {
+                    RadioReward(reward) => radio_reward.push(json!({
+                        "hotspot_key":  PublicKey::try_from(reward.hotspot_key)?.to_string(),
+                        "cbsd_id": reward.cbsd_id,
+                        "poc_reward": reward.poc_reward,
+                        "boosted_hexes": reward.boosted_hexes,
+                    })),
+                    GatewayReward(reward) => gateway_reward.push(json!({
+                        "hotspot_key": PublicKey::try_from(reward.hotspot_key)?.to_string(),
+                        "dc_transfer_reward": reward.dc_transfer_reward,
+                    })),
+                    SubscriberReward(reward) => subscriber_reward.push(json!({
+                        "subscriber_id": uuid::Uuid::from_slice(&reward.subscriber_id).unwrap(),
+                        "discovery_location_amount": reward.discovery_location_amount,
+                    })),
+                    ServiceProviderReward(reward) => service_provider_reward.push(json!({
+                        "service_provider": reward.service_provider_id,
+                        "amount": reward.amount,
+                    })),
+                    UnallocatedReward(reward) => unallocated_reward.push(json!({
+                        "unallocated_reward_type": reward.reward_type,
+                        "amount": reward.amount,
+                    })),
+                },
+                None => todo!(),
+            }
+        }
+
+        print_json(&json!({
+            "radio_reward": radio_reward,
+            "gateway_reward": gateway_reward,
+            "subscriber_reward": subscriber_reward,
+            "service_provider_reward": service_provider_reward,
+            "unallocated_reward": unallocated_reward,
+        }))?;
+
+        Ok(())
+    }
+}

file_store/src/cli/mod.rs

@@ -1,5 +1,6 @@
 pub mod bucket;
 pub mod dump;
+pub mod dump_mobile_rewards;
 pub mod info;
 
 use crate::Result;

file_store/src/main.rs

@@ -1,6 +1,6 @@
 use clap::Parser;
 use file_store::{
-    cli::{bucket, dump, info},
+    cli::{bucket, dump, dump_mobile_rewards, info},
     Result, Settings,
 };
 use std::path;
@@ -29,6 +29,7 @@ pub enum Cmd {
     Info(info::Cmd),
     Dump(dump::Cmd),
     Bucket(Box<bucket::Cmd>),
+    DumpMobileRewards(dump_mobile_rewards::Cmd),
 }
 
 impl Cmd {
@@ -37,6 +38,7 @@ impl Cmd {
             Cmd::Info(cmd) => cmd.run(&settings).await,
             Cmd::Dump(cmd) => cmd.run(&settings).await,
             Cmd::Bucket(cmd) => cmd.run(&settings).await,
+            Cmd::DumpMobileRewards(cmd) => cmd.run(&settings).await,
         }
     }
 }

mobile_config/src/settings.rs

@@ -1,3 +1,4 @@
+use anyhow::Context;
 use config::{Config, Environment, File};
 use serde::Deserialize;
 use std::{
@@ -65,7 +66,9 @@ impl Settings {
     }
 
     pub fn signing_keypair(&self) -> anyhow::Result<helium_crypto::Keypair> {
-        let data = std::fs::read(&self.signing_keypair).map_err(helium_crypto::Error::from)?;
+        let data = std::fs::read(&self.signing_keypair)
+            .map_err(helium_crypto::Error::from)
+            .with_context(|| format!("reading keypair from settings: {}", self.signing_keypair))?;
         Ok(helium_crypto::Keypair::try_from(&data[..])?)
     }
 

mobile_verifier/migrations/29_footfall.sql

@@ -0,0 +1 @@
+ALTER TABLE hexes ADD COLUMN footfall oracle_assignment;

mobile_verifier/src/boosting_oracles/assignment.rs

@@ -40,18 +40,7 @@ impl fmt::Display for Assignment {
     }
 }
 
-pub fn urbanization_multiplier(urbanization: Assignment) -> Decimal {
-    use Assignment::*;
-
-    match urbanization {
-        A => dec!(1.0),
-        B => dec!(0.25),
-        C => dec!(0.0),
-    }
-}
-
-#[allow(dead_code)]
-pub fn urbanization_and_footfall_multiplier(
+pub fn footfall_and_urbanization_multiplier(
     footfall: Assignment,
     urbanization: Assignment,
 ) -> Decimal {

mobile_verifier/src/boosting_oracles/mod.rs

@@ -2,11 +2,70 @@ pub mod assignment;
 
 use std::collections::HashMap;
 
-use crate::geofence::GeofenceValidator;
+use crate::{
+    geofence::{Geofence, GeofenceValidator},
+    Settings,
+};
 pub use assignment::Assignment;
 use hextree::disktree::DiskTreeMap;
 
-pub trait DiskTreeLike: Send + Sync + 'static {
+pub fn make_hex_boost_data(
+    settings: &Settings,
+    usa_geofence: Geofence,
+) -> anyhow::Result<HexBoostData<impl HexAssignment, impl HexAssignment>> {
+    let urban_disktree = DiskTreeMap::open(&settings.urbanization_data_set)?;
+    let footfall_disktree = DiskTreeMap::open(&settings.footfall_data_set)?;
+
+    let urbanization = UrbanizationData::new(urban_disktree, usa_geofence);
+    let footfall_data = FootfallData::new(footfall_disktree);
+    let hex_boost_data = HexBoostData::new(urbanization, footfall_data);
+
+    Ok(hex_boost_data)
+}
+
+pub trait HexAssignment: Send + Sync {
+    fn assignment(&self, cell: hextree::Cell) -> anyhow::Result<Assignment>;
+}
+
+pub struct HexBoostData<Urban, Foot> {
+    pub urbanization: Urban,
+    pub footfall: Foot,
+}
+
+pub struct UrbanizationData<Urban, Geo> {
+    urbanized: Urban,
+    usa_geofence: Geo,
+}
+
+pub struct FootfallData<Foot> {
+    footfall: Foot,
+}
+
+impl<Urban, Foot> HexBoostData<Urban, Foot> {
+    pub fn new(urbanization: Urban, footfall: Foot) -> Self {
+        Self {
+            urbanization,
+            footfall,
+        }
+    }
+}
+
+impl<Urban, Geo> UrbanizationData<Urban, Geo> {
+    pub fn new(urbanized: Urban, usa_geofence: Geo) -> Self {
+        Self {
+            urbanized,
+            usa_geofence,
+        }
+    }
+}
+
+impl<Foot> FootfallData<Foot> {
+    pub fn new(footfall: Foot) -> Self {
+        Self { footfall }
+    }
+}
+
+trait DiskTreeLike: Send + Sync {
     fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>>;
 }
 
@@ -16,54 +75,61 @@ impl DiskTreeLike for DiskTreeMap {
     }
 }
 
-impl DiskTreeLike for HashMap<hextree::Cell, Vec<u8>> {
+impl DiskTreeLike for std::collections::HashSet<hextree::Cell> {
     fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>> {
-        Ok(self.get(&cell).map(|x| (cell, x.as_slice())))
+        match self.contains(&cell) {
+            true => Ok(Some((cell, &[]))),
+            false => Ok(None),
+        }
     }
 }
 
-pub struct MockDiskTree;
+impl<Urban, Geo> HexAssignment for UrbanizationData<Urban, Geo>
+where
+    Urban: DiskTreeLike,
+    Geo: GeofenceValidator<hextree::Cell>,
+{
+    fn assignment(&self, cell: hextree::Cell) -> anyhow::Result<Assignment> {
+        if !self.usa_geofence.in_valid_region(&cell) {
+            return Ok(Assignment::C);
+        }
 
-impl DiskTreeLike for MockDiskTree {
-    fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>> {
-        Ok(Some((cell, &[])))
+        match self.urbanized.get(cell)?.is_some() {
+            true => Ok(Assignment::A),
+            false => Ok(Assignment::B),
+        }
     }
 }
 
-pub struct Urbanization<DT, GF> {
-    urbanized: DT,
-    usa_geofence: GF,
-}
+impl<Foot> HexAssignment for FootfallData<Foot>
+where
+    Foot: DiskTreeLike,
+{
+    fn assignment(&self, cell: hextree::Cell) -> anyhow::Result<Assignment> {
+        let Some((_, vals)) = self.footfall.get(cell)? else {
+            return Ok(Assignment::C);
+        };
 
-impl<DT, GF> Urbanization<DT, GF> {
-    pub fn new(urbanized: DT, usa_geofence: GF) -> Self {
-        Self {
-            urbanized,
-            usa_geofence,
+        match vals {
+            &[x] if x >= 1 => Ok(Assignment::A),
+            &[0] => Ok(Assignment::B),
+            other => anyhow::bail!("unexpected disktree data: {cell:?} {other:?}"),
         }
     }
 }
 
-impl<DT, GF> Urbanization<DT, GF>
-where
-    DT: DiskTreeLike,
-    GF: GeofenceValidator<u64>,
-{
-    fn is_urbanized(&self, location: u64) -> anyhow::Result<bool> {
-        let cell = hextree::Cell::from_raw(location)?;
-        let result = self.urbanized.get(cell)?;
-        Ok(result.is_some())
+impl HexAssignment for Assignment {
+    fn assignment(&self, _cell: hextree::Cell) -> anyhow::Result<Assignment> {
+        Ok(*self)
     }
+}
 
-    pub fn hex_assignment(&self, hex: u64) -> anyhow::Result<Assignment> {
-        let assignment = if self.usa_geofence.in_valid_region(&hex) {
-            if self.is_urbanized(hex)? {
-                Assignment::A
-            } else {
-                Assignment::B
-            }
-        } else {
-            Assignment::C
+impl HexAssignment for HashMap<hextree::Cell, bool> {
+    fn assignment(&self, cell: hextree::Cell) -> anyhow::Result<Assignment> {
+        let assignment = match self.get(&cell) {
+            Some(true) => Assignment::A,
+            Some(false) => Assignment::B,
+            None => Assignment::C,
         };
         Ok(assignment)
     }