Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modeled Coverage - Phase 1 #578

Closed
wants to merge 50 commits into from
Closed
Show file tree
Hide file tree
Changes from 45 commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
0479b5f
Add ingest service for coverage object
maplant Jun 21, 2023
498685e
implement basic algorithm for computing coverage points
maplant Jun 26, 2023
bab457f
Fix bug where multiple signal levels would get mixed
maplant Jun 26, 2023
60814f7
Implement logic for modeled coverage
maplant Jun 27, 2023
e52dc60
Fmt
maplant Jun 27, 2023
e35ee73
Convert rewarding to take coverage into account
maplant Jun 28, 2023
7e04c77
Convert over most of the reward share tests, add fixes
Jul 3, 2023
54bcaed
Port over one more test
Jul 5, 2023
ca0434d
Address remaining issues
Jul 7, 2023
da1ccb2
Update file_store/src/error.rs
Jul 7, 2023
8a1e2a6
Update mobile_verifier/src/coverage.rs
Jul 10, 2023
57da60b
Fix tests, add setting, improve some of the code
maplant Jul 10, 2023
ba97a7e
Add coverage_object column to heartbeats
maplant Jul 10, 2023
74d6eb8
s/no/none/g
maplant Jul 10, 2023
64fa0b3
Correct hex_coverage table
maplant Jul 11, 2023
8c52ceb
Merge remote-tracking branch 'origin/main' into map/modeled-coverage
maplant Jul 11, 2023
a1c254d
Fmt and Fix
maplant Jul 11, 2023
6619bfd
Move coverage object around to satisfy OCD
maplant Jul 11, 2023
99505a3
Clippy
maplant Jul 12, 2023
6b5c69e
Fix CellHeartbeat to allow backwards compat w/ empty coverage_object
maplant Jul 12, 2023
035e4a9
Fix coverage claim time with first_timestamp
maplant Jul 12, 2023
242ad5a
Address comments
maplant Jul 13, 2023
349a92e
Move coverage claim time adjustment outside of stream
maplant Jul 13, 2023
567bd35
Move coverage claim time adjustment into heartbeat save
maplant Jul 13, 2023
a5c9d4d
Adjust migration
maplant Jul 17, 2023
9d1dffb
Move things around and cache
maplant Jul 17, 2023
0f5cd3d
Add fun todo :-)
maplant Jul 17, 2023
cb0ae3c
Correct seniority updates
maplant Jul 18, 2023
d886af8
Merge remote-tracking branch 'origin/main' into map/modeled-coverage
maplant Jul 18, 2023
5cd1b75
Fix coverage claim time adjustment
maplant Jul 18, 2023
fee3eb3
Fix seniority
maplant Jul 19, 2023
e35c82b
CMove to insert or update model for seniority
maplant Jul 19, 2023
985c990
Refactor coverage and heartbeats a little bit
maplant Jul 21, 2023
d066c29
Merge remote-tracking branch 'origin/main' into map/modeled-coverage
maplant Jul 24, 2023
a71a601
Refactor into_rewards a bit
maplant Jul 24, 2023
1deea42
Clippy
maplant Jul 24, 2023
9c88a27
Delete old hex coverages
maplant Jul 24, 2023
45638d5
Fix deletion and validated heartbeats stream
maplant Jul 25, 2023
cf527c1
Remove code that isn't part of phase 1
maplant Jul 25, 2023
c37f583
Go back to modeled coverage migration
maplant Jul 25, 2023
f4ca43e
Address comments
maplant Jul 31, 2023
aaa172f
Fix bad insert
maplant Jul 31, 2023
f6ee1fa
No longer need mutability!
maplant Jul 31, 2023
20e656e
Fmt
maplant Jul 31, 2023
d95f40d
Fix bug where we update seniority for non valid heartbeats
maplant Jul 31, 2023
a662860
Update heartbeats to not validate coverage objects
maplant Aug 1, 2023
49c2f1e
Address comments
maplant Aug 8, 2023
3c8fb99
Merge remote-tracking branch 'origin/main' into map/modeled-coverage-…
maplant Aug 8, 2023
4223a4c
Fix error in query
maplant Aug 8, 2023
1de328f
Merge remote-tracking branch 'origin/main' into map/modeled-coverage-…
maplant Aug 17, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ prost = "*"
once_cell = "1"
lazy_static = "1"
config = {version="0", default-features=false, features=["toml"]}
h3o = "0"
xorf = {version = "0", features = ["serde"] }
h3o = {version = "0", features = ["serde"]}
xorf = {version = "0", features = ["serde"]}
bytes = "*"
structopt = "0"
bincode = "1"
Expand All @@ -99,6 +99,7 @@ itertools = "*"
data-credits = {git = "https://github.com/helium/helium-program-library.git", tag = "v0.1.0"}
helium-sub-daos = {git = "https://github.com/helium/helium-program-library.git", tag = "v0.1.0"}
price-oracle = {git = "https://github.com/helium/helium-program-library.git", tag = "v0.1.0"}
uuid = {version = "1.3.4", features = ["v4", "serde"]}

[patch.crates-io]
sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb34d3e926347071dbb88d" }
2 changes: 2 additions & 0 deletions file_store/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ sqlx = {workspace = true}
async-trait = {workspace = true}
derive_builder = "0"
retainer = {workspace = true}
uuid = {workspace = true}
h3o = {workspace = true}

[dev-dependencies]
hex-literal = "0"
Expand Down
104 changes: 104 additions & 0 deletions file_store/src/coverage.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
use crate::{
error::DecodeError,
traits::{MsgDecode, TimestampDecode},
Error, Result,
};
use chrono::{DateTime, Utc};
use helium_crypto::PublicKeyBinary;
use helium_proto::services::poc_mobile::{
CoverageObjectIngestReportV1, CoverageObjectReqV1,
RadioHexSignalLevel as RadioHexSignalLevelProto, SignalLevel,
};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct RadioHexSignalLevel {
pub location: h3o::CellIndex,
pub signal_level: SignalLevel,
pub signal_power: i32,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct CoverageObject {
pub pub_key: PublicKeyBinary,
maplant marked this conversation as resolved.
Show resolved Hide resolved
pub uuid: Uuid,
pub cbsd_id: String,
pub coverage_claim_time: DateTime<Utc>,
pub indoor: bool,
pub coverage: Vec<RadioHexSignalLevel>,
pub signature: Vec<u8>,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct CoverageObjectIngestReport {
pub received_timestamp: DateTime<Utc>,
pub report: CoverageObject,
}

impl MsgDecode for CoverageObject {
type Msg = CoverageObjectReqV1;
}

impl MsgDecode for CoverageObjectIngestReport {
type Msg = CoverageObjectIngestReportV1;
}

impl TryFrom<CoverageObjectIngestReportV1> for CoverageObjectIngestReport {
type Error = Error;

fn try_from(v: CoverageObjectIngestReportV1) -> Result<Self> {
Ok(Self {
received_timestamp: v.received_timestamp.to_timestamp_millis()?,
report: v
.report
.ok_or_else(|| Error::not_found("ingest coverage object report"))?
.try_into()?,
})
}
}

impl TryFrom<CoverageObjectReqV1> for CoverageObject {
type Error = Error;

fn try_from(v: CoverageObjectReqV1) -> Result<Self> {
let coverage: Result<Vec<RadioHexSignalLevel>> = v
.coverage
.into_iter()
.map(RadioHexSignalLevel::try_from)
.collect();
Ok(Self {
pub_key: v.pub_key.into(),
uuid: Uuid::from_slice(&v.uuid).map_err(DecodeError::from)?,
cbsd_id: v.cbsd_id,
coverage_claim_time: v.coverage_claim_time.to_timestamp()?,
indoor: v.indoor,
coverage: coverage?,
signature: v.signature,
})
}
}

impl TryFrom<RadioHexSignalLevelProto> for RadioHexSignalLevel {
type Error = Error;

fn try_from(v: RadioHexSignalLevelProto) -> Result<Self> {
Ok(Self {
signal_level: SignalLevel::from_i32(v.signal_level).ok_or_else(|| {
DecodeError::unsupported_signal_level("coverage_object_req_v1", v.signal_level)
})?,
signal_power: v.signal_power,
location: v.location.parse().map_err(DecodeError::from)?,
})
}
}

impl From<RadioHexSignalLevel> for RadioHexSignalLevelProto {
fn from(rhsl: RadioHexSignalLevel) -> RadioHexSignalLevelProto {
RadioHexSignalLevelProto {
signal_level: rhsl.signal_level as i32,
signal_power: rhsl.signal_power,
location: rhsl.location.to_string(),
}
}
}
10 changes: 10 additions & 0 deletions file_store/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,14 @@ pub enum DecodeError {
UnsupportedParticipantSide(String, i32),
#[error("unsupported verification status, type: {0}, value: {1}")]
UnsupportedStatusReason(String, i32),
#[error("unsupported signal level, type: {0}, value: {1}")]
UnsupportedSignalLevel(String, i32),
#[error("invalid unix timestamp {0}")]
InvalidTimestamp(u64),
#[error("Uuid error: {0}")]
UuidError(#[from] uuid::Error),
#[error("Invalid cell index error: {0}")]
InvalidCellIndexError(#[from] h3o::error::InvalidCellIndex),
}

#[derive(Error, Debug)]
Expand Down Expand Up @@ -127,6 +133,10 @@ impl DecodeError {
pub fn unsupported_status_reason<E: ToString>(msg1: E, msg2: i32) -> Error {
Error::Decode(Self::UnsupportedInvalidReason(msg1.to_string(), msg2))
}

pub fn unsupported_signal_level(msg1: impl ToString, msg2: i32) -> Error {
Error::Decode(Self::UnsupportedSignalLevel(msg1.to_string(), msg2))
}
}

impl From<helium_crypto::Error> for Error {
Expand Down
10 changes: 10 additions & 0 deletions file_store/src/file_info.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,9 @@ pub const VALID_DATA_TRANSFER_SESSION: &str = "valid_data_transfer_session";
pub const PRICE_REPORT: &str = "price_report";
pub const MOBILE_REWARD_SHARE: &str = "mobile_reward_share";
pub const MAPPER_MSG: &str = "mapper_msg";
pub const COVERAGE_OBJECT: &str = "coverage_object";
pub const COVERAGE_OBJECT_INGEST_REPORT: &str = "coverage_object_ingest_report";
pub const SENIORITY_UPDATE: &str = "seniority_update";

#[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)]
#[serde(rename_all = "snake_case")]
Expand Down Expand Up @@ -163,7 +165,9 @@ pub enum FileType {
SubscriberLocationIngestReport,
VerifiedSubscriberLocationIngestReport,
MapperMsg,
CoverageObject,
CoverageObjectIngestReport,
SeniorityUpdate,
}

impl fmt::Display for FileType {
Expand Down Expand Up @@ -204,7 +208,9 @@ impl fmt::Display for FileType {
Self::PriceReport => PRICE_REPORT,
Self::MobileRewardShare => MOBILE_REWARD_SHARE,
Self::MapperMsg => MAPPER_MSG,
Self::CoverageObject => COVERAGE_OBJECT,
Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT,
Self::SeniorityUpdate => SENIORITY_UPDATE,
};
f.write_str(s)
}
Expand Down Expand Up @@ -248,7 +254,9 @@ impl FileType {
Self::PriceReport => PRICE_REPORT,
Self::MobileRewardShare => MOBILE_REWARD_SHARE,
Self::MapperMsg => MAPPER_MSG,
Self::CoverageObject => COVERAGE_OBJECT,
Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT,
Self::SeniorityUpdate => SENIORITY_UPDATE,
}
}
}
Expand Down Expand Up @@ -292,7 +300,9 @@ impl FromStr for FileType {
PRICE_REPORT => Self::PriceReport,
MOBILE_REWARD_SHARE => Self::MobileRewardShare,
MAPPER_MSG => Self::MapperMsg,
COVERAGE_OBJECT => Self::CoverageObject,
COVERAGE_OBJECT_INGEST_REPORT => Self::CoverageObjectIngestReport,
SENIORITY_UPDATE => Self::SeniorityUpdate,
_ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))),
};
Ok(result)
Expand Down
9 changes: 9 additions & 0 deletions file_store/src/heartbeat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use chrono::{DateTime, Utc};
use helium_crypto::PublicKeyBinary;
use helium_proto::services::poc_mobile::{CellHeartbeatIngestReportV1, CellHeartbeatReqV1};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct CellHeartbeat {
Expand All @@ -18,6 +19,13 @@ pub struct CellHeartbeat {
pub operation_mode: bool,
pub cbsd_category: String,
pub cbsd_id: String,
pub coverage_object: Vec<u8>,
}

impl CellHeartbeat {
pub fn coverage_object(&self) -> Option<Uuid> {
Uuid::from_slice(&self.coverage_object).ok()
}
}

#[derive(Serialize, Deserialize, Debug, Clone)]
Expand Down Expand Up @@ -47,6 +55,7 @@ impl TryFrom<CellHeartbeatReqV1> for CellHeartbeat {
operation_mode: v.operation_mode,
cbsd_category: v.cbsd_category,
cbsd_id: v.cbsd_id,
coverage_object: v.coverage_object,
})
}
}
Expand Down
1 change: 1 addition & 0 deletions file_store/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub mod cli;
pub mod coverage;
pub mod entropy_report;
mod error;
mod file_info;
Expand Down
4 changes: 3 additions & 1 deletion mobile_verifier/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,6 @@ reward-scheduler = {path = "../reward_scheduler"}
price = {path = "../price"}
rand = {workspace = true}
async-trait = {workspace = true}
retainer = {workspace = true}
retainer = {workspace = true}
uuid = {workspace = true}
h3o = {workspace = true}
28 changes: 28 additions & 0 deletions mobile_verifier/migrations/15_modeled_coverage.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
CREATE TYPE signal_level AS ENUM (
'none',
'low',
'medium',
'high'
);

CREATE TABLE hex_coverage (
uuid UUID NOT NULL,
hex BIGINT NOT NULL,
indoor BOOLEAN NOT NULL,
cbsd_id TEXT NOT NULL,
signal_level signal_level NOT NULL,
coverage_claim_time TIMESTAMPTZ NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL,
maplant marked this conversation as resolved.
Show resolved Hide resolved
PRIMARY KEY (uuid, hex)
);

CREATE TABLE seniority (
cbsd_id TEXT NOT NULL,
seniority_ts TIMESTAMPTZ NOT NULL,
last_heartbeat TIMESTAMPTZ NOT NULL,
uuid UUID NOT NULL,
PRIMARY KEY (cbsd_id, seniority_ts)
);

-- Coverage object can be NULL
ALTER TABLE heartbeats ADD COLUMN coverage_object UUID;
Loading