diff --git a/CHANGELOG.md b/CHANGELOG.md index f35c74f8..710c8c1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.2.0-alpha.1]-2022-12-07 ### Changed -- Use the instances workload (cpu load) to tune the results. +- Use Boavizta API v0.2.x. +- Take in consideration the instances workload (cpu load) to calculate the impacts. ## [0.1.1]- 2022-12-07 diff --git a/Cargo.lock b/Cargo.lock index 68b235b8..912e7fef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,15 +494,16 @@ dependencies = [ [[package]] name = "boavizta_api_sdk" -version = "0.1.2" +version = "0.2.0-alpha.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f744d2ef9af3bca02537b5f73e8261f6e918055f0899836cf5d2ed37de888aa1" +checksum = "5879f50b84ae02f28158af5e612cd08886cf25ef422064837c3802f4ad8178c8" dependencies = [ "reqwest", "serde", "serde_derive", "serde_json", "url", + "uuid", ] [[package]] @@ -607,7 +608,7 @@ dependencies = [ [[package]] name = "cloud-scanner-cli" -version = "0.1.1" +version = "0.2.0-alpha.1" dependencies = [ "anyhow", "async-trait", @@ -633,7 +634,7 @@ dependencies = [ [[package]] name = "cloud-scanner-lambda" -version = "0.1.1" +version = "0.2.0-alpha.1" dependencies = [ "cloud-scanner-cli", "envy", @@ -2666,6 +2667,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "uuid" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +dependencies = [ + "serde", +] + [[package]] name = "valuable" version = "0.1.0" diff --git a/README.md b/README.md index 91357e7c..f95d6391 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,6 @@ At the moment: - Cloud scanner returns _empty_ impacts (i.e. zero values) for EC2 the instance _types_ that are not listed in Boavizta database. - `--aws-region` flag only supports eu-based aws regions (eu-east-1,eu-central-1,eu-north-1,eu-south-1,eu-west-1,eu-west-2,eu-west-3). -- Returns _default_ impacts of AWS instances. It does not yet analyses instance usage (cpu workload) to calculate the impacts, but rather returns the _default_ impact data provided by Boavizta API for each instance type for a given use duration. (i.e. using instance CPU load through the `measured` command line flag has no effect). - Filtering instances by tag is not yet supported. This is work in progress, and development version may already implement theses functionalities. So have a look at the [changelog](https://github.com/Boavizta/cloud-scanner/blob/main/CHANGELOG.md) and [Issues · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues) on this repository. diff --git a/cloud-scanner-cli/Cargo.toml b/cloud-scanner-cli/Cargo.toml index 6fe2795a..11463432 100644 --- a/cloud-scanner-cli/Cargo.toml +++ b/cloud-scanner-cli/Cargo.toml @@ -2,9 +2,9 @@ authors = ["boavizta.org", "Olivier de Meringo "] edition = "2021" name = "cloud-scanner-cli" -version = "0.1.1" +version = "0.2.0-alpha.1" [dependencies] -boavizta_api_sdk = "0.1.2" +boavizta_api_sdk = "0.2.0-alpha.2" aws-types = "0.49.0" chrono = "^0.4" isocountry = "^0.3" diff --git a/cloud-scanner-cli/src/aws_inventory.rs b/cloud-scanner-cli/src/aws_inventory.rs index 8e73a22d..b98e29b6 100644 --- a/cloud-scanner-cli/src/aws_inventory.rs +++ b/cloud-scanner-cli/src/aws_inventory.rs @@ -85,7 +85,7 @@ impl AwsInventory { /// async fn get_average_cpu(self, instance_id: &str) -> Result { let res = self - .get_average_cpu_usage_of_last_5_minutes(instance_id) + .get_average_cpu_usage_of_last_10_minutes(instance_id) .await .with_context(|| { format!( @@ -94,31 +94,31 @@ impl AwsInventory { ) })?; if let Some(points) = res.datapoints { - // dbg!(points.clone()); if !points.is_empty() { - if points.len() > 1 { - warn!("Some datapoints were skipped when getting instance CPU usage, whe expected a single result but received {}. Only the first was considered", points.len()); + debug!("Averaging cpu load datapoint: {:#?}", points); + let mut sum: f64 = 0.0; + for x in &points { + sum = sum + x.average().unwrap(); } - let first_point = &points[0]; - return Ok(first_point.average.unwrap()); + let avg = sum / points.len() as f64; + return Ok(avg); } } warn!( - "No CPU load data was returned for instance {}, it is likely stopped, using 0 as load", + "Unable to get CPU load of instance {}, it is likely stopped, using 0 as load", instance_id ); Ok(0 as f64) } - /// Returns the instance CPU utilization usage on the last 24 hours - /// duration seconds seems to be the sampling period - async fn get_average_cpu_usage_of_last_5_minutes( + /// Returns the instance CPU utilization usage on the last 10 minutes + async fn get_average_cpu_usage_of_last_10_minutes( self, instance_id: &str, ) -> Result { - // We want statistics about the last 5 minutes using 60 sec sample - let measure_duration = Duration::minutes(5); - let sample_period_seconds = 60; + // We want statistics about the last 10 minutes using 5min sample + let measure_duration = Duration::minutes(10); + let sample_period_seconds = 300; // 5*60 (the default granularity of cloudwatch standard CPU metris) let now: chrono::DateTime = Utc::now(); let start_time: chrono::DateTime = now - measure_duration; @@ -228,16 +228,19 @@ mod tests { // Verify tests from here #[tokio::test] #[ignore] - async fn test_get_instance_usage_metrics_of_running_instance() { + async fn get_cpu_usage_metrics_of_running_instance_should_return_right_number_of_data_points() { let inventory: AwsInventory = AwsInventory::new("eu-west-1").await; - let res = inventory - .get_average_cpu_usage_of_last_5_minutes(&RUNNING_INSTANCE_ID) + .get_average_cpu_usage_of_last_10_minutes(&RUNNING_INSTANCE_ID) .await .unwrap(); let datapoints = res.datapoints.unwrap(); - println!("{:#?}", datapoints); - assert_eq!(1, datapoints.len()); + assert!( + 0 < datapoints.len() && datapoints.len() < 3, + "Stange number of datapoint returned. I was expecting 1 or 2 but got {} .\n {:#?}", + datapoints.len(), + datapoints + ) } #[tokio::test] @@ -246,11 +249,11 @@ mod tests { let inventory: AwsInventory = AwsInventory::new("eu-west-1").await; let instance_id = "i-03e0b3b1246001382"; let res = inventory - .get_average_cpu_usage_of_last_5_minutes(instance_id) + .get_average_cpu_usage_of_last_10_minutes(instance_id) .await .unwrap(); let datapoints = res.datapoints.unwrap(); - assert_eq!(0, datapoints.len()); + assert_eq!(0, datapoints.len(), "Wrong number of datapoint returned"); } #[tokio::test] @@ -258,7 +261,7 @@ mod tests { let inventory: AwsInventory = AwsInventory::new("eu-west-1").await; let instance_id = "IDONOTEXISTS"; let res = inventory - .get_average_cpu_usage_of_last_5_minutes(instance_id) + .get_average_cpu_usage_of_last_10_minutes(instance_id) .await .unwrap(); let datapoints = res.datapoints.unwrap(); diff --git a/cloud-scanner-cli/src/boavizta_api_v1.rs b/cloud-scanner-cli/src/boavizta_api_v1.rs index d32f9e9f..bbac37c6 100644 --- a/cloud-scanner-cli/src/boavizta_api_v1.rs +++ b/cloud-scanner-cli/src/boavizta_api_v1.rs @@ -6,7 +6,7 @@ use anyhow::Result; /// Get impacts of cloud resources through Boavizta API use boavizta_api_sdk::apis::cloud_api; use boavizta_api_sdk::apis::configuration; -use boavizta_api_sdk::models::UsageCloud; +use boavizta_api_sdk::models::{Allocation, UsageCloud}; /// Access data of Boavizta API v1 pub struct BoaviztaApiV1 { @@ -35,15 +35,17 @@ impl BoaviztaApiV1 { let instance_type = cr.resource_type; let verbose = Some(false); let mut usage_cloud: UsageCloud = UsageCloud::new(); - //let cru = cr.usage.unwrap(); - //usage_cloud.hours_use_time = Some((cru.usage_duration_seconds / 3600) as f32); usage_cloud.hours_use_time = Some(usage_duration_hours.to_owned()); usage_cloud.usage_location = Some(cr.location.iso_country_code.to_owned()); + if let Some(usage) = cr.usage { + usage_cloud.time_workload = Some(usage.average_cpu_load as f32); + } let res = cloud_api::instance_cloud_impact_v1_cloud_aws_post( &self.configuration, Some(instance_type.as_str()), verbose, + Some(Allocation::Total), Some(usage_cloud), ) .await; @@ -72,27 +74,6 @@ impl BoaviztaApiV1 { } } -/* - -#[derive(Debug)] -pub struct CloudResourceWithImpacts { - cloud_resource: CloudResource, - resource_impacts: Impacts, -} - -/// Impacts of an individual resource -#[derive(Debug, Default)] -pub struct Impacts { - pub adp_manufacture_kgsbeq: f64, - pub adp_use_kgsbeq: f64, - pub pe_manufacture_megajoules: f64, - pub pe_use_megajoules: f64, - pub gwp_manufacture_kgco2eq: f64, - pub gwp_use_kgco2eq: f64, -} - -*/ - #[async_trait] impl ImpactProvider for BoaviztaApiV1 { /// Get cloud resources impacts from the Boavizta API @@ -151,24 +132,28 @@ mod tests { use super::*; use crate::UsageLocation; - const TEST_API_URL: &str = "https://api.boavizta.org"; + // const TEST_API_URL: &str = "https://api.boavizta.org"; + // Test against local version of Boavizta API + // const TEST_API_URL: &str = "http:/localhost:5000"; + // Test against dev version of Boavizta API + const TEST_API_URL: &str = "https://dev.api.boavizta.org"; const DEFAULT_RAW_IMPACTS_OF_M6GXLARGE_1HRS_FR: &str = r#" { "adp": { - "manufacture": 0.0084, + "manufacture": 0.0083, "unit":"kgSbeq", - "use": 7.5e-10 + "use": 9e-10 }, "gwp": { - "manufacture": 87.0, + "manufacture": 83.0, "unit": "kgCO2eq", - "use": 0.0015 + "use": 0.002 }, "pe": { "manufacture": 1100.0, "unit": "MJ", - "use": 0.17 + "use": 0.2 } } "#; @@ -205,6 +190,43 @@ mod tests { assert_eq!(expected, res); } + #[tokio::test] + async fn should_retrieve_different_pe_impacts_for_different_cpu_load() { + let instance1: CloudResource = CloudResource { + id: "inst-1".to_string(), + location: UsageLocation::from("eu-west-3"), + resource_type: "m6g.xlarge".to_string(), + usage: Some(CloudResourceUsage { + average_cpu_load: 100.0, // Will not be considered in v1 + usage_duration_seconds: 3600, + }), + }; + + let instance1_1percent: CloudResource = CloudResource { + id: "inst-2".to_string(), + location: UsageLocation::from("eu-west-3"), + resource_type: "m6g.xlarge".to_string(), + usage: Some(CloudResourceUsage { + average_cpu_load: 1.0, // Will not be considered in v1 + usage_duration_seconds: 3600, + }), + }; + + let api: BoaviztaApiV1 = BoaviztaApiV1::new(TEST_API_URL); + let one_hour = 1.0 as f32; + + let mut instances: Vec = Vec::new(); + instances.push(instance1); + instances.push(instance1_1percent); + + let res = api.get_impacts(instances, &one_hour).await.unwrap(); + + let r0 = res[0].resource_impacts.clone().unwrap(); + let r1 = res[1].resource_impacts.clone().unwrap(); + assert_eq!(0.2, r0.pe_use_megajoules); + assert_eq!(0.09, r1.pe_use_megajoules); + } + #[tokio::test] async fn should_retrieve_multiple_default_impacts_fr() { let instance1: CloudResource = CloudResource { @@ -254,8 +276,8 @@ mod tests { let r1 = res[1].resource_impacts.clone().unwrap(); let r2 = res[2].resource_impacts.clone().is_none(); - assert_eq!(0.17, r0.pe_use_megajoules); - assert_eq!(0.17, r1.pe_use_megajoules); + assert_eq!(0.2, r0.pe_use_megajoules); + assert_eq!(0.2, r1.pe_use_megajoules); assert_eq!(true, r2); } @@ -280,120 +302,11 @@ mod tests { boa_impacts_to_cloud_resource_with_impacts(&instance1, &raw_impacts, &one_hour); assert_eq!( - 0.17, + 0.2, cloud_resource_with_impacts .resource_impacts .unwrap() .pe_use_megajoules ); } - /* - #[tokio::test] - async fn get_instance_default_impacts_through_sdk_works() { - let api : BoaviztaApiV1 = BoaviztaApiV1::new(TEST_API_URL); - - let instance_type = Some("m6g.xlarge"); - let verbose = Some(false); - let usage_cloud: Option = Some(UsageCloud::new()); - - let res = api.get_individual_impacts(resources) - - let res = cloud_api::instance_cloud_impact_v1_cloud_aws_post( - &configuration, - instance_type, - verbose, - usage_cloud, - ) - .await; - assert!(res.is_ok()); - let json = res.unwrap(); - println!("{:?}", json); - println!("{}", json); - } - - #[tokio::test] - async fn get_default_impact_of_m6gxlarge() { - // Parse the string of data into serde_json::Value. - let expected: serde_json::Value = serde_json::from_str(DEFAULT_IMPACT_OF_M6XLARGE).unwrap(); - - let usage_cloud: UsageCloud = UsageCloud::new(); - let instance: aws_sdk_ec2::model::Instance = aws_sdk_ec2::model::Instance::builder() - .set_instance_type(Some(aws_sdk_ec2::model::InstanceType::M6gXlarge)) - .build(); - let impacts = get_impacts(&instance, usage_cloud, TEST_API_URL).await; - - assert_eq!(expected, impacts.unwrap()); - } - - #[tokio::test] - async fn test_get_impacts_of_m6xlarge_without_region() { - // Parse the string of data into serde_json::Value. - let expected: serde_json::Value = serde_json::from_str(DEFAULT_IMPACT_OF_M6XLARGE).unwrap(); - - let usage_cloud: UsageCloud = UsageCloud::new(); - //usage_cloud.days_use_time = Some(4 as f32); - - let instance: aws_sdk_ec2::model::Instance = aws_sdk_ec2::model::Instance::builder() - .set_instance_type(Some(aws_sdk_ec2::model::InstanceType::M6gXlarge)) - .build(); - let impacts = get_impacts(&instance, usage_cloud, TEST_API_URL).await; - - assert_eq!(expected, impacts.unwrap()); - } - - #[tokio::test] - async fn test_get_impacts_of_m6xlarge_with_fr_region() { - // Parse the string of data into serde_json::Value. - let expected: serde_json::Value = - serde_json::from_str(DEFAULT_IMPACT_OF_M6XLARGE_FR).unwrap(); - - let mut usage_cloud: UsageCloud = UsageCloud::new(); - //usage_cloud.days_use_time = Some(4 as f32); - usage_cloud.usage_location = Some(String::from("FRA")); - - // impl std::convert::From<&str> for InstanceType { - // fn from(s: &str) -> Self { - // match s { - // "a1.2xlarge" => InstanceType::A12xlarge, - // "a1.4xlarge" => InstanceType::A14xlarge, - // "a1.large" => InstanceType::A1Large, - // "a1.medium" => InstanceType::A1Medium, - // "a1.metal" => InstanceType::A1Metal, - // "a1.xlarge" => InstanceType::A1Xlarge, - - // let itype: aws_sdk_ec2::model::InstanceType = - // aws_sdk_ec2::model::InstanceType::from("m6g.xlarge"); - let instance: aws_sdk_ec2::model::Instance = aws_sdk_ec2::model::Instance::builder() - .set_instance_type(Some(aws_sdk_ec2::model::InstanceType::M6gXlarge)) - .build(); - - let impacts = get_impacts(&instance, usage_cloud, TEST_API_URL).await; - - assert_eq!(expected, impacts.unwrap()); - } - - #[tokio::test] - async fn get_instance_default_impacts_through_sdk_fails_for_some_instance_types() { - let mut configuration = configuration::Configuration::new(); - configuration.base_path = String::from(TEST_API_URL); - - let known_failing_types = vec!["t2.xlarge", "t2.micro", "t2.small", "g3.4xlarge"]; - - for failing_type in known_failing_types { - let instance_type = Some(failing_type); - let verbose = Some(false); - let usage_cloud: Option = Some(UsageCloud::new()); - - let res = cloud_api::instance_cloud_impact_v1_cloud_aws_post( - &configuration, - instance_type, - verbose, - usage_cloud, - ) - .await; - - assert!(res.is_err()); - } - } - */ } diff --git a/cloud-scanner-lambda/Cargo.toml b/cloud-scanner-lambda/Cargo.toml index 16cf48fa..9c36a186 100644 --- a/cloud-scanner-lambda/Cargo.toml +++ b/cloud-scanner-lambda/Cargo.toml @@ -2,7 +2,7 @@ authors = ["boavizta.org", "Olivier de Meringo "] edition = "2021" name = "cloud-scanner-lambda" -version = "0.1.1" +version = "0.2.0-alpha.1" [[bin]] name = "bootstrap-scan" path = "src/main.rs" diff --git a/docker-compose.yml b/docker-compose.yml index 216a5fe2..51981f75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: cloud_scanner: container_name: "cloud_scanner_boa" hostname: cloud_scanner - image: ghcr.io/boavizta/cloud-scanner-cli:0.1.1 + image: ghcr.io/boavizta/cloud-scanner-cli:0.2.0-alpha.1 command: - -b http://boavizta_api:5000 - -vv @@ -53,7 +53,7 @@ services: boavizta_api: container_name: "boavizta_api" hostname: boavizta - image: ghcr.io/boavizta/boaviztapi:0.1.2 + image: ghcr.io/boavizta/boaviztapi:0.2.0 ports: - "5000:5000" networks: diff --git a/docs/book.toml b/docs/book.toml index 9a175f95..a8b21fc8 100644 --- a/docs/book.toml +++ b/docs/book.toml @@ -5,6 +5,6 @@ multilingual = false src = "src" title = "Boavizta cloud scanner 📡" -# Deacivate link checker by default +# Deactivate link checker by default # cargo install mdbook-linkcheck # [output.linkcheck] \ No newline at end of file diff --git a/docs/src/intro.md b/docs/src/intro.md index 64d55fac..48e64736 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -23,6 +23,4 @@ Cloud-scanner is an Open Source application maintained here: