From 5650652466ccc18765420fab44153f666c0ac3a6 Mon Sep 17 00:00:00 2001 From: cn-kali-team Date: Wed, 6 Dec 2023 18:16:08 +0800 Subject: [PATCH] update --- .github/workflows/fly.yml | 4 +- .gitignore | 3 +- README.md | 4 +- cve/src/api/mod.rs | 1 + cvss/src/severity.rs | 2 +- cwe/src/structured_text.rs | 5 + {doc => docs}/cve.png | Bin {doc => docs}/details.png | Bin helper/examples/cpe-api-example.rs | 2 +- helper/examples/cve-api-example.rs | 2 +- helper/src/bin/api_to_db.rs | 183 ------------------ helper/src/bin/cpe_match.rs | 36 ---- helper/src/bin/cpe_to_db.rs | 29 --- .../src/{bin/cve_to_db.rs => import_cve.rs} | 154 ++++++++------- .../src/{bin/cwe_to_db.rs => import_cwe.rs} | 31 ++- helper/src/lib.rs | 8 +- helper/src/main.rs | 52 +++++ nvd-api/src/lib.rs | 4 +- nvd-api/src/v2/mod.rs | 9 +- nvd-api/src/v2/products.rs | 4 +- nvd-api/src/v2/vulnerabilities.rs | 7 +- nvd-server/src/api/cwe_api.rs | 25 +++ nvd-server/src/api/mod.rs | 6 + nvd-server/src/modules/cwe_db.rs | 63 ++++++ nvd-server/src/modules/mod.rs | 2 +- nvd-yew/src/component/cvss.rs | 20 +- nvd-yew/src/component/cvss_tags.rs | 6 +- nvd-yew/src/component/mod.rs | 2 + nvd-yew/src/component/weaknesses.rs | 81 ++++++++ nvd-yew/src/modules/cwe.rs | 30 +++ nvd-yew/src/modules/mod.rs | 1 + nvd-yew/src/routes/cve.rs | 36 +++- nvd-yew/src/services/cve.rs | 5 + 33 files changed, 449 insertions(+), 368 deletions(-) rename {doc => docs}/cve.png (100%) rename {doc => docs}/details.png (100%) delete mode 100644 helper/src/bin/api_to_db.rs delete mode 100644 helper/src/bin/cpe_match.rs delete mode 100644 helper/src/bin/cpe_to_db.rs rename helper/src/{bin/cve_to_db.rs => import_cve.rs} (69%) rename helper/src/{bin/cwe_to_db.rs => import_cwe.rs} (62%) create mode 100644 helper/src/main.rs create mode 100644 nvd-server/src/api/cwe_api.rs create mode 100644 nvd-yew/src/component/weaknesses.rs create mode 100644 nvd-yew/src/modules/cwe.rs diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 0e6ae31..ab4ecf5 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -1,8 +1,8 @@ name: Fly Deploy on: push: - branches: - - main + tags: + - "v*" jobs: deploy: name: Deploy app diff --git a/.gitignore b/.gitignore index 1770e07..00a5dbf 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,9 @@ Cargo.lock *.xml.gz *.json.gz *.xml.zip -/helper/examples/nvdcve/data-feeds.html +/helper/examples/nvdcve/ /.env /nvd-server/nvd-er.mwb.bak /nvd-server/dist/ /dist/ +/helper/examples/nvdcwe/ diff --git a/README.md b/README.md index 15d6474..1b5ea7d 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ - A collection of lib for working with [National Vulnerability Database](https://nvd.nist.gov/). ## CVE -![CVE列表](doc/cve.png) +![CVE列表](docs/cve.png) ## Details -![CVE详情](doc/details.png) +![CVE详情](docs/details.png) ## Libraries diff --git a/cve/src/api/mod.rs b/cve/src/api/mod.rs index 2a434c8..da083f8 100644 --- a/cve/src/api/mod.rs +++ b/cve/src/api/mod.rs @@ -29,6 +29,7 @@ pub enum VulnStatus { #[serde(rename = "Undergoing Analysis")] UndergoingAnalysis, Rejected, + Received, } #[cfg(test)] diff --git a/cvss/src/severity.rs b/cvss/src/severity.rs index eaba457..2d3e954 100644 --- a/cvss/src/severity.rs +++ b/cvss/src/severity.rs @@ -103,7 +103,7 @@ pub enum SeverityTypeV2 { impl Display for SeverityTypeV2 { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "S:{}", self.as_str()) + write!(f, "{}", self.as_str()) } } diff --git a/cwe/src/structured_text.rs b/cwe/src/structured_text.rs index 363160a..de649e3 100644 --- a/cwe/src/structured_text.rs +++ b/cwe/src/structured_text.rs @@ -26,6 +26,11 @@ pub enum StructuredTextType { #[serde(rename(deserialize = "$value"), default)] children: Vec>, }, + #[serde(rename(deserialize = "sup"))] + Sup { + #[serde(rename(deserialize = "$value"), default)] + children: Vec>, + }, #[serde(rename(deserialize = "b"))] XhtmlB { #[serde(rename(deserialize = "$value"), default)] diff --git a/doc/cve.png b/docs/cve.png similarity index 100% rename from doc/cve.png rename to docs/cve.png diff --git a/doc/details.png b/docs/details.png similarity index 100% rename from doc/details.png rename to docs/details.png diff --git a/helper/examples/cpe-api-example.rs b/helper/examples/cpe-api-example.rs index 3d216fa..22496b7 100644 --- a/helper/examples/cpe-api-example.rs +++ b/helper/examples/cpe-api-example.rs @@ -1,5 +1,5 @@ -use nvd_api::ApiVersion; use nvd_api::v2::products::{CpeMatchParameters, CpeParameters}; +use nvd_api::ApiVersion; // https://cwe.mitre.org/data/downloads.html // curl -s -k https://cwe.mitre.org/data/downloads.html |grep -Eo '(/[^"]*\.xml.zip)'|xargs -I % wget -c https://cwe.mitre.org% #[tokio::main] diff --git a/helper/examples/cve-api-example.rs b/helper/examples/cve-api-example.rs index 0b27549..1160989 100644 --- a/helper/examples/cve-api-example.rs +++ b/helper/examples/cve-api-example.rs @@ -1,5 +1,5 @@ -use nvd_api::ApiVersion; use nvd_api::v2::vulnerabilities::{CveHistoryParameters, CveParameters}; +use nvd_api::ApiVersion; // https://cwe.mitre.org/data/downloads.html // curl -s -k https://cwe.mitre.org/data/downloads.html |grep -Eo '(/[^"]*\.xml.zip)'|xargs -I % wget -c https://cwe.mitre.org% #[tokio::main] diff --git a/helper/src/bin/api_to_db.rs b/helper/src/bin/api_to_db.rs deleted file mode 100644 index 8b38954..0000000 --- a/helper/src/bin/api_to_db.rs +++ /dev/null @@ -1,183 +0,0 @@ -use cached::proc_macro::cached; -use cached::SizedCache; -use chrono::{NaiveDateTime, Utc}; -use cve::v4::{CVEContainer, CVEItem}; -use diesel::mysql::MysqlConnection; -use helper::init_db_pool; -use nvd_api::v2::vulnerabilities::CveParameters; -use nvd_api::v2::LastModDate; -use nvd_api::ApiVersion; -use nvd_server::error::DBResult; -use nvd_server::modules::cve_db::CreateCve; -use nvd_server::modules::cve_product_db::CreateCveProductByName; -use nvd_server::modules::product_db::{CreateProduct, QueryProductById}; -use nvd_server::modules::vendor_db::CreateVendors; -use nvd_server::modules::{Cve, CveProduct, Product, Vendor}; -use std::str::FromStr; - -// https://cwe.mitre.org/data/downloads.html -// curl -s -k https://cwe.mitre.org/data/downloads.html |grep -Eo '(/[^"]*\.xml.zip)'|xargs -I % wget -c https://cwe.mitre.org% - -fn import_to_db(connection: &mut MysqlConnection, cve_item: CVEItem) -> DBResult { - let id = cve_item.cve.meta.id; - let y = id.split('-').nth(1).unwrap_or_default(); - let new_post = CreateCve { - id: id.clone(), - created_at: cve_item.published_date, - updated_at: cve_item.last_modified_date, - references: serde_json::json!(cve_item.cve.references.reference_data), - description: serde_json::json!(cve_item.cve.description.description_data), - severity: cve_item.impact.severity(), - metrics: serde_json::json!(cve_item.impact), - assigner: cve_item.cve.meta.assigner, - configurations: serde_json::json!(cve_item.configurations.nodes), - year: i32::from_str(y).unwrap_or_default(), - weaknesses: serde_json::json!(cve_item.cve.problem_type.problem_type_data), - timeline: Default::default(), - }; - // 插入到数据库 - match Cve::create(connection, &new_post) { - Ok(cve_id) => { - // 插入cpe_match关系表 - for node in cve_item.configurations.nodes { - for vendor_product in node.vendor_product() { - import_vendor_product_to_db(connection, vendor_product.clone()); - create_cve_product( - connection, - cve_id.id.clone(), - vendor_product.vendor, - vendor_product.product, - ); - } - } - } - Err(err) => { - println!("Cve::create: {err:?}"); - } - } - Ok(new_post.id) -} - -pub fn create_cve_product( - conn: &mut MysqlConnection, - cve_id: String, - vendor: String, - product: String, -) -> String { - // 构建待插入对象 - let cp = CreateCveProductByName { - cve_id, - vendor, - product, - }; - // 插入到数据库 - match CveProduct::create_by_name(conn, &cp) { - Ok(_cp) => {} - Err(err) => { - println!("create_cve_product: {err:?}:{cp:?}"); - } - } - String::new() -} - -#[cached( - type = "SizedCache>", - create = "{ SizedCache::with_size(100) }", - convert = r#"{ format!("{:?}", product.to_owned()) }"# -)] -fn import_vendor_product_to_db(connection: &mut MysqlConnection, product: cpe::Product) -> Vec { - let vendor_id = create_vendor(connection, product.vendor, None); - create_product(connection, vendor_id, product.product, product.part) -} - -#[cached( - type = "SizedCache>", - create = "{ SizedCache::with_size(100) }", - convert = r#"{ format!("{}", name.to_owned()) }"# -)] -pub fn create_vendor( - conn: &mut MysqlConnection, - name: String, - description: Option, -) -> Vec { - if let Ok(v) = Vendor::query_by_name(conn, &name) { - return v.id; - } - // 构建待插入对象 - let new_post = CreateVendors { - id: uuid::Uuid::new_v4().as_bytes().to_vec(), - name, - description, - official: u8::from(true), - homepage: None, - }; - // 插入到数据库 - if let Err(err) = Vendor::create(conn, &new_post) { - println!("create_vendor: {err:?}"); - } - new_post.id -} - -#[cached( - type = "SizedCache>", - create = "{ SizedCache::with_size(100) }", - convert = r#"{ format!("{}:{:?}", name.to_owned(),vendor.to_owned()) }"# -)] -pub fn create_product( - conn: &mut MysqlConnection, - vendor: Vec, - name: String, - part: String, -) -> Vec { - let q = QueryProductById { - vendor_id: vendor.clone(), - name: name.clone(), - }; - if let Ok(v) = Product::query_by_id(conn, &q) { - return v.id; - } - // 构建待插入对象 - let new_post = CreateProduct { - id: uuid::Uuid::new_v4().as_bytes().to_vec(), - vendor_id: vendor, - name, - description: None, - official: u8::from(true), - part, - homepage: None, - }; - // 插入到数据库 - if let Err(err) = Product::create(conn, &new_post) { - println!("create_product: {err:?}"); - } - new_post.id -} - -fn main() { - // let connection_pool = init_db_pool(); - // let api = nvd_api::NVDApi::new(None, ApiVersion::default()).unwrap(); - let now = Utc::now(); - let two_h = - println!("{:?}", now); - // api.cve(CveParameters{ - // cpe_name: None, - // cve_id: None, - // cvss_v2_metrics: None, - // cvss_v2_severity: None, - // cvss_v3_metrics: None, - // cvss_v3_severity: None, - // cwe_id: None, - // has_cert_alerts: None, - // has_cert_notes: None, - // has_kev: None, - // has_oval: None, - // is_vulnerable: None, - // keyword: None, - // last_mod: Some(LastModDate{ last_mod_start_date: "".to_string(), last_mod_end_date: "".to_string() }), - // no_rejected: None, - // pub_date: None, - // limit_offset: None, - // source_identifier: None, - // virtual_match: None, - // }) -} diff --git a/helper/src/bin/cpe_match.rs b/helper/src/bin/cpe_match.rs deleted file mode 100644 index 9603094..0000000 --- a/helper/src/bin/cpe_match.rs +++ /dev/null @@ -1,36 +0,0 @@ -use diesel::{BelongingToDsl, ExpressionMethods, QueryDsl, RunQueryDsl}; -use helper::init_db_pool; -use nvd_server::modules::cve_product_db::CreateCveProductByName; -use nvd_server::modules::{CveProduct, Product, Vendor}; -use nvd_server::schema::{products, vendors}; -use std::ops::DerefMut; -// mysql> drop database nvd; -// Query OK, 7 rows affected (0.04 sec) -// -// mysql> RESET MASTER; -// Query OK, 0 rows affected (0.01 sec) -// -// mysql> RESET SLAVE; -// Query OK, 0 rows affected, 1 warning (0.00 sec) -// -// mysql> PURGE BINARY LOGS BEFORE NOW(); -// Query OK, 0 rows affected, 1 warning (0.00 sec) - -fn main() { - let connection_pool = init_db_pool(); - // 联表查询 - let vendor_id: Vendor = vendors::table - .filter(vendors::name.eq("microsoft")) - .first(connection_pool.get().unwrap().deref_mut()) - .unwrap(); - let _product = Product::belonging_to(&vendor_id) - .filter(products::name.eq("windows_nt")) - .first::(connection_pool.get().unwrap().deref_mut()) - .unwrap(); - let cp = CreateCveProductByName { - cve_id: "CVE-1999-0595".to_string(), - vendor: "microsoft".to_string(), - product: "windows_nt".to_string(), - }; - CveProduct::create_by_name(connection_pool.get().unwrap().deref_mut(), &cp).unwrap(); -} diff --git a/helper/src/bin/cpe_to_db.rs b/helper/src/bin/cpe_to_db.rs deleted file mode 100644 index b628de4..0000000 --- a/helper/src/bin/cpe_to_db.rs +++ /dev/null @@ -1,29 +0,0 @@ -use cpe::dictionary::CPEList; - -use std::fs::File; -use std::io::BufReader; - -// 建立连接 - -// https://cwe.mitre.org/data/downloads.html -// curl -s -k https://cwe.mitre.org/data/downloads.html |grep -Eo '(/[^"]*\.xml.zip)'|xargs -I % wget -c https://cwe.mitre.org% - -fn main() { - let gz_open_file = File::open("examples/nvdcve/official-cpe-dictionary_v2.3.xml.gz").unwrap(); - let gz_decoder = flate2::read::GzDecoder::new(gz_open_file); - let file = BufReader::new(gz_decoder); - let c: CPEList = quick_xml::de::from_reader(file).unwrap(); - // let mut flag = false; - for cpe_item in c.cpe_item.into_iter() { - let _vendor = cpe_item.cpe23_item.name.vendor.to_string(); - let _product = cpe_item.cpe23_item.name.product.to_string(); - // 已经弃用的不再加入数据库 - if cpe_item.name.contains("pdgsoft") { - println!("{}", cpe_item.cpe23_item.name); - } - if cpe_item.deprecated { - continue; - } - continue; - } -} diff --git a/helper/src/bin/cve_to_db.rs b/helper/src/import_cve.rs similarity index 69% rename from helper/src/bin/cve_to_db.rs rename to helper/src/import_cve.rs index 9af6b80..077e3f3 100644 --- a/helper/src/bin/cve_to_db.rs +++ b/helper/src/import_cve.rs @@ -1,63 +1,17 @@ use cached::proc_macro::cached; use cached::SizedCache; -use cve::v4::{CVEContainer, CVEItem}; +use cve::v4::CVEItem; use diesel::mysql::MysqlConnection; -use helper::init_db_pool; use nvd_server::error::DBResult; use nvd_server::modules::cve_db::CreateCve; use nvd_server::modules::cve_product_db::CreateCveProductByName; use nvd_server::modules::product_db::{CreateProduct, QueryProductById}; use nvd_server::modules::vendor_db::CreateVendors; use nvd_server::modules::{Cve, CveProduct, Product, Vendor}; -use std::fs::File; -use std::io::BufReader; -use std::ops::DerefMut; use std::str::FromStr; -// https://cwe.mitre.org/data/downloads.html -// curl -s -k https://cwe.mitre.org/data/downloads.html |grep -Eo '(/[^"]*\.xml.zip)'|xargs -I % wget -c https://cwe.mitre.org% - -fn import_to_db(connection: &mut MysqlConnection, cve_item: CVEItem) -> DBResult { - let id = cve_item.cve.meta.id; - let y = id.split('-').nth(1).unwrap_or_default(); - let new_post = CreateCve { - id: id.clone(), - created_at: cve_item.published_date, - updated_at: cve_item.last_modified_date, - references: serde_json::json!(cve_item.cve.references.reference_data), - description: serde_json::json!(cve_item.cve.description.description_data), - severity: cve_item.impact.severity(), - metrics: serde_json::json!(cve_item.impact), - assigner: cve_item.cve.meta.assigner, - configurations: serde_json::json!(cve_item.configurations.nodes), - year: i32::from_str(y).unwrap_or_default(), - weaknesses: serde_json::json!(cve_item.cve.problem_type.problem_type_data), - timeline: Default::default(), - }; - // 插入到数据库 - match Cve::create(connection, &new_post) { - Ok(cve_id) => { - // 插入cpe_match关系表 - for node in cve_item.configurations.nodes { - for vendor_product in node.vendor_product() { - import_vendor_product_to_db(connection, vendor_product.clone()); - create_cve_product( - connection, - cve_id.id.clone(), - vendor_product.vendor, - vendor_product.product, - ); - } - } - } - Err(err) => { - println!("Cve::create: {err:?}"); - } - } - Ok(new_post.id) -} - -pub fn create_cve_product( +// curl --compressed https://nvd.nist.gov/vuln/data-feeds -o-|grep -Eo '(/feeds\/[^"]*\.json\.gz)'|xargs -I % wget -c https://nvd.nist.gov% +fn create_cve_product( conn: &mut MysqlConnection, cve_id: String, vendor: String, @@ -94,11 +48,7 @@ fn import_vendor_product_to_db(connection: &mut MysqlConnection, product: cpe::P create = "{ SizedCache::with_size(100) }", convert = r#"{ format!("{}", name.to_owned()) }"# )] -pub fn create_vendor( - conn: &mut MysqlConnection, - name: String, - description: Option, -) -> Vec { +fn create_vendor(conn: &mut MysqlConnection, name: String, description: Option) -> Vec { if let Ok(v) = Vendor::query_by_name(conn, &name) { return v.id; } @@ -122,7 +72,7 @@ pub fn create_vendor( create = "{ SizedCache::with_size(100) }", convert = r#"{ format!("{}:{:?}", name.to_owned(),vendor.to_owned()) }"# )] -pub fn create_product( +fn create_product( conn: &mut MysqlConnection, vendor: Vec, name: String, @@ -152,18 +102,88 @@ pub fn create_product( new_post.id } -fn main() { - let connection_pool = init_db_pool(); - for y in 2002..2024 { - let p = format!("helper/examples/nvdcve/nvdcve-1.1-{y}.json.gz"); - println!("{p}"); - let gz_open_file = File::open(p).unwrap(); - let gz_decoder = flate2::read::GzDecoder::new(gz_open_file); - let file = BufReader::new(gz_decoder); - let c: CVEContainer = serde_json::from_reader(file).unwrap(); - for w in c.CVE_Items { - import_to_db(connection_pool.get().unwrap().deref_mut(), w).unwrap_or_default(); +pub fn import_from_archive( + connection: &mut MysqlConnection, + cve_item: CVEItem, +) -> DBResult { + let id = cve_item.cve.meta.id; + let y = id.split('-').nth(1).unwrap_or_default(); + let new_post = CreateCve { + id: id.clone(), + created_at: cve_item.published_date, + updated_at: cve_item.last_modified_date, + references: serde_json::json!(cve_item.cve.references.reference_data), + description: serde_json::json!(cve_item.cve.description.description_data), + severity: cve_item.impact.severity(), + metrics: serde_json::json!(cve_item.impact), + assigner: cve_item.cve.meta.assigner, + configurations: serde_json::json!(cve_item.configurations.nodes), + year: i32::from_str(y).unwrap_or_default(), + weaknesses: serde_json::json!(cve_item.cve.problem_type.problem_type_data), + timeline: Default::default(), + }; + // 插入到数据库 + match Cve::create(connection, &new_post) { + Ok(cve_id) => { + // 插入cpe_match关系表 + for node in cve_item.configurations.nodes { + for vendor_product in node.vendor_product() { + import_vendor_product_to_db(connection, vendor_product.clone()); + create_cve_product( + connection, + cve_id.id.clone(), + vendor_product.vendor, + vendor_product.product, + ); + } + } + } + Err(err) => { + println!("Cve::create: {err:?}"); + } + } + Ok(new_post.id) +} + +pub fn import_from_api( + connection: &mut MysqlConnection, + cve_item: cve::api::CVE, +) -> DBResult { + let id = cve_item.id; + let y = id.split('-').nth(1).unwrap_or_default(); + let new_post = CreateCve { + id: id.clone(), + created_at: cve_item.published, + updated_at: cve_item.last_modified, + references: serde_json::json!(cve_item.references), + description: serde_json::json!(cve_item.descriptions), + severity: cve_item.metrics.severity(), + metrics: serde_json::json!(cve_item.metrics), + assigner: cve_item.source_identifier, + configurations: serde_json::json!(cve_item.configurations), + year: i32::from_str(y).unwrap_or_default(), + weaknesses: serde_json::json!(cve_item.weaknesses), + timeline: Default::default(), + }; + // 插入到数据库 + match Cve::create(connection, &new_post) { + Ok(cve_id) => { + // 插入cpe_match关系表 + for node in cve_item.configurations { + for vendor_product in node.vendor_product() { + import_vendor_product_to_db(connection, vendor_product.clone()); + create_cve_product( + connection, + cve_id.id.clone(), + vendor_product.vendor, + vendor_product.product, + ); + } + } + } + Err(err) => { + println!("Cve::create: {err:?}"); } - // break; } + Ok(new_post.id) } diff --git a/helper/src/bin/cwe_to_db.rs b/helper/src/import_cwe.rs similarity index 62% rename from helper/src/bin/cwe_to_db.rs rename to helper/src/import_cwe.rs index 04f65e0..2a022d4 100644 --- a/helper/src/bin/cwe_to_db.rs +++ b/helper/src/import_cwe.rs @@ -1,6 +1,7 @@ +use crate::init_db_pool; use cwe::weakness_catalog::WeaknessCatalog; -use diesel::mysql::MysqlConnection; -use helper::init_db_pool; +use cwe::weaknesses::Weakness; +use diesel::MysqlConnection; use nvd_server::modules::cwe_db::CreateCwe; use nvd_server::modules::Cwe; use std::fs::File; @@ -8,35 +9,25 @@ use std::io::BufReader; use std::ops::DerefMut; // https://cwe.mitre.org/data/downloads.html // curl -s -k https://cwe.mitre.org/data/downloads.html |grep -Eo '(/[^"]*\.xml.zip)'|xargs -I % wget -c https://cwe.mitre.org% -fn import_to_db( - connection: &mut MysqlConnection, - id: i32, - name: String, - description: String, -) -> i32 { - println!("import_to_db: {id}:{name}"); + +fn import_cwe_from_archive(connection: &mut MysqlConnection, w: Weakness) -> i32 { let new_post = CreateCwe { - id, - name, - description, + id: w.id, + name: w.name, + description: w.description, }; // 插入到数据库 let _v = Cwe::create(connection, &new_post); new_post.id } -fn main() { +pub fn import_cwe() { let connection_pool = init_db_pool(); - let zip_open_file = File::open("examples/nvdcwe/cwec_latest.xml.zip").unwrap(); + let zip_open_file = File::open("helper/examples/nvdcwe/cwec_latest.xml.zip").unwrap(); let mut zip_archive = zip::ZipArchive::new(zip_open_file).unwrap(); let file = BufReader::new(zip_archive.by_index(0).unwrap()); let c: WeaknessCatalog = quick_xml::de::from_reader(file).unwrap(); for w in c.weaknesses.weaknesses { - import_to_db( - connection_pool.get().unwrap().deref_mut(), - w.id, - w.name, - w.description, - ); + import_cwe_from_archive(connection_pool.get().unwrap().deref_mut(), w); } } diff --git a/helper/src/lib.rs b/helper/src/lib.rs index b5b5be9..f7d7f0c 100644 --- a/helper/src/lib.rs +++ b/helper/src/lib.rs @@ -1,6 +1,10 @@ +mod import_cve; +mod import_cwe; + use diesel::r2d2::ConnectionManager; use diesel::{r2d2, MysqlConnection}; - +pub use import_cve::{import_from_api, import_from_archive}; +pub use import_cwe::import_cwe; pub type Connection = MysqlConnection; pub type Pool = r2d2::Pool>; @@ -8,7 +12,7 @@ pub type Pool = r2d2::Pool>; pub fn init_db_pool() -> Pool { let database_url = dotenvy::var("DATABASE_URL").expect("DATABASE_URL must be set"); let manager = ConnectionManager::::new(database_url); - r2d2::Pool::builder() + Pool::builder() .build(manager) .expect("Failed to create pool.") } diff --git a/helper/src/main.rs b/helper/src/main.rs new file mode 100644 index 0000000..bd6cb3a --- /dev/null +++ b/helper/src/main.rs @@ -0,0 +1,52 @@ +use chrono::{Duration, Utc}; +use cve::v4::CVEContainer; +use helper::{import_from_api, import_from_archive, init_db_pool}; +use nvd_api::pagination::Object; +use nvd_api::v2::vulnerabilities::CveParameters; +use nvd_api::v2::LastModDate; +use nvd_api::ApiVersion; +use std::fs::File; +use std::io::BufReader; +use std::ops::DerefMut; + +#[tokio::main] +async fn main() { + // import_cwe(); + with_archive(); + std::process::exit(0); + + let connection_pool = init_db_pool(); + let api = nvd_api::NVDApi::new(None, ApiVersion::default()).unwrap(); + let now = Utc::now(); + // 每两个小时拉取三小时内的更新数据入库 + let three_hours = now - Duration::hours(3); + let param = CveParameters { + last_mod: Some(LastModDate { + last_mod_start_date: three_hours.to_rfc3339(), + last_mod_end_date: now.to_rfc3339(), + }), + ..CveParameters::default() + }; + let resp = api.cve(param).await.unwrap(); + if let Object::Vulnerabilities(vs) = resp.results { + for v in vs { + import_from_api(connection_pool.get().unwrap().deref_mut(), v.cve).unwrap(); + } + } +} + +fn with_archive() { + let connection_pool = init_db_pool(); + for y in (2002..2024).rev() { + let p = format!("helper/examples/nvdcve/nvdcve-1.1-{y}.json.gz"); + println!("{p}"); + let gz_open_file = File::open(p).unwrap(); + let gz_decoder = flate2::read::GzDecoder::new(gz_open_file); + let file = BufReader::new(gz_decoder); + let c: CVEContainer = serde_json::from_reader(file).unwrap(); + for w in c.CVE_Items { + import_from_archive(connection_pool.get().unwrap().deref_mut(), w).unwrap_or_default(); + } + // break; + } +} diff --git a/nvd-api/src/lib.rs b/nvd-api/src/lib.rs index 1c204ce..1cd7843 100644 --- a/nvd-api/src/lib.rs +++ b/nvd-api/src/lib.rs @@ -14,6 +14,7 @@ pub struct NVDApi { version: String, client: reqwest::Client, } + pub enum ApiVersion { V2_0, } @@ -23,6 +24,7 @@ impl Default for ApiVersion { Self::V2_0 } } + impl ToString for ApiVersion { fn to_string(&self) -> String { match self { @@ -30,6 +32,7 @@ impl ToString for ApiVersion { } } } + impl NVDApi { pub fn new(api_token: Option, version: ApiVersion) -> Result { let mut headers = reqwest::header::HeaderMap::new(); @@ -62,7 +65,6 @@ impl NVDApi { .text() .await .map_err(|source| Error::ResponseIo { source })?; - // println!("{}", json); let result = serde_json::from_str(&json).map_err(|source| Error::JsonParse { source })?; Ok(result) } diff --git a/nvd-api/src/v2/mod.rs b/nvd-api/src/v2/mod.rs index ccbb904..2c34e5a 100644 --- a/nvd-api/src/v2/mod.rs +++ b/nvd-api/src/v2/mod.rs @@ -4,22 +4,25 @@ pub mod api; pub mod products; pub mod vulnerabilities; -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct LimitOffset { pub results_per_page: Option, pub start_index: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct Keyword { pub keyword_exact_match: bool, pub keyword_search: String, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] + +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct LastModDate { + /// rfc3339 string pub last_mod_start_date: String, + /// rfc3339 string pub last_mod_end_date: String, } diff --git a/nvd-api/src/v2/products.rs b/nvd-api/src/v2/products.rs index 59ca4dc..c8aa848 100644 --- a/nvd-api/src/v2/products.rs +++ b/nvd-api/src/v2/products.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// /// Please note, new users are discouraged from starting with the 1.0 API as it will be retired in 2023 but you may still view documentation for the 1.0 Vulnerability and 1.0 Product APIs. /// -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct CpeParameters { pub cpe_name_id: Option, @@ -22,7 +22,7 @@ pub struct CpeParameters { pub limit_offset: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct CpeMatchParameters { pub cve_id: Option, diff --git a/nvd-api/src/v2/vulnerabilities.rs b/nvd-api/src/v2/vulnerabilities.rs index cb00d1d..76a2963 100644 --- a/nvd-api/src/v2/vulnerabilities.rs +++ b/nvd-api/src/v2/vulnerabilities.rs @@ -3,7 +3,7 @@ use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; // https://nvd.nist.gov/developers/vulnerabilities -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct CveParameters { pub cpe_name: Option, @@ -32,7 +32,7 @@ pub struct CveParameters { pub virtual_match: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct VirtualMatch { pub virtual_match_string: String, @@ -125,6 +125,7 @@ pub enum EventName { pub struct CveChanges { pub change: Change, } + #[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct Change { @@ -135,6 +136,7 @@ pub struct Change { pub created: NaiveDateTime, pub details: Vec
, } + #[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] #[serde(rename_all = "camelCase")] pub struct Details { @@ -143,6 +145,7 @@ pub struct Details { pub old_value: Option, pub new_value: Option, } + #[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Eq)] pub enum Action { Added, diff --git a/nvd-server/src/api/cwe_api.rs b/nvd-server/src/api/cwe_api.rs new file mode 100644 index 0000000..917f1cd --- /dev/null +++ b/nvd-server/src/api/cwe_api.rs @@ -0,0 +1,25 @@ +use crate::modules::cwe_db::QueryCwe; +use crate::modules::Cwe; +use crate::{ApiResponse, Pool}; +use actix_web::{get, web, Error, HttpResponse}; +use std::ops::DerefMut; + +#[get("/{id}")] +async fn api_cwe_id(id: web::Path, pool: web::Data) -> Result { + let contact = web::block(move || { + let mut conn = pool.get()?; + Cwe::query_by_id(conn.deref_mut(), &id) + }) + .await??; + Ok(HttpResponse::Ok().json(contact)) +} + +#[get("")] +async fn api_cwe_list(args: web::Query, pool: web::Data) -> ApiResponse { + let contact = web::block(move || { + let mut conn = pool.get()?; + Cwe::query(conn.deref_mut(), &args) + }) + .await??; + Ok(HttpResponse::Ok().json(contact)) +} diff --git a/nvd-server/src/api/mod.rs b/nvd-server/src/api/mod.rs index 8727cca..9925467 100644 --- a/nvd-server/src/api/mod.rs +++ b/nvd-server/src/api/mod.rs @@ -1,4 +1,5 @@ mod cve_api; +mod cwe_api; mod product_api; mod vendor_api; @@ -16,5 +17,10 @@ pub fn api_route(cfg: &mut web::ServiceConfig) { .service(vendor_api::api_vendor_name) .service(vendor_api::api_vendor_list), ) + .service( + web::scope("/cwe") + .service(cwe_api::api_cwe_id) + .service(cwe_api::api_cwe_list), + ) .service(web::scope("/product").service(product_api::api_product_list)); } diff --git a/nvd-server/src/modules/cwe_db.rs b/nvd-server/src/modules/cwe_db.rs index 9bb590a..2a093c0 100644 --- a/nvd-server/src/modules/cwe_db.rs +++ b/nvd-server/src/modules/cwe_db.rs @@ -1,8 +1,10 @@ use crate::error::{DBError, DBResult}; use crate::modules::Cwe; use crate::schema::cwes; +use crate::DB; use diesel::prelude::*; use diesel::result::{DatabaseErrorKind, Error as DieselError}; +use serde::{Deserialize, Serialize}; #[derive(Insertable)] #[diesel(table_name = cwes)] @@ -12,6 +14,46 @@ pub struct CreateCwe { pub description: String, } +#[derive(Debug, Serialize, Deserialize)] +pub struct CweCount { + pub result: Vec, + pub total: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct QueryCwe { + pub id: Option, + pub name: Option, + pub limit: Option, + pub offset: Option, +} + +impl QueryCwe { + fn query<'a>( + &'a self, + _conn: &mut MysqlConnection, + mut query: cwes::BoxedQuery<'a, DB>, + ) -> DBResult> { + if let Some(name) = &self.name { + let name = format!("%{name}%"); + query = query.filter(cwes::name.like(name)); + } + if let Some(id) = &self.id { + query = query.filter(cwes::id.eq(id)); + } + Ok(query) + } + fn total(&self, conn: &mut MysqlConnection) -> DBResult { + let query = self.query(conn, cwes::table.into_boxed())?; + // 统计查询全部，分页用 + Ok( + query + .select(diesel::dsl::count(cwes::id)) + .first::(conn)?, + ) + } +} + impl Cwe { // 创建弱点枚举 pub fn create(conn: &mut MysqlConnection, args: &CreateCwe) -> DBResult { @@ -31,4 +73,25 @@ impl Cwe { .first::(conn)?, ) } + pub fn query_by_id(conn: &mut MysqlConnection, id: &i32) -> DBResult { + Ok( + cwes::dsl::cwes + .filter(cwes::id.eq(id)) + .first::(conn)?, + ) + } + pub fn query(conn: &mut MysqlConnection, args: &QueryCwe) -> DBResult { + let total = args.total(conn)?; + let offset = args.offset.unwrap_or(0).abs(); + let limit = std::cmp::min(args.limit.to_owned().unwrap_or(10).abs(), 10); + let result = { + let query = args.query(conn, cwes::table.into_boxed())?; + query + .offset(offset) + .limit(limit) + .order(cwes::name.asc()) + .load::(conn)? + }; + Ok(CweCount { result, total }) + } } diff --git a/nvd-server/src/modules/mod.rs b/nvd-server/src/modules/mod.rs index 09a8ef5..92ad137 100644 --- a/nvd-server/src/modules/mod.rs +++ b/nvd-server/src/modules/mod.rs @@ -66,7 +66,7 @@ pub struct Vendor { pub created_at: NaiveDateTime, } -#[derive(Queryable, Debug)] +#[derive(Queryable, Identifiable, Selectable, Debug, PartialEq, Serialize, Deserialize)] pub struct Cwe { pub id: i32, pub name: String, diff --git a/nvd-yew/src/component/cvss.rs b/nvd-yew/src/component/cvss.rs index e3662f7..789f44f 100644 --- a/nvd-yew/src/component/cvss.rs +++ b/nvd-yew/src/component/cvss.rs @@ -23,6 +23,8 @@ impl Component for CVSS3 { let v3 = ctx.props().v3.clone().unwrap(); let cvss_v3 = v3.cvss_v3.clone(); let score = v3.cvss_v3.base_score; + let source = v3.source.clone(); + let who = v3.r#type.clone(); let exploit_ability_score = round_score(v3.exploitability_score); let impact_score = round_score(v3.impact_score); html! { @@ -42,8 +44,12 @@ impl Component for CVSS3 {
- - {"V3 Legend"} +
    +
  • + {format!("{:?}",who)}
    • +
  • + {source}
    • +
@@ -94,6 +100,8 @@ impl Component for CVSS2 { fn view(&self, ctx: &Context) -> Html { let v2 = ctx.props().v2.clone().unwrap(); + let who = v2.r#type.clone(); + let source = v2.source.clone(); let cvss_v2 = v2.cvss_v2.clone(); let score = v2.cvss_v2.base_score; let exploit_ability_score = round_score(v2.exploitability_score); @@ -115,8 +123,12 @@ impl Component for CVSS2 {
- - {"V2 Legend"} +
    +
  • + {format!("{:?}",who)}
    • +
  • + {source}
    • +
diff --git a/nvd-yew/src/component/cvss_tags.rs b/nvd-yew/src/component/cvss_tags.rs index 3302c5a..52f64aa 100644 --- a/nvd-yew/src/component/cvss_tags.rs +++ b/nvd-yew/src/component/cvss_tags.rs @@ -24,11 +24,7 @@ pub fn cvss3(metric: Option<&cvss::v3::ImpactMetricV3>) -> Html { let severity_class = match metric { None => "bg-secondary", Some(m) => { - score = format!( - "{} {}", - m.cvss_v3.base_score, - m.cvss_v3.base_severity - ); + score = format!("{} {}", m.cvss_v3.base_score, m.cvss_v3.base_severity); match m.cvss_v3.base_severity { SeverityType::None => "bg-secondary", SeverityType::Low => "bg-info", diff --git a/nvd-yew/src/component/mod.rs b/nvd-yew/src/component/mod.rs index 5eaadea..331d935 100644 --- a/nvd-yew/src/component/mod.rs +++ b/nvd-yew/src/component/mod.rs @@ -5,6 +5,7 @@ mod cvss; pub mod cvss_tags; mod pagination; mod tooltip_popover; +mod weaknesses; pub use cve_configuration::{CVEConfiguration, CVEConfigurationProps}; pub use cve_query::{CVEQuery, CVEQueryProps}; @@ -12,3 +13,4 @@ pub use cve_row::{CVERow, CveProps}; pub use cvss::{CVSS2, CVSS3}; pub use pagination::{Pagination, PaginationProps}; pub use tooltip_popover::TooltipPopover; +pub use weaknesses::CWEDetails; diff --git a/nvd-yew/src/component/weaknesses.rs b/nvd-yew/src/component/weaknesses.rs new file mode 100644 index 0000000..63487d9 --- /dev/null +++ b/nvd-yew/src/component/weaknesses.rs @@ -0,0 +1,81 @@ +use crate::console_log; +use crate::modules::cwe::Cwe; +use crate::services::cve::cwe_details; +use crate::services::FetchState; +use std::str::FromStr; +use yew::prelude::*; + +// 单行的cve信息，和点击供应商，产品回调 +#[derive(PartialEq, Clone, Properties)] +pub struct WeaknessesProps { + pub id: String, +} + +pub struct CWEDetails { + cwe: Option, +} + +pub enum Msg { + SetFetchState(FetchState), + Send, +} + +impl Component for CWEDetails { + type Message = Msg; + type Properties = WeaknessesProps; + + fn create(_ctx: &Context) -> Self { + Self { cwe: None } + } + + fn update(&mut self, ctx: &Context, msg: Self::Message) -> bool { + match msg { + Msg::SetFetchState(state) => { + match state { + FetchState::Success(data) => self.cwe = Some(data), + FetchState::Failed(err) => { + console_log!("{:?}", err); + } + } + return true; + } + Msg::Send => { + let id = i32::from_str(ctx.props().id.trim_start_matches("CWE-")).unwrap_or(0); + ctx.link().send_future(async move { + match cwe_details(id).await { + Ok(data) => Msg::SetFetchState(FetchState::Success(data)), + Err(err) => Msg::SetFetchState(FetchState::Failed(err)), + } + }); + } + } + false + } + fn view(&self, _ctx: &Context) -> Html { + if let Some(cwe) = self.cwe.clone() { + let mut description = cwe.description.chars(); + return html! { +
+
+ +

{cwe.name}{format!("CWE-{}",cwe.id)}

+
+
+
+ +
+
+
+

{description.next().unwrap_or_default()}{description.collect::()}

+
+
+ }; + } + html!() + } + fn rendered(&mut self, ctx: &Context, first_render: bool) { + if first_render { + ctx.link().send_message(Msg::Send); + } + } +} diff --git a/nvd-yew/src/modules/cwe.rs b/nvd-yew/src/modules/cwe.rs new file mode 100644 index 0000000..367c061 --- /dev/null +++ b/nvd-yew/src/modules/cwe.rs @@ -0,0 +1,30 @@ +use serde::{Deserialize, Serialize}; +use yew::prelude::*; +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Properties)] +pub struct Cwe { + pub id: i32, + pub name: String, + pub description: String, +} +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)] +pub struct CweInfoList { + // 结果数据 + pub result: Vec, + // 分页每页 + pub limit: i64, + // 分页偏移 + pub offset: i64, + // 结果总数 + pub total: i64, + #[serde(skip)] + pub query: QueryCwe, +} +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone, Properties)] +pub struct QueryCwe { + // 精准CWE编号 + pub id: Option, + // 分页每页 + pub limit: Option, + // 分页偏移 + pub offset: Option, +} diff --git a/nvd-yew/src/modules/mod.rs b/nvd-yew/src/modules/mod.rs index 8693f06..97499c7 100644 --- a/nvd-yew/src/modules/mod.rs +++ b/nvd-yew/src/modules/mod.rs @@ -1 +1,2 @@ pub mod cve; +pub mod cwe; diff --git a/nvd-yew/src/routes/cve.rs b/nvd-yew/src/routes/cve.rs index 6856523..be85ad1 100644 --- a/nvd-yew/src/routes/cve.rs +++ b/nvd-yew/src/routes/cve.rs @@ -1,5 +1,5 @@ use crate::component::cvss_tags::{cvss2, cvss3}; -use crate::component::{CVEConfiguration, CVEConfigurationProps, CVSS2, CVSS3}; +use crate::component::{CVEConfiguration, CVEConfigurationProps, CWEDetails, CVSS2, CVSS3}; use crate::console_log; use crate::modules::cve::Cve; use crate::services::cve::cve_details; @@ -96,9 +96,10 @@ impl Component for CVEDetails {
{self.description(cve.description.clone())}
- {self.cvss(cve.clone())} - {self.references(cve.references)} - {self.configurations(cve.configurations)} + {self.cvss(cve.clone())} + {self.references(cve.references)} + {self.weaknesses(cve.weaknesses.clone())} + {self.configurations(cve.configurations)}
@@ -183,7 +184,7 @@ impl CVEDetails { {reference.into_iter().map(|r|{ html!{ - {r.name} + {r.url} {r.source} @@ -211,4 +212,29 @@ impl CVEDetails { }; html! {} } + + fn weaknesses(&self, ws: Vec) -> Html { + // CVE-2006-5757有多个cwe + html! { +
+
+
+ +
+
+ {ws.into_iter().flat_map(|w|w.description).map(|d|{ + html!{} + }).collect::()} + +
+
+
+
+
+ } + } } diff --git a/nvd-yew/src/services/cve.rs b/nvd-yew/src/services/cve.rs index 830f686..d69921b 100644 --- a/nvd-yew/src/services/cve.rs +++ b/nvd-yew/src/services/cve.rs @@ -1,6 +1,7 @@ use super::request_get; use crate::error::Error; use crate::modules::cve::{Cve, CveInfoList, QueryCve}; +use crate::modules::cwe::Cwe; pub async fn cve_list(query: QueryCve) -> Result { request_get::("cve".to_string(), query).await @@ -8,3 +9,7 @@ pub async fn cve_list(query: QueryCve) -> Result { pub async fn cve_details(id: String) -> Result { request_get::<(), Cve>(format!("cve/{}", id), ()).await } + +pub async fn cwe_details(id: i32) -> Result { + request_get::<(), Cwe>(format!("cwe/{}", id), ()).await +}