From 52fce44f662cd90729a031cdd1fdd6728ff3b473 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 14 Jan 2025 14:56:10 +0100 Subject: [PATCH] feat: add a way to find related SBOMs by CPE Also, get rid of the central analysis graph instance. --- Cargo.lock | 26 +- Cargo.toml | 3 +- common/Cargo.toml | 1 + common/src/purl.rs | 9 +- etc/test-data/spdx/simple.json | 3 +- modules/analysis/Cargo.toml | 2 + modules/analysis/README.md | 7 + modules/analysis/src/endpoints.rs | 73 +- modules/analysis/src/error.rs | 4 + modules/analysis/src/lib.rs | 7 +- modules/analysis/src/model.rs | 44 +- modules/analysis/src/service.rs | 1337 ----------------- modules/analysis/src/service/load.rs | 380 +++++ modules/analysis/src/service/mod.rs | 494 ++++++ modules/analysis/src/service/query.rs | 75 + modules/analysis/src/service/test.rs | 410 +++++ modules/fundamental/Cargo.toml | 1 + .../src/ai/service/tools/package_info.rs | 16 +- modules/fundamental/src/endpoints.rs | 4 +- modules/fundamental/src/sbom/endpoints/mod.rs | 103 +- .../fundamental/src/sbom/endpoints/query.rs | 61 + modules/fundamental/src/sbom/model/mod.rs | 44 +- modules/fundamental/src/sbom/service/sbom.rs | 131 +- modules/fundamental/src/sbom/service/test.rs | 28 +- modules/fundamental/src/test/common.rs | 4 +- modules/fundamental/tests/sbom/reingest.rs | 3 +- .../tests/sbom/spdx/corner_cases.rs | 17 +- modules/importer/Cargo.toml | 1 + .../src/runner/clearly_defined/mod.rs | 6 +- .../runner/clearly_defined_curation/mod.rs | 6 +- modules/importer/src/runner/csaf/mod.rs | 6 +- modules/importer/src/runner/cve/mod.rs | 6 +- modules/importer/src/runner/cwe/mod.rs | 6 +- modules/importer/src/runner/mod.rs | 2 + modules/importer/src/runner/osv/mod.rs | 6 +- modules/importer/src/runner/sbom/mod.rs | 6 +- modules/importer/src/server/mod.rs | 5 + modules/ingestor/src/endpoints.rs | 4 +- modules/ingestor/src/service/mod.rs | 56 +- .../src/service/sbom/clearly_defined.rs | 2 +- .../service/sbom/clearly_defined_curation.rs | 2 +- .../ingestor/src/service/sbom/cyclonedx.rs | 2 +- modules/ingestor/src/service/sbom/spdx.rs | 2 +- modules/ingestor/tests/common.rs | 13 +- openapi.yaml | 94 +- server/src/profile/api.rs | 10 +- server/src/profile/importer.rs | 12 +- test-context/src/lib.rs | 2 +- xtask/src/dataset.rs | 2 + 49 files changed, 1861 insertions(+), 1677 deletions(-) create mode 100644 modules/analysis/README.md delete mode 100644 modules/analysis/src/service.rs create mode 100644 modules/analysis/src/service/load.rs create mode 100644 modules/analysis/src/service/mod.rs create mode 100644 modules/analysis/src/service/query.rs create mode 100644 modules/analysis/src/service/test.rs create mode 100644 modules/fundamental/src/sbom/endpoints/query.rs diff --git a/Cargo.lock b/Cargo.lock index b533110de..0d5beb36c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2529,6 +2529,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.0.35" @@ -3784,7 +3790,7 @@ dependencies = [ "ena", "itertools 0.11.0", "lalrpop-util", - "petgraph", + "petgraph 0.6.5", "regex", "regex-syntax 0.8.5", "string_cache", @@ -5078,7 +5084,17 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.7.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", "indexmap 2.7.0", "serde", "serde_derive", @@ -8315,6 +8331,7 @@ dependencies = [ "native-tls", "packageurl", "pem", + "percent-encoding", "postgresql_embedded", "rand", "regex", @@ -8441,6 +8458,7 @@ dependencies = [ "bytes", "bytesize", "chrono", + "cpe", "criterion", "csaf", "hex", @@ -8449,7 +8467,7 @@ dependencies = [ "log", "packageurl", "parking_lot 0.12.3", - "petgraph", + "petgraph 0.7.1", "sea-orm", "sea-query", "serde", @@ -8525,6 +8543,7 @@ dependencies = [ "trustify-common", "trustify-cvss", "trustify-entity", + "trustify-module-analysis", "trustify-module-ingestor", "trustify-module-storage", "trustify-test-context", @@ -8597,6 +8616,7 @@ dependencies = [ "trustify-auth", "trustify-common", "trustify-entity", + "trustify-module-analysis", "trustify-module-ingestor", "trustify-module-storage", "trustify-test-context", diff --git a/Cargo.toml b/Cargo.toml index 9e622c548..a721b0e27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,8 @@ packageurl = "0.3.0" parking_lot = "0.12" peak_alloc = "0.2.0" pem = "3" -petgraph = { version = "0.6.5", features = ["serde-1"] } +percent-encoding = "2.3.1" +petgraph = { version = "0.7.1", features = ["serde-1"] } prometheus = "0.13.3" quick-xml = "0.37.0" rand = "0.8.5" diff --git a/common/Cargo.toml b/common/Cargo.toml index 30fa4d6f3..eb323a323 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -23,6 +23,7 @@ log = { workspace = true } native-tls = { workspace = true } packageurl = { workspace = true } pem = { workspace = true } +percent-encoding = { workspace = true } postgresql_embedded = { workspace = true, features = ["blocking", "tokio"] } regex = { workspace = true } reqwest = { workspace = true, features = ["native-tls"] } diff --git a/common/src/purl.rs b/common/src/purl.rs index 510aedd5b..a2c0b2e80 100644 --- a/common/src/purl.rs +++ b/common/src/purl.rs @@ -1,4 +1,5 @@ use packageurl::PackageUrl; +use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use serde::{ de::{Error, Visitor}, Deserialize, Deserializer, Serialize, Serializer, @@ -157,7 +158,7 @@ impl Display for Purl { "?{}", self.qualifiers .iter() - .map(|(k, v)| format!("{}={}", k, v)) + .map(|(k, v)| format!("{}={}", k, utf8_percent_encode(v, NON_ALPHANUMERIC))) .collect::>() .join("&") ) @@ -250,7 +251,7 @@ mod tests { async fn purl_oci() -> Result<(), anyhow::Error> { let purl: Purl = serde_json::from_str( r#" - "pkg:oci/ose-cluster-network-operator@sha256:0170ba5eebd557fd9f477d915bb7e0d4c1ad6cd4c1852d4b1ceed7a2817dd5d2?repository_url=registry.redhat.io/openshift4/ose-cluster-network-operator&tag=v4.11.0-202403090037.p0.g33da9fb.assembly.stream.el8" + "pkg:oci/ose-cluster-network-operator@sha256:0170ba5eebd557fd9f477d915bb7e0d4c1ad6cd4c1852d4b1ceed7a2817dd5d2?repository_url=registry.redhat.io%2Fopenshift4%2Fose%2Dcluster%2Dnetwork%2Doperator&tag=v4%2E11%2E0%2D202403090037%2Ep0%2Eg33da9fb%2Eassembly%2Estream%2Eel8" "#, ) .unwrap(); @@ -273,12 +274,12 @@ mod tests { Some(&"v4.11.0-202403090037.p0.g33da9fb.assembly.stream.el8".to_string()) ); - let purl: Purl = "pkg:oci/ose-cluster-network-operator@sha256:0170ba5eebd557fd9f477d915bb7e0d4c1ad6cd4c1852d4b1ceed7a2817dd5d2?repository_url=registry.redhat.io/openshift4/ose-cluster-network-operator&tag=v4.11.0-202403090037.p0.g33da9fb.assembly.stream.el8".try_into()?; + let purl: Purl = "pkg:oci/ose-cluster-network-operator@sha256:0170ba5eebd557fd9f477d915bb7e0d4c1ad6cd4c1852d4b1ceed7a2817dd5d2?repository_url=registry%2Eredhat%2Eio%2Eopenshift4%2Eose%2Dcluster%2Dnetwork%2Doperator&tag=v4%2E11%2E0%2D202403090037%2Ep0%2Eg33da9fb%2Eassembly%2Estream%2Eel8".try_into()?; let json = serde_json::to_string(&purl).unwrap(); assert_eq!( json, - r#""pkg:oci/ose-cluster-network-operator@sha256:0170ba5eebd557fd9f477d915bb7e0d4c1ad6cd4c1852d4b1ceed7a2817dd5d2?repository_url=registry.redhat.io/openshift4/ose-cluster-network-operator&tag=v4.11.0-202403090037.p0.g33da9fb.assembly.stream.el8""# + r#""pkg:oci/ose-cluster-network-operator@sha256:0170ba5eebd557fd9f477d915bb7e0d4c1ad6cd4c1852d4b1ceed7a2817dd5d2?repository_url=registry%2Eredhat%2Eio%2Eopenshift4%2Eose%2Dcluster%2Dnetwork%2Doperator&tag=v4%2E11%2E0%2D202403090037%2Ep0%2Eg33da9fb%2Eassembly%2Estream%2Eel8""# ); Ok(()) } diff --git a/etc/test-data/spdx/simple.json b/etc/test-data/spdx/simple.json index 2fc01dec1..ef4f01ced 100644 --- a/etc/test-data/spdx/simple.json +++ b/etc/test-data/spdx/simple.json @@ -12,7 +12,6 @@ "documentNamespace": "uri:just-an-example", "name": "simple", "packages": [ - { "SPDXID": "SPDXRef-A", "copyrightText": "NOASSERTION", @@ -26,7 +25,7 @@ { "referenceCategory": "SECURITY", "referenceLocator": "cpe:/a:redhat:simple:1::el9", - "referenceType": "cpe23Type" + "referenceType": "cpe22Type" } ], "filesAnalyzed": false, diff --git a/modules/analysis/Cargo.toml b/modules/analysis/Cargo.toml index 0b2f1acda..980dd0f1c 100644 --- a/modules/analysis/Cargo.toml +++ b/modules/analysis/Cargo.toml @@ -13,12 +13,14 @@ trustify-entity = { workspace = true } actix-http = { workspace = true } actix-web = { workspace = true } anyhow = { workspace = true } +cpe = { workspace = true } log = { workspace = true } petgraph = { workspace = true } parking_lot= { workspace = true } sea-orm = { workspace = true } sea-query = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } utoipa = { workspace = true, features = ["actix_extras", "uuid"] } diff --git a/modules/analysis/README.md b/modules/analysis/README.md new file mode 100644 index 000000000..773d78911 --- /dev/null +++ b/modules/analysis/README.md @@ -0,0 +1,7 @@ +# Analysis Graph + +## Get a component + +```bash +http localhost:8080/api/v2/analysis/root-component/B +``` diff --git a/modules/analysis/src/endpoints.rs b/modules/analysis/src/endpoints.rs index eea34e800..5b09a70be 100644 --- a/modules/analysis/src/endpoints.rs +++ b/modules/analysis/src/endpoints.rs @@ -65,7 +65,7 @@ pub async fn search_component_root_components( ) -> actix_web::Result { Ok(HttpResponse::Ok().json( service - .retrieve_root_components(search, paginated, db.as_ref()) + .retrieve_root_components(&search, paginated, db.as_ref()) .await?, )) } @@ -92,13 +92,13 @@ pub async fn get_component_root_components( let purl: Purl = Purl::from_str(&key).map_err(Error::Purl)?; Ok(HttpResponse::Ok().json( service - .retrieve_root_components_by_purl(purl, paginated, db.as_ref()) + .retrieve_root_components(&purl, paginated, db.as_ref()) .await?, )) } else { Ok(HttpResponse::Ok().json( service - .retrieve_root_components_by_name(key.to_string(), paginated, db.as_ref()) + .retrieve_root_components(&key.to_string(), paginated, db.as_ref()) .await?, )) } @@ -125,7 +125,7 @@ pub async fn search_component_deps( ) -> actix_web::Result { Ok(HttpResponse::Ok().json( service - .retrieve_deps(search, paginated, db.as_ref()) + .retrieve_deps(&search, paginated, db.as_ref()) .await?, )) } @@ -150,15 +150,11 @@ pub async fn get_component_deps( ) -> actix_web::Result { if key.starts_with("pkg:") { let purl: Purl = Purl::from_str(&key).map_err(Error::Purl)?; - Ok(HttpResponse::Ok().json( - service - .retrieve_deps_by_purl(purl, paginated, db.as_ref()) - .await?, - )) + Ok(HttpResponse::Ok().json(service.retrieve_deps(&purl, paginated, db.as_ref()).await?)) } else { Ok(HttpResponse::Ok().json( service - .retrieve_deps_by_name(key.to_string(), paginated, db.as_ref()) + .retrieve_deps(&key.to_string(), paginated, db.as_ref()) .await?, )) } @@ -187,8 +183,14 @@ mod test { let request: Request = TestRequest::get().uri(uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; - if response["items"][0]["purl"] == "pkg:rpm/redhat/BB@0.0.0" - || response["items"][1]["purl"] == "pkg:rpm/redhat/BB@0.0.0" + if response["items"][0]["purl"] + .as_array() + .unwrap() + .contains(&Value::from("pkg:rpm/redhat/BB@0.0.0")) + || response["items"][1]["purl"] + .as_array() + .unwrap() + .contains(&Value::from("pkg:rpm/redhat/BB@0.0.0")) { assert_eq!(&response["total"], 2); } else { @@ -200,10 +202,13 @@ mod test { let uri = "/api/v2/analysis/root-component?q=BB"; let request: Request = TestRequest::get().uri(uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; - assert_eq!(response["items"][0]["purl"], "pkg:rpm/redhat/BB@0.0.0"); + assert_eq!( + response["items"][0]["purl"], + Value::from(["pkg:rpm/redhat/BB@0.0.0"]) + ); assert_eq!( response["items"][0]["ancestors"][0]["purl"], - "pkg:rpm/redhat/AA@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/AA@0.0.0?arch=src"]) ); assert_eq!(&response["total"], 1); @@ -224,10 +229,13 @@ mod test { let response: Value = app.call_and_read_body_json(request).await; - assert_eq!(response["items"][0]["purl"], "pkg:rpm/redhat/B@0.0.0"); + assert_eq!( + response["items"][0]["purl"], + Value::from(["pkg:rpm/redhat/B@0.0.0"]) + ); assert_eq!( response["items"][0]["ancestors"][0]["purl"], - "pkg:rpm/redhat/A@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/A@0.0.0?arch=src"]) ); assert_eq!(&response["total"], 1); Ok(()) @@ -247,10 +255,13 @@ mod test { let response: Value = app.call_and_read_body_json(request).await; - assert_eq!(response["items"][0]["purl"], "pkg:rpm/redhat/B@0.0.0"); + assert_eq!( + response["items"][0]["purl"], + Value::from(["pkg:rpm/redhat/B@0.0.0"]) + ); assert_eq!( response["items"][0]["ancestors"][0]["purl"], - "pkg:rpm/redhat/A@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/A@0.0.0?arch=src"]) ); assert_eq!(&response["total"], 1); Ok(()) @@ -276,7 +287,7 @@ mod test { assert_eq!( response["items"][0]["purl"], - "pkg:maven/net.spy/spymemcached@2.12.1?type=jar" + Value::from(["pkg:maven/net.spy/spymemcached@2.12.1?type=jar"]) ); assert_eq!( response["items"][0]["document_id"], @@ -284,7 +295,7 @@ mod test { ); assert_eq!( response["items"][0]["ancestors"][0]["purl"], - "pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.11.Final-redhat-00001?type=pom&repository_url=https%3a%2f%2fmaven.repository.redhat.com%2fga%2f" + Value::from(["pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.11.Final-redhat-00001?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=pom"]) ); assert_eq!(&response["total"], 2); @@ -334,20 +345,22 @@ mod test { let request: Request = TestRequest::get().uri(uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; + println!("Result: {response:#?}"); + assert_eq!( response["items"][0]["purl"], - "pkg:rpm/redhat/A@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/A@0.0.0?arch=src"]), ); assert_eq!( response["items"][0]["deps"][0]["purl"], - "pkg:rpm/redhat/EE@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/EE@0.0.0?arch=src"]), ); assert_eq!( response["items"][0]["deps"][1]["purl"], - "pkg:rpm/redhat/B@0.0.0" + Value::from(["pkg:rpm/redhat/B@0.0.0"]), ); - assert_eq!(&response["total"], 3); + assert_eq!(&response["total"], 2); Ok(()) } @@ -364,15 +377,15 @@ mod test { assert_eq!( response["items"][0]["purl"], - "pkg:rpm/redhat/A@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/A@0.0.0?arch=src"]), ); assert_eq!( response["items"][0]["deps"][0]["purl"], - "pkg:rpm/redhat/EE@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/EE@0.0.0?arch=src"]), ); assert_eq!( response["items"][0]["deps"][1]["purl"], - "pkg:rpm/redhat/B@0.0.0" + Value::from(["pkg:rpm/redhat/B@0.0.0"]), ); assert_eq!(&response["total"], 1); @@ -392,11 +405,11 @@ mod test { assert_eq!( response["items"][0]["purl"], - "pkg:rpm/redhat/AA@0.0.0?arch=src" + Value::from(["pkg:rpm/redhat/AA@0.0.0?arch=src"]), ); assert_eq!( response["items"][0]["deps"][0]["purl"], - "pkg:rpm/redhat/BB@0.0.0" + Value::from(["pkg:rpm/redhat/BB@0.0.0"]), ); assert_eq!(&response["total"], 1); Ok(()) @@ -420,7 +433,7 @@ mod test { assert_eq!( response["items"][0]["purl"], - "pkg:maven/net.spy/spymemcached@2.12.1?type=jar" + Value::from(["pkg:maven/net.spy/spymemcached@2.12.1?type=jar"]), ); assert_eq!(&response["total"], 2); Ok(()) diff --git a/modules/analysis/src/error.rs b/modules/analysis/src/error.rs index 3030f1a28..ed3a72569 100644 --- a/modules/analysis/src/error.rs +++ b/modules/analysis/src/error.rs @@ -1,7 +1,9 @@ use actix_http::StatusCode; use actix_web::body::BoxBody; use actix_web::{HttpResponse, ResponseError}; +use cpe::uri::OwnedUri; use sea_orm::DbErr; +use std::str::FromStr; use trustify_common::error::ErrorInformation; use trustify_common::id::IdError; use trustify_common::purl::PurlErr; @@ -17,6 +19,8 @@ pub enum Error { #[error(transparent)] Purl(#[from] PurlErr), #[error(transparent)] + Cpe(::Err), + #[error(transparent)] Actix(#[from] actix_web::Error), #[error("Invalid request {msg}")] BadRequest { msg: String, status: StatusCode }, diff --git a/modules/analysis/src/lib.rs b/modules/analysis/src/lib.rs index c77c634a6..45345e0ad 100644 --- a/modules/analysis/src/lib.rs +++ b/modules/analysis/src/lib.rs @@ -1,11 +1,8 @@ pub mod endpoints; - -pub mod service; - pub mod error; - +pub mod service; pub use error::Error; - pub mod model; + #[cfg(test)] pub mod test; diff --git a/modules/analysis/src/model.rs b/modules/analysis/src/model.rs index 4169fe411..12fc434ee 100644 --- a/modules/analysis/src/model.rs +++ b/modules/analysis/src/model.rs @@ -1,18 +1,17 @@ -use parking_lot::RwLock; use petgraph::Graph; use serde::Serialize; -use std::{ - collections::HashMap, - fmt, - sync::{Arc, OnceLock}, -}; +use std::{collections::HashMap, fmt}; +use trustify_common::cpe::Cpe; +use trustify_common::purl::Purl; use trustify_entity::relationship::Relationship; use utoipa::ToSchema; #[derive(Debug, Clone, PartialEq, Eq, ToSchema, serde::Serialize)] pub struct AnalysisStatus { - pub sbom_count: i32, - pub graph_count: i32, + /// The number of SBOMs found in the database + pub sbom_count: u32, + /// The number of graphs loaded in memory + pub graph_count: u32, } impl fmt::Display for AnalysisStatus { @@ -25,7 +24,8 @@ impl fmt::Display for AnalysisStatus { pub struct PackageNode { pub sbom_id: String, pub node_id: String, - pub purl: String, + pub purl: Vec, + pub cpe: Vec, pub name: String, pub version: String, pub published: String, @@ -35,7 +35,7 @@ pub struct PackageNode { } impl fmt::Display for PackageNode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.purl) + write!(f, "{:?}", self.purl) } } @@ -44,14 +44,15 @@ pub struct AncNode { pub sbom_id: String, pub node_id: String, pub relationship: String, - pub purl: String, + pub purl: Vec, + pub cpe: Vec, pub name: String, pub version: String, } impl fmt::Display for AncNode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.purl) + write!(f, "{:?}", self.purl) } } @@ -59,7 +60,8 @@ impl fmt::Display for AncNode { pub struct AncestorSummary { pub sbom_id: String, pub node_id: String, - pub purl: String, + pub purl: Vec, + pub cpe: Vec, pub name: String, pub version: String, pub published: String, @@ -74,7 +76,8 @@ pub struct DepNode { pub sbom_id: String, pub node_id: String, pub relationship: String, - pub purl: String, + pub purl: Vec, + pub cpe: Vec, pub name: String, pub version: String, #[schema(no_recursion)] @@ -82,14 +85,15 @@ pub struct DepNode { } impl fmt::Display for DepNode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.purl) + write!(f, "{:?}", self.purl) } } #[derive(Debug, Clone, Serialize, ToSchema)] pub struct DepSummary { pub sbom_id: String, pub node_id: String, - pub purl: String, + pub purl: Vec, + pub cpe: Vec, pub name: String, pub version: String, pub published: String, @@ -103,8 +107,6 @@ pub struct GraphMap { map: HashMap>, } -static G: OnceLock>> = OnceLock::new(); - impl GraphMap { // Create a new instance of GraphMap pub fn new() -> Self { @@ -147,12 +149,6 @@ impl GraphMap { self.map.keys().cloned().collect() } - // Get the singleton instance of GraphMap - pub fn get_instance() -> Arc> { - G.get_or_init(|| Arc::new(RwLock::new(GraphMap::new()))) - .clone() - } - // Clear all graphs from the map pub fn clear(&mut self) { self.map.clear(); diff --git a/modules/analysis/src/service.rs b/modules/analysis/src/service.rs deleted file mode 100644 index bae2999a1..000000000 --- a/modules/analysis/src/service.rs +++ /dev/null @@ -1,1337 +0,0 @@ -use crate::{ - model::{AnalysisStatus, AncNode, AncestorSummary, DepNode, DepSummary, GraphMap, PackageNode}, - Error, -}; -use petgraph::{ - algo::is_cyclic_directed, - graph::{Graph, NodeIndex}, - visit::{NodeIndexable, VisitMap, Visitable}, - Direction, -}; -use sea_orm::{ - prelude::ConnectionTrait, ColumnTrait, DatabaseBackend, DbErr, EntityOrSelect, EntityTrait, - QueryFilter, QueryOrder, QueryResult, QuerySelect, QueryTrait, Statement, -}; -use sea_query::Order; -use std::{ - collections::{HashMap, HashSet}, - str::FromStr, -}; -use tracing::instrument; -use trustify_common::{ - db::query::{Filtering, Query, Value}, - model::{Paginated, PaginatedResults}, - purl::Purl, -}; -use trustify_entity::{relationship::Relationship, sbom, sbom_node}; -use uuid::Uuid; - -#[derive(Default)] -pub struct AnalysisService {} - -pub fn dep_nodes( - graph: &Graph, - node: NodeIndex, - visited: &mut HashSet, -) -> Vec { - let mut depnodes = Vec::new(); - fn dfs( - graph: &Graph, - node: NodeIndex, - depnodes: &mut Vec, - visited: &mut HashSet, - ) { - if visited.contains(&node) { - return; - } - visited.insert(node); - for neighbor in graph.neighbors_directed(node, Direction::Incoming) { - if let Some(dep_packagenode) = graph.node_weight(neighbor).cloned() { - // Attempt to find the edge and get the relationship in a more elegant way - if let Some(relationship) = graph - .find_edge(neighbor, node) - .and_then(|edge_index| graph.edge_weight(edge_index)) - { - let dep_node = DepNode { - sbom_id: dep_packagenode.sbom_id, - node_id: dep_packagenode.node_id, - relationship: relationship.to_string(), - purl: dep_packagenode.purl.to_string(), - name: dep_packagenode.name.to_string(), - version: dep_packagenode.version.to_string(), - deps: dep_nodes(graph, neighbor, visited), - }; - depnodes.push(dep_node); - dfs(graph, neighbor, depnodes, visited); - } - } else { - log::warn!( - "Processing descendants node weight for neighbor {:?} not found", - neighbor - ); - } - } - } - dfs(graph, node, &mut depnodes, visited); - depnodes -} - -pub fn ancestor_nodes( - graph: &Graph, - node: NodeIndex, -) -> Vec { - let mut discovered = graph.visit_map(); - let mut ancestor_nodes = Vec::new(); - let mut stack = Vec::new(); - - stack.push(graph.from_index(node.index())); - - while let Some(node) = stack.pop() { - if discovered.visit(node) { - for succ in graph.neighbors_directed(node, Direction::Outgoing) { - if !discovered.is_visited(&succ) { - if let Some(anc_packagenode) = graph.node_weight(succ).cloned() { - if let Some(edge) = graph.find_edge(node, succ) { - if let Some(relationship) = graph.edge_weight(edge) { - let anc_node = AncNode { - sbom_id: anc_packagenode.sbom_id, - node_id: anc_packagenode.node_id, - relationship: relationship.to_string(), - purl: anc_packagenode.purl, - name: anc_packagenode.name, - version: anc_packagenode.version, - }; - ancestor_nodes.push(anc_node); - stack.push(succ); - } else { - log::warn!( - "Edge weight not found for edge between {:?} and {:?}", - node, - succ - ); - } - } else { - log::warn!("Edge not found between {:?} and {:?}", node, succ); - } - } else { - log::warn!("Processing ancestors, node value for {:?} not found", succ); - } - } - } - if graph.neighbors_directed(node, Direction::Outgoing).count() == 0 { - continue; // we are at the root - } - } - } - ancestor_nodes -} - -pub async fn get_implicit_relationships( - connection: &C, - distinct_sbom_id: &str, -) -> Result, DbErr> { - let sql = r#" - SELECT - sbom.document_id, - sbom.sbom_id, - sbom.published::text, - get_purl(t1.qualified_purl_id) as purl, - t1_node.node_id, - t1_node.name AS node_name, - t1_version.version AS node_version, - product.name AS product_name, - product_version.version AS product_version - FROM - sbom - LEFT JOIN - product_version ON sbom.sbom_id = product_version.sbom_id - LEFT JOIN - product ON product_version.product_id = product.id - LEFT JOIN - sbom_node t1_node ON sbom.sbom_id = t1_node.sbom_id - LEFT JOIN - package_relates_to_package prtp ON t1_node.node_id = prtp.left_node_id OR t1_node.node_id = prtp.right_node_id - LEFT JOIN - sbom_package_purl_ref t1 ON t1_node.node_id = t1.node_id AND t1.sbom_id = sbom.sbom_id - LEFT JOIN - sbom_package t1_version ON t1_node.node_id = t1_version.node_id AND t1_version.sbom_id = sbom.sbom_id - WHERE - prtp.left_node_id IS NULL AND prtp.right_node_id IS NULL - AND - sbom.sbom_id = $1 - "#; - - let uuid = match Uuid::parse_str(distinct_sbom_id) { - Ok(uuid) => uuid, - Err(_) => return Err(sea_orm::DbErr::Custom("Invalid SBOM ID".to_string())), - }; - let stmt = Statement::from_sql_and_values(DatabaseBackend::Postgres, sql, [uuid.into()]); - let results: Vec = connection.query_all(stmt).await?; - - Ok(results) -} - -pub async fn get_relationships( - connection: &C, - distinct_sbom_id: &str, -) -> Result, DbErr> { - // Retrieve all SBOM components that have defined relationships - let sql = r#" - SELECT - sbom.document_id, - sbom.sbom_id, - sbom.published::text, - t1.node_id AS left_node_id, - get_purl(t1.qualified_purl_id) AS left_qualified_purl, - t1_node.name AS left_node_name, - t1_version.version AS left_node_version, - package_relates_to_package.relationship, - t2.node_id AS right_node_id, - get_purl(t2.qualified_purl_id) AS right_qualified_purl, - t2_node.name AS right_node_name, - t2_version.version AS right_node_version, - product.name AS product_name, - product_version.version AS product_version - FROM - sbom - LEFT JOIN - product_version ON sbom.sbom_id = product_version.sbom_id - LEFT JOIN - product ON product_version.product_id = product.id - LEFT JOIN - package_relates_to_package ON sbom.sbom_id = package_relates_to_package.sbom_id - LEFT JOIN - sbom_package_purl_ref t1 ON sbom.sbom_id = t1.sbom_id AND t1.node_id = package_relates_to_package.left_node_id - LEFT JOIN - sbom_node t1_node ON sbom.sbom_id = t1_node.sbom_id AND t1_node.node_id = package_relates_to_package.left_node_id - LEFT JOIN - sbom_package t1_version ON sbom.sbom_id = t1_version.sbom_id AND t1_version.node_id = package_relates_to_package.left_node_id - LEFT JOIN - sbom_package_purl_ref t2 ON sbom.sbom_id = t2.sbom_id AND t2.node_id = package_relates_to_package.right_node_id - LEFT JOIN - sbom_node t2_node ON sbom.sbom_id = t2_node.sbom_id AND t2_node.node_id = package_relates_to_package.right_node_id - LEFT JOIN - sbom_package t2_version ON sbom.sbom_id = t2_version.sbom_id AND t2_version.node_id = package_relates_to_package.right_node_id - WHERE - package_relates_to_package.relationship IN (0, 1, 8, 13, 14, 15) - AND sbom.sbom_id = $1; - "#; - - let uuid = match Uuid::parse_str(distinct_sbom_id) { - Ok(uuid) => uuid, - Err(_) => return Err(sea_orm::DbErr::Custom("Invalid SBOM ID".to_string())), - }; - let stmt = Statement::from_sql_and_values(DatabaseBackend::Postgres, sql, [uuid.into()]); - let results: Vec = connection.query_all(stmt).await?; - - Ok(results) -} - -pub async fn load_graphs(connection: &C, distinct_sbom_ids: &Vec) { - let graph_map = GraphMap::get_instance(); - { - for distinct_sbom_id in distinct_sbom_ids { - if !graph_map.read().contains_key(distinct_sbom_id) { - // lazy load graphs - let mut g: Graph = Graph::new(); - let mut nodes = HashMap::new(); - - let mut describedby_purl: String = Default::default(); - - // Set relationships explicitly defined in SBOM - match get_relationships(connection, &distinct_sbom_id.to_string()).await { - Ok(results) => { - for row in results { - let ( - sbom_published, - document_id, - product_name, - product_version, - left_node_id, - left_purl_string, - left_node_name, - left_node_version, - right_node_id, - right_purl_string, - right_node_name, - right_node_version, - relationship, - ) = { - let default_value = "NOVALUE".to_string(); // TODO: this eventually will have different defaults. - ( - row.try_get("", "published") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "document_id") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "product_name") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "product_version") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "left_node_id") - .unwrap_or(default_value.clone()), - row.try_get("", "left_qualified_purl") - .unwrap_or(default_value.clone()), - row.try_get("", "left_node_name") - .unwrap_or(default_value.clone()), - row.try_get("", "left_node_version") - .unwrap_or(default_value.clone()), - row.try_get("", "right_node_id") - .unwrap_or(default_value.clone()), - row.try_get("", "right_qualified_purl") - .unwrap_or(default_value.clone()), - row.try_get("", "right_node_name") - .unwrap_or(default_value.clone()), - row.try_get("", "right_node_version") - .unwrap_or(default_value.clone()), - row.try_get("", "relationship") - .unwrap_or(Relationship::ContainedBy), - ) - }; - - if relationship == Relationship::DescribedBy { - // Save for implicit relationships performed later - describedby_purl = left_purl_string.clone(); - } else { - let p1 = match nodes.get(&left_purl_string) { - Some(&node_index) => node_index, // already exists - None => { - let new_node = PackageNode { - sbom_id: distinct_sbom_id.clone(), - node_id: left_node_id.clone(), - purl: left_purl_string.clone(), - name: left_node_name.clone(), - version: left_node_version.clone(), - published: sbom_published.clone(), - document_id: document_id.clone(), - product_name: product_name.clone(), - product_version: product_version.clone(), - }; - let i = g.add_node(new_node); - nodes.insert(left_purl_string.clone(), i); - i - } - }; - - let p2 = match nodes.get(&right_purl_string) { - Some(&node_index) => node_index, // already exists - None => { - let new_node = PackageNode { - sbom_id: distinct_sbom_id.clone(), - node_id: right_node_id.clone(), - purl: right_purl_string.clone(), - name: right_node_name.clone(), - version: right_node_version.clone(), - published: sbom_published.clone(), - document_id: document_id.clone(), - product_name: product_name.clone(), - product_version: product_version.clone(), - }; - let i = g.add_node(new_node); - nodes.insert(right_purl_string.clone(), i); - i - } - }; - - g.add_edge(p1, p2, relationship); - } - } - } - Err(err) => { - log::error!("Error fetching graph relationships: {}", err); - } - } - - // Set relationships implicitly defined in SBOM - match get_implicit_relationships(connection, &distinct_sbom_id.to_string()).await { - Ok(results) => { - for row in results { - let ( - sbom_published, - document_id, - product_name, - product_version, - node_id, - purl, - node_name, - node_version, - ) = { - let default_value = "NOVALUE".to_string(); // TODO: this eventually will have different defaults. - ( - row.try_get("", "published") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "document_id") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "product_name") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "product_version") - .unwrap_or_else(|_| default_value.clone()), - row.try_get("", "node_id").unwrap_or(default_value.clone()), - row.try_get("", "purl").unwrap_or(default_value.clone()), - row.try_get("", "node_name") - .unwrap_or(default_value.clone()), - row.try_get("", "node_version") - .unwrap_or(default_value.clone()), - ) - }; - - let p1 = match nodes.get(&purl) { - Some(&node_index) => node_index, // already exists - None => { - let new_node = PackageNode { - sbom_id: distinct_sbom_id.clone(), - node_id: node_id.clone(), - purl: purl.clone(), - name: node_name.clone(), - version: node_version.clone(), - published: sbom_published.clone(), - document_id: document_id.clone(), - product_name: product_name.clone(), - product_version: product_version.clone(), - }; - let i = g.add_node(new_node); - nodes.insert(purl.clone(), i); - i - } - }; - if let Some(describedby_node_index) = nodes.get(&describedby_purl) { - g.add_edge(p1, *describedby_node_index, Relationship::Undefined); - } else { - log::warn!("No 'describes' relationship found in {} SBOM, no implicit relationship set.", distinct_sbom_id); - } - } - } - Err(err) => { - log::error!("Error fetching graph relationships: {}", err); - } - } - - graph_map.write().insert(distinct_sbom_id.to_string(), g); - } - } - } -} - -impl AnalysisService { - pub fn new() -> Self { - let _ = GraphMap::get_instance(); - Self {} - } - - pub async fn load_graphs( - &self, - distinct_sbom_ids: Vec, - connection: &C, - ) -> Result<(), Error> { - load_graphs(connection, &distinct_sbom_ids).await; - - Ok(()) - } - pub async fn load_all_graphs(&self, connection: &C) -> Result<(), Error> { - // retrieve all sboms in trustify - let distinct_sbom_ids = sbom::Entity::find() - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - Ok(()) - } - - pub fn clear_all_graphs(&self) -> Result<(), Error> { - let graph_manager = GraphMap::get_instance(); - let mut manager = graph_manager.write(); - manager.clear(); - Ok(()) - } - - pub async fn status( - &self, - connection: &C, - ) -> Result { - let distinct_sbom_ids = sbom::Entity::find() - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await?; - - let graph_manager = GraphMap::get_instance(); - let manager = graph_manager.read(); - Ok(AnalysisStatus { - sbom_count: distinct_sbom_ids.len() as i32, - graph_count: manager.len() as i32, - }) - } - - pub fn query_ancestor_graph( - component_name: Option, - component_purl: Option, - query: Option, - distinct_sbom_ids: Vec, - ) -> Vec { - let mut components = Vec::new(); - let graph_manager = GraphMap::get_instance(); - { - // RwLock for reading hashmap - let graph_read_guard = graph_manager.read(); - for distinct_sbom_id in &distinct_sbom_ids { - if let Some(graph) = graph_read_guard.get(distinct_sbom_id.to_string().as_str()) { - if is_cyclic_directed(graph) { - log::warn!( - "analysis graph of sbom {} has circular references!", - distinct_sbom_id - ); - } - - let mut visited = HashSet::new(); - - // Iterate over matching node indices and process them directly - graph - .node_indices() - .filter(|&i| { - if let Some(component_name) = &component_name { - graph - .node_weight(i) - .map(|node| node.name.eq(component_name)) - .unwrap_or(false) - } else if let Some(component_purl) = component_purl.clone() { - if let Some(node) = graph.node_weight(i) { - match Purl::from_str(&node.purl).map_err(Error::Purl) { - Ok(purl) => purl == component_purl, - Err(err) => { - log::warn!( - "Error retrieving purl from analysis graph {}", - err - ); - false - } - } - } else { - false // Return false if the node does not exist - } - } else if let Some(query) = &query { - graph - .node_weight(i) - .map(|node| { - query.apply(&HashMap::from([ - ("sbom_id", Value::String(&node.sbom_id)), - ("node_id", Value::String(&node.node_id)), - ("name", Value::String(&node.name)), - ("version", Value::String(&node.version)), - ])) - }) - .unwrap_or(false) - } else { - false - } - }) - .for_each(|node_index| { - if !visited.contains(&node_index) { - visited.insert(node_index); - - if let Some(find_match_package_node) = graph.node_weight(node_index) - { - log::debug!("matched!"); - components.push(AncestorSummary { - sbom_id: find_match_package_node.sbom_id.to_string(), - node_id: find_match_package_node.node_id.to_string(), - purl: find_match_package_node.purl.to_string(), - name: find_match_package_node.name.to_string(), - version: find_match_package_node.version.to_string(), - published: find_match_package_node.published.to_string(), - document_id: find_match_package_node - .document_id - .to_string(), - product_name: find_match_package_node - .product_name - .to_string(), - product_version: find_match_package_node - .product_version - .to_string(), - ancestors: ancestor_nodes(graph, node_index), - }); - } - } - }); - } - } - } - - components - } - - pub async fn query_deps_graph( - component_name: Option, - component_purl: Option, - query: Option, - distinct_sbom_ids: Vec, - ) -> Vec { - let mut components = Vec::new(); - let graph_manager = GraphMap::get_instance(); - { - // RwLock for reading hashmap - let graph_read_guard = graph_manager.read(); - for distinct_sbom_id in &distinct_sbom_ids { - if let Some(graph) = graph_read_guard.get(distinct_sbom_id.to_string().as_str()) { - if is_cyclic_directed(graph) { - log::warn!( - "analysis graph of sbom {} has circular references!", - distinct_sbom_id - ); - } - - let mut visited = HashSet::new(); - - // Iterate over matching node indices and process them directly - graph - .node_indices() - .filter(|&i| { - if let Some(component_name) = &component_name { - graph - .node_weight(i) - .map(|node| node.name.eq(component_name)) - .unwrap_or(false) - } else if let Some(component_purl) = component_purl.clone() { - if let Some(node) = graph.node_weight(i) { - match Purl::from_str(&node.purl).map_err(Error::Purl) { - Ok(purl) => purl == component_purl, - Err(err) => { - log::warn!( - "Error retrieving purl from analysis graph {}", - err - ); - false - } - } - } else { - false // Return false if the node does not exist - } - } else if let Some(query) = &query { - graph - .node_weight(i) - .map(|node| { - query.apply(&HashMap::from([ - ("sbom_id", Value::String(&node.sbom_id)), - ("node_id", Value::String(&node.node_id)), - ("name", Value::String(&node.name)), - ("version", Value::String(&node.version)), - ])) - }) - .unwrap_or(false) - } else { - false - } - }) - .for_each(|node_index| { - if !visited.contains(&node_index) { - visited.insert(node_index); - - if let Some(find_match_package_node) = graph.node_weight(node_index) - { - log::debug!("matched!"); - components.push(DepSummary { - sbom_id: find_match_package_node.sbom_id.to_string(), - node_id: find_match_package_node.node_id.to_string(), - purl: find_match_package_node.purl.to_string(), - name: find_match_package_node.name.to_string(), - version: find_match_package_node.version.to_string(), - published: find_match_package_node.published.to_string(), - document_id: find_match_package_node - .document_id - .to_string(), - product_name: find_match_package_node - .product_name - .to_string(), - product_version: find_match_package_node - .product_version - .to_string(), - deps: dep_nodes(graph, node_index, &mut HashSet::new()), - }); - } - } - }); - } - } - } - - components - } - - #[instrument(skip(self, connection), err)] - pub async fn retrieve_root_components( - &self, - query: Query, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { - let search_sbom_node_name_subquery = sbom_node::Entity::find() - .filtering(query.clone())? - .select_only() - .column(sbom_node::Column::SbomId) - .distinct() - .into_query(); - let distinct_sbom_ids: Vec = sbom::Entity::find() - .filter(sbom::Column::SbomId.in_subquery(search_sbom_node_name_subquery)) - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - let components = AnalysisService::query_ancestor_graph( - None, - None, - Option::from(query), - distinct_sbom_ids, - ); - - Ok(paginated.paginate_array(&components)) - } - - pub async fn retrieve_all_sbom_roots_by_name( - &self, - sbom_id: Uuid, - component_name: String, - connection: &C, - ) -> Result, Error> { - // This function searches for a component(s) by name in a specific sbom, then returns that components - // root components. - - let distinct_sbom_ids = vec![sbom_id.to_string()]; - load_graphs(connection, &distinct_sbom_ids).await; - - let components = AnalysisService::query_ancestor_graph( - Option::from(component_name), - None, - None, - distinct_sbom_ids, - ); - - let mut root_components = Vec::new(); - for component in components { - if let Some(last_ancestor) = component.ancestors.last() { - if !root_components.contains(last_ancestor) { - // we want distinct list - root_components.push(last_ancestor.clone()); - } - } - } - - Ok(root_components) - } - - #[instrument(skip(self, connection), err)] - pub async fn retrieve_root_components_by_name( - &self, - component_name: String, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { - let search_sbom_node_exact_name_subquery = sbom_node::Entity::find() - .filter(sbom_node::Column::Name.eq(component_name.as_str())) - .select_only() - .column(sbom_node::Column::SbomId) - .distinct() - .into_query(); - let distinct_sbom_ids: Vec = sbom::Entity::find() - .filter(sbom::Column::SbomId.in_subquery(search_sbom_node_exact_name_subquery)) - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - let components = AnalysisService::query_ancestor_graph( - Option::from(component_name), - None, - None, - distinct_sbom_ids, - ); - - Ok(paginated.paginate_array(&components)) - } - - #[instrument(skip(self, connection), err)] - pub async fn retrieve_root_components_by_purl( - &self, - component_purl: Purl, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { - let search_sbom_node_exact_name_subquery = sbom_node::Entity::find() - .filter(sbom_node::Column::Name.eq(component_purl.name.as_str())) - .select_only() - .column(sbom_node::Column::SbomId) - .distinct() - .into_query(); - let distinct_sbom_ids: Vec = sbom::Entity::find() - .filter(sbom::Column::SbomId.in_subquery(search_sbom_node_exact_name_subquery)) - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - let components = AnalysisService::query_ancestor_graph( - None, - Option::from(component_purl), - None, - distinct_sbom_ids, - ); - - Ok(paginated.paginate_array(&components)) - } - - #[instrument(skip(self, connection), err)] - pub async fn retrieve_deps( - &self, - query: Query, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { - let search_sbom_node_name_subquery = sbom_node::Entity::find() - .filtering(query.clone())? - .select_only() - .column(sbom_node::Column::SbomId) - .distinct() - .into_query(); - let distinct_sbom_ids: Vec = sbom::Entity::find() - .filter(sbom::Column::SbomId.in_subquery(search_sbom_node_name_subquery)) - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - let components = - AnalysisService::query_deps_graph(None, None, Option::from(query), distinct_sbom_ids) - .await; - - Ok(paginated.paginate_array(&components)) - } - - pub async fn retrieve_deps_by_name( - &self, - component_name: String, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { - let search_sbom_node_exact_name_subquery = sbom_node::Entity::find() - .filter(sbom_node::Column::Name.eq(component_name.as_str())) - .select_only() - .column(sbom_node::Column::SbomId) - .distinct() - .into_query(); - let distinct_sbom_ids: Vec = sbom::Entity::find() - .filter(sbom::Column::SbomId.in_subquery(search_sbom_node_exact_name_subquery)) - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - let components = AnalysisService::query_deps_graph( - Option::from(component_name), - None, - None, - distinct_sbom_ids, - ) - .await; - - Ok(paginated.paginate_array(&components)) - } - - pub async fn retrieve_deps_by_purl( - &self, - component_purl: Purl, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { - let search_sbom_node_exact_name_subquery = sbom_node::Entity::find() - .filter(sbom_node::Column::Name.eq(component_purl.name.as_str())) - .select_only() - .column(sbom_node::Column::SbomId) - .distinct() - .into_query(); - let distinct_sbom_ids: Vec = sbom::Entity::find() - .filter(sbom::Column::SbomId.in_subquery(search_sbom_node_exact_name_subquery)) - .select() - .order_by(sbom::Column::DocumentId, Order::Asc) - .order_by(sbom::Column::Published, Order::Desc) - .all(connection) - .await? - .into_iter() - .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String - .collect(); - - load_graphs(connection, &distinct_sbom_ids).await; - - let components = AnalysisService::query_deps_graph( - None, - Option::from(component_purl), - None, - distinct_sbom_ids, - ) - .await; - - Ok(paginated.paginate_array(&components)) - } -} - -#[cfg(test)] -mod test { - use super::*; - - use test_context::test_context; - use test_log::test; - use trustify_common::model::Paginated; - use trustify_test_context::TrustifyContext; - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_analysis_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json", "spdx/simple.json"]) - .await?; //double ingestion intended - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_root_components(Query::q("DD"), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - "pkg:rpm/redhat/AA@0.0.0?arch=src".to_string() - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-AA".to_string() - ); - assert_eq!(analysis_graph.total, 1); - - // ensure we set implicit relationship on component with no defined relationships - let analysis_graph = service - .retrieve_root_components(Query::q("EE"), Paginated::default(), &ctx.db) - .await - .unwrap(); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_analysis_cyclonedx_service( - ctx: &TrustifyContext, - ) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["cyclonedx/simple.json", "cyclonedx/simple.json"]) - .await?; //double ingestion intended - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_root_components(Query::q("DD"), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - "pkg:rpm/redhat/AA@0.0.0?arch=src".to_string() - ); - let node = analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap(); - assert_eq!(node.node_id, "aa".to_string()); - assert_eq!(node.name, "AA".to_string()); - assert_eq!(analysis_graph.total, 1); - - // ensure we set implicit relationship on component with no defined relationships - let analysis_graph = service - .retrieve_root_components(Query::q("EE"), Paginated::default(), &ctx.db) - .await - .unwrap(); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_by_name_analysis_service( - ctx: &TrustifyContext, - ) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json"]).await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_root_components_by_name("B".to_string(), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - "pkg:rpm/redhat/A@0.0.0?arch=src".to_string() - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-A".to_string() - ); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_by_purl_analysis_service( - ctx: &TrustifyContext, - ) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json"]).await?; - - let service = AnalysisService::new(); - - let component_purl: Purl = Purl::from_str("pkg:rpm/redhat/B@0.0.0").map_err(Error::Purl)?; - - let analysis_graph = service - .retrieve_root_components_by_purl(component_purl, Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - "pkg:rpm/redhat/A@0.0.0?arch=src".to_string() - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-A".to_string() - ); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_quarkus_analysis_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents([ - "spdx/quarkus-bom-3.2.11.Final-redhat-00001.json", - "spdx/quarkus-bom-3.2.12.Final-redhat-00002.json", - ]) - .await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_root_components(Query::q("spymemcached"), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!(analysis_graph.items.last().unwrap().ancestors.last().unwrap().purl, - "pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.12.Final-redhat-00002?type=pom&repository_url=https%3a%2f%2fmaven.repository.redhat.com%2fga%2f".to_string() - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-e24fec28-1001-499c-827f-2e2e5f2671b5".to_string() - ); - - assert_eq!(analysis_graph.total, 2); - Ok(()) - } - - // TODO: this test passes when run individually. - #[test_context(TrustifyContext)] - #[test(tokio::test)] - #[ignore] - async fn test_status_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json"]).await?; - - let service = AnalysisService::new(); - let _load_all_graphs = service.load_all_graphs(&ctx.db).await; - let analysis_status = service.status(&ctx.db).await.unwrap(); - - assert_eq!(analysis_status.sbom_count, 1); - assert_eq!(analysis_status.graph_count, 1); - - let _clear_all_graphs = service.clear_all_graphs(); - - ctx.ingest_documents([ - "spdx/quarkus-bom-3.2.11.Final-redhat-00001.json", - "spdx/quarkus-bom-3.2.12.Final-redhat-00002.json", - ]) - .await?; - - let analysis_status = service.status(&ctx.db).await.unwrap(); - - assert_eq!(analysis_status.sbom_count, 3); - assert_eq!(analysis_status.graph_count, 0); - - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json"]).await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_deps(Query::q("AA"), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!(analysis_graph.total, 1); - - // ensure we set implicit relationship on component with no defined relationships - let analysis_graph = service - .retrieve_root_components(Query::q("EE"), Paginated::default(), &ctx.db) - .await - .unwrap(); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_deps_cyclonedx_service( - ctx: &TrustifyContext, - ) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["cyclonedx/simple.json"]).await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_deps(Query::q("AA"), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!(analysis_graph.total, 1); - - // ensure we set implicit relationship on component with no defined relationships - let analysis_graph = service - .retrieve_root_components(Query::q("EE"), Paginated::default(), &ctx.db) - .await - .unwrap(); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_by_name_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json"]).await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_deps_by_name("A".to_string(), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!( - analysis_graph.items[0].purl, - "pkg:rpm/redhat/A@0.0.0?arch=src".to_string() - ); - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_simple_by_purl_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/simple.json"]).await?; - - let service = AnalysisService::new(); - - let component_purl: Purl = - Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src").map_err(Error::Purl)?; - - let analysis_graph = service - .retrieve_deps_by_purl(component_purl, Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!( - analysis_graph.items[0].purl, - "pkg:rpm/redhat/AA@0.0.0?arch=src".to_string() - ); - - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_quarkus_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents([ - "spdx/quarkus-bom-3.2.11.Final-redhat-00001.json", - "spdx/quarkus-bom-3.2.12.Final-redhat-00002.json", - ]) - .await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_deps(Query::q("spymemcached"), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!(analysis_graph.total, 2); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_circular_deps_cyclonedx_service( - ctx: &TrustifyContext, - ) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["cyclonedx/cyclonedx-circular.json"]) - .await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_deps_by_name("junit-bom".to_string(), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_circular_deps_spdx_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/loop.json"]).await?; - - let service = AnalysisService::new(); - - let analysis_graph = service - .retrieve_deps_by_name("A".to_string(), Paginated::default(), &ctx.db) - .await - .unwrap(); - - assert_eq!(analysis_graph.total, 1); - Ok(()) - } - - #[test_context(TrustifyContext)] - #[test(tokio::test)] - async fn test_retrieve_all_sbom_roots_by_name1( - ctx: &TrustifyContext, - ) -> Result<(), anyhow::Error> { - ctx.ingest_documents(["spdx/quarkus-bom-3.2.11.Final-redhat-00001.json"]) - .await?; - - let service = AnalysisService::new(); - let component_name = "quarkus-vertx-http".to_string(); - - let analysis_graph = service - .retrieve_root_components(Query::q(&component_name), Paginated::default(), &ctx.db) - .await?; - - let sbom_id = analysis_graph - .items - .last() - .unwrap() - .sbom_id - .parse::()?; - - let roots = service - .retrieve_all_sbom_roots_by_name(sbom_id, component_name, &ctx.db) - .await?; - - assert_eq!(roots.last().unwrap().name, "quarkus-bom"); - - Ok(()) - } -} diff --git a/modules/analysis/src/service/load.rs b/modules/analysis/src/service/load.rs new file mode 100644 index 000000000..631ae5e37 --- /dev/null +++ b/modules/analysis/src/service/load.rs @@ -0,0 +1,380 @@ +use crate::{model::PackageNode, service::AnalysisService}; +use petgraph::Graph; +use sea_orm::{ConnectionTrait, DatabaseBackend, DbErr, QueryResult, Statement}; +use std::collections::HashMap; +use trustify_common::{cpe::Cpe, purl::Purl}; +use trustify_entity::{cpe::CpeDto, relationship::Relationship}; +use uuid::Uuid; + +pub async fn get_implicit_relationships( + connection: &C, + distinct_sbom_id: &str, +) -> Result, DbErr> { + let sql = r#" + SELECT + sbom.document_id, + sbom.sbom_id, + sbom.published::text, + array_agg(get_purl(t1.qualified_purl_id)) as purl, + array_agg(row_to_json(t2_cpe)) AS cpe, + t1_node.node_id AS node_id, + t1_node.name AS node_name, + t1_version.version AS node_version, + product.name AS product_name, + product_version.version AS product_version + FROM + sbom + LEFT JOIN + product_version ON sbom.sbom_id = product_version.sbom_id + LEFT JOIN + product ON product_version.product_id = product.id + LEFT JOIN + sbom_node t1_node ON sbom.sbom_id = t1_node.sbom_id + LEFT JOIN + package_relates_to_package prtp ON t1_node.node_id = prtp.left_node_id OR t1_node.node_id = prtp.right_node_id + LEFT JOIN + sbom_package_purl_ref t1 ON t1_node.node_id = t1.node_id AND t1.sbom_id = sbom.sbom_id + LEFT JOIN + sbom_package_cpe_ref t2 ON t1_node.node_id = t2.node_id AND t2.sbom_id = sbom.sbom_id + LEFT JOIN + cpe t2_cpe ON t2.cpe_id = t2_cpe.id + LEFT JOIN + sbom_package t1_version ON t1_node.node_id = t1_version.node_id AND t1_version.sbom_id = sbom.sbom_id + WHERE + prtp.left_node_id IS NULL AND prtp.right_node_id IS NULL + AND + sbom.sbom_id = $1 + GROUP BY + sbom.document_id, + sbom.sbom_id, + sbom.published, + t1_node.node_id, + t1_node.name, + t1_version.version, + product.name, + product_version.version + "#; + + let uuid = match Uuid::parse_str(distinct_sbom_id) { + Ok(uuid) => uuid, + Err(_) => return Err(DbErr::Custom("Invalid SBOM ID".to_string())), + }; + let stmt = Statement::from_sql_and_values(DatabaseBackend::Postgres, sql, [uuid.into()]); + let results: Vec = connection.query_all(stmt).await?; + + Ok(results) +} + +pub async fn get_relationships( + connection: &C, + distinct_sbom_id: &str, +) -> Result, DbErr> { + // Retrieve all SBOM components that have defined relationships + let sql = r#" + SELECT + sbom.document_id, + sbom.sbom_id, + sbom.published::text, + package_relates_to_package.left_node_id AS left_node_id, + array_agg(get_purl(t1.qualified_purl_id)) AS left_qualified_purl, + array_agg(row_to_json(t3_cpe)) AS left_cpe, + t1_node.name AS left_node_name, + t1_version.version AS left_node_version, + package_relates_to_package.relationship, + package_relates_to_package.right_node_id AS right_node_id, + array_agg(get_purl(t2.qualified_purl_id)) AS right_qualified_purl, + array_agg(row_to_json(t4_cpe)) AS right_cpe, + t2_node.name AS right_node_name, + t2_version.version AS right_node_version, + product.name AS product_name, + product_version.version AS product_version + FROM + sbom + LEFT JOIN + product_version ON sbom.sbom_id = product_version.sbom_id + LEFT JOIN + product ON product_version.product_id = product.id + LEFT JOIN + package_relates_to_package ON sbom.sbom_id = package_relates_to_package.sbom_id + LEFT JOIN + sbom_package_purl_ref t1 ON sbom.sbom_id = t1.sbom_id AND t1.node_id = package_relates_to_package.left_node_id + LEFT JOIN + sbom_package_cpe_ref t3 ON sbom.sbom_id = t3.sbom_id AND t3.node_id = package_relates_to_package.left_node_id + LEFT JOIN + sbom_node t1_node ON sbom.sbom_id = t1_node.sbom_id AND t1_node.node_id = package_relates_to_package.left_node_id + LEFT JOIN + sbom_package t1_version ON sbom.sbom_id = t1_version.sbom_id AND t1_version.node_id = package_relates_to_package.left_node_id + LEFT JOIN + sbom_package_purl_ref t2 ON sbom.sbom_id = t2.sbom_id AND t2.node_id = package_relates_to_package.right_node_id + LEFT JOIN + sbom_package_cpe_ref t4 ON sbom.sbom_id = t4.sbom_id AND t4.node_id = package_relates_to_package.right_node_id + LEFT JOIN + sbom_node t2_node ON sbom.sbom_id = t2_node.sbom_id AND t2_node.node_id = package_relates_to_package.right_node_id + LEFT JOIN + sbom_package t2_version ON sbom.sbom_id = t2_version.sbom_id AND t2_version.node_id = package_relates_to_package.right_node_id + LEFT JOIN + cpe t3_cpe ON t3.cpe_id = t3_cpe.id + LEFT JOIN + cpe t4_cpe ON t4.cpe_id = t4_cpe.id + WHERE + package_relates_to_package.relationship IN (0, 1, 8, 13, 14, 15) + AND + sbom.sbom_id = $1 + GROUP BY + sbom.document_id, + sbom.sbom_id, + sbom.published, + package_relates_to_package.left_node_id, + t1_node.name, + t1_version.version, + package_relates_to_package.relationship, + package_relates_to_package.right_node_id, + t2_node.name, + t2_version.version, + product.name, + product_version.version +"#; + + let uuid = match Uuid::parse_str(distinct_sbom_id) { + Ok(uuid) => uuid, + Err(_) => return Err(DbErr::Custom("Invalid SBOM ID".to_string())), + }; + let stmt = Statement::from_sql_and_values(DatabaseBackend::Postgres, sql, [uuid.into()]); + let results: Vec = connection.query_all(stmt).await?; + + Ok(results) +} + +fn to_purls(purls: Vec) -> Vec { + purls + .into_iter() + .filter_map(|purl| Purl::try_from(purl).ok()) + .collect() +} + +fn to_cpes(cpes: Vec) -> Vec { + cpes.into_iter() + .flat_map(|cpe| { + serde_json::from_value::(cpe) + .ok() + .and_then(|cpe| Cpe::try_from(cpe).ok()) + }) + .collect() +} + +impl AnalysisService { + pub async fn load_graphs( + &self, + connection: &C, + distinct_sbom_ids: &Vec, + ) -> Result<(), DbErr> { + for distinct_sbom_id in distinct_sbom_ids { + if !self.graph.read().contains_key(distinct_sbom_id) { + // lazy load graphs + let mut g: Graph = Graph::new(); + let mut nodes = HashMap::new(); + + let mut describedby_node_id: Option = Default::default(); + + // Set relationships explicitly defined in SBOM + match get_relationships(connection, &distinct_sbom_id.to_string()).await { + Ok(results) => { + for row in results { + let ( + sbom_published, + document_id, + product_name, + product_version, + left_node_id, + left_purl_string, + left_cpe_json, + left_node_name, + left_node_version, + right_node_id, + right_purl_string, + right_cpe_json, + right_node_name, + right_node_version, + relationship, + ) = { + let default_value = "NOVALUE".to_string(); // TODO: this eventually will have different defaults. + ( + row.try_get("", "published") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "document_id") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "product_name") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "product_version") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "left_node_id") + .unwrap_or(default_value.clone()), + row.try_get::>("", "left_qualified_purl") + .unwrap_or_default(), + row.try_get("", "left_cpe") + .ok() + .unwrap_or_else(Vec::::new), + row.try_get("", "left_node_name") + .unwrap_or(default_value.clone()), + row.try_get("", "left_node_version") + .unwrap_or(default_value.clone()), + row.try_get("", "right_node_id") + .unwrap_or(default_value.clone()), + row.try_get::>("", "right_qualified_purl") + .unwrap_or_default(), + row.try_get("", "right_cpe") + .ok() + .unwrap_or_else(Vec::::new), + row.try_get("", "right_node_name") + .unwrap_or(default_value.clone()), + row.try_get("", "right_node_version") + .unwrap_or(default_value.clone()), + row.try_get("", "relationship") + .unwrap_or(Relationship::ContainedBy), + ) + }; + + /*log::info!( + "Row - left_node: {left_node_id:?}, right_node: {right_node_id:?}", + ); + log::info!( + "Row - left_cpe: {left_cpe_json:?}, right_cpe: {right_cpe_json:?}", + ); + log::info!( + "Row - left_purl: {left_purl_string:?}, right_purl: {right_purl_string:?}", + );*/ + + if relationship == Relationship::DescribedBy { + // Save for implicit relationships performed later + describedby_node_id = Some(left_node_id); + } else { + let p1 = match nodes.get(&left_node_id) { + Some(&node_index) => node_index, // already exists + None => { + let new_node = PackageNode { + sbom_id: distinct_sbom_id.clone(), + node_id: left_node_id.clone(), + purl: to_purls(left_purl_string.clone()), + cpe: to_cpes(left_cpe_json), + name: left_node_name.clone(), + version: left_node_version.clone(), + published: sbom_published.clone(), + document_id: document_id.clone(), + product_name: product_name.clone(), + product_version: product_version.clone(), + }; + let i = g.add_node(new_node); + nodes.insert(left_node_id.clone(), i); + i + } + }; + + let p2 = match nodes.get(&right_node_id) { + Some(&node_index) => node_index, // already exists + None => { + let new_node = PackageNode { + sbom_id: distinct_sbom_id.clone(), + node_id: right_node_id.clone(), + purl: to_purls(right_purl_string.clone()), + cpe: to_cpes(right_cpe_json), + name: right_node_name.clone(), + version: right_node_version.clone(), + published: sbom_published.clone(), + document_id: document_id.clone(), + product_name: product_name.clone(), + product_version: product_version.clone(), + }; + let i = g.add_node(new_node); + nodes.insert(right_node_id.clone(), i); + i + } + }; + + g.add_edge(p1, p2, relationship); + } + } + } + Err(err) => { + log::error!("Error fetching graph relationships: {}", err); + } + } + + // Set relationships implicitly defined in SBOM + match get_implicit_relationships(connection, &distinct_sbom_id.to_string()).await { + Ok(results) => { + for row in results { + let ( + sbom_published, + document_id, + product_name, + product_version, + node_id, + purl, + cpe, + node_name, + node_version, + ) = { + let default_value = "NOVALUE".to_string(); // TODO: this eventually will have different defaults. + ( + row.try_get("", "published") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "document_id") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "product_name") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "product_version") + .unwrap_or_else(|_| default_value.clone()), + row.try_get("", "node_id").unwrap_or(default_value.clone()), + row.try_get::>("", "purl").unwrap_or_default(), + row.try_get("", "cpe") + .ok() + .unwrap_or_else(Vec::::new), + row.try_get("", "node_name") + .unwrap_or(default_value.clone()), + row.try_get("", "node_version") + .unwrap_or(default_value.clone()), + ) + }; + + let p1 = match nodes.get(&node_id) { + Some(&node_index) => node_index, // already exists + None => { + let new_node = PackageNode { + sbom_id: distinct_sbom_id.clone(), + node_id: node_id.clone(), + purl: to_purls(purl), + cpe: to_cpes(cpe), + name: node_name.clone(), + version: node_version.clone(), + published: sbom_published.clone(), + document_id: document_id.clone(), + product_name: product_name.clone(), + product_version: product_version.clone(), + }; + let i = g.add_node(new_node); + nodes.insert(node_id.clone(), i); + i + } + }; + + if let Some(describedby_node_index) = + describedby_node_id.as_ref().and_then(|id| nodes.get(id)) + { + g.add_edge(p1, *describedby_node_index, Relationship::Undefined); + } else { + log::warn!("No 'describes' relationship found in {} SBOM, no implicit relationship set.", distinct_sbom_id); + } + } + } + Err(err) => { + log::error!("Error fetching graph relationships: {}", err); + } + } + + self.graph.write().insert(distinct_sbom_id.to_string(), g); + } + } + + Ok(()) + } +} diff --git a/modules/analysis/src/service/mod.rs b/modules/analysis/src/service/mod.rs new file mode 100644 index 000000000..0b8ed8ccc --- /dev/null +++ b/modules/analysis/src/service/mod.rs @@ -0,0 +1,494 @@ +mod load; +mod query; + +#[cfg(test)] +mod test; + +use crate::{ + model::{AnalysisStatus, AncNode, AncestorSummary, DepNode, DepSummary, GraphMap, PackageNode}, + Error, +}; +use parking_lot::RwLock; +use petgraph::{ + algo::is_cyclic_directed, + graph::{Graph, NodeIndex}, + visit::{NodeIndexable, VisitMap, Visitable}, + Direction, +}; +use query::*; +use sea_orm::{ + prelude::ConnectionTrait, ColumnTrait, EntityOrSelect, EntityTrait, QueryFilter, QueryOrder, + QuerySelect, QueryTrait, RelationTrait, +}; +use sea_query::{JoinType, Order, SelectStatement}; +use std::collections::{HashMap, HashSet}; +use std::fmt::Debug; +use std::sync::Arc; +use tracing::instrument; +use trustify_common::{ + db::query::{Filtering, Value}, + model::{Paginated, PaginatedResults}, +}; +use trustify_entity::{ + relationship::Relationship, sbom, sbom_node, sbom_package, sbom_package_cpe_ref, + sbom_package_purl_ref, +}; +use uuid::Uuid; + +#[derive(Clone, Default)] +pub struct AnalysisService { + graph: Arc>, +} + +pub fn dep_nodes( + graph: &Graph, + node: NodeIndex, + visited: &mut HashSet, +) -> Vec { + let mut depnodes = Vec::new(); + fn dfs( + graph: &Graph, + node: NodeIndex, + depnodes: &mut Vec, + visited: &mut HashSet, + ) { + if visited.contains(&node) { + return; + } + visited.insert(node); + for neighbor in graph.neighbors_directed(node, Direction::Incoming) { + if let Some(dep_packagenode) = graph.node_weight(neighbor).cloned() { + // Attempt to find the edge and get the relationship in a more elegant way + if let Some(relationship) = graph + .find_edge(neighbor, node) + .and_then(|edge_index| graph.edge_weight(edge_index)) + { + let dep_node = DepNode { + sbom_id: dep_packagenode.sbom_id, + node_id: dep_packagenode.node_id, + relationship: relationship.to_string(), + purl: dep_packagenode.purl.clone(), + cpe: dep_packagenode.cpe.clone(), + name: dep_packagenode.name.to_string(), + version: dep_packagenode.version.to_string(), + deps: dep_nodes(graph, neighbor, visited), + }; + depnodes.push(dep_node); + dfs(graph, neighbor, depnodes, visited); + } + } else { + log::warn!( + "Processing descendants node weight for neighbor {:?} not found", + neighbor + ); + } + } + } + + dfs(graph, node, &mut depnodes, visited); + + depnodes +} + +pub fn ancestor_nodes( + graph: &Graph, + node: NodeIndex, +) -> Vec { + let mut discovered = graph.visit_map(); + let mut ancestor_nodes = Vec::new(); + let mut stack = Vec::new(); + + stack.push(graph.from_index(node.index())); + + while let Some(node) = stack.pop() { + if discovered.visit(node) { + for succ in graph.neighbors_directed(node, Direction::Outgoing) { + if !discovered.is_visited(&succ) { + if let Some(anc_packagenode) = graph.node_weight(succ).cloned() { + if let Some(edge) = graph.find_edge(node, succ) { + if let Some(relationship) = graph.edge_weight(edge) { + let anc_node = AncNode { + sbom_id: anc_packagenode.sbom_id, + node_id: anc_packagenode.node_id, + relationship: relationship.to_string(), + purl: anc_packagenode.purl, + cpe: anc_packagenode.cpe, + name: anc_packagenode.name, + version: anc_packagenode.version, + }; + ancestor_nodes.push(anc_node); + stack.push(succ); + } else { + log::warn!( + "Edge weight not found for edge between {:?} and {:?}", + node, + succ + ); + } + } else { + log::warn!("Edge not found between {:?} and {:?}", node, succ); + } + } else { + log::warn!("Processing ancestors, node value for {:?} not found", succ); + } + } + } + if graph.neighbors_directed(node, Direction::Outgoing).count() == 0 { + continue; // we are at the root + } + } + } + ancestor_nodes +} + +impl AnalysisService { + pub fn new() -> Self { + Self::default() + } + + pub async fn load_all_graphs(&self, connection: &C) -> Result<(), Error> { + // retrieve all sboms in trustify + + let distinct_sbom_ids = sbom::Entity::find() + .select() + .order_by(sbom::Column::DocumentId, Order::Asc) + .order_by(sbom::Column::Published, Order::Desc) + .all(connection) + .await? + .into_iter() + .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String + .collect(); + + self.load_graphs(connection, &distinct_sbom_ids).await?; + + Ok(()) + } + + pub fn clear_all_graphs(&self) -> Result<(), Error> { + let mut manager = self.graph.write(); + manager.clear(); + Ok(()) + } + + pub async fn status( + &self, + connection: &C, + ) -> Result { + let distinct_sbom_ids = sbom::Entity::find() + .select() + .order_by(sbom::Column::DocumentId, Order::Asc) + .order_by(sbom::Column::Published, Order::Desc) + .all(connection) + .await?; + + let manager = self.graph.read(); + Ok(AnalysisStatus { + sbom_count: distinct_sbom_ids.len() as u32, + graph_count: manager.len() as u32, + }) + } + + pub fn query_ancestor_graph<'a>( + &self, + query: impl Into>, + distinct_sbom_ids: Vec, + ) -> Vec { + let query = query.into(); + let mut components = Vec::new(); + + // RwLock for reading hashmap + let graph_read_guard = self.graph.read(); + for distinct_sbom_id in &distinct_sbom_ids { + if let Some(graph) = graph_read_guard.get(distinct_sbom_id.to_string().as_str()) { + if is_cyclic_directed(graph) { + log::warn!( + "analysis graph of sbom {} has circular references!", + distinct_sbom_id + ); + } + + let mut visited = HashSet::new(); + + // Iterate over matching node indices and process them directly + graph + .node_indices() + .filter(|&i| { + match &query { + GraphQuery::Component(ComponentReference::Name(name)) => graph + .node_weight(i) + .map(|node| node.name.eq(name)) + .unwrap_or(false), + GraphQuery::Component(ComponentReference::Purl(component_purl)) => { + if let Some(node) = graph.node_weight(i) { + node.purl.contains(component_purl) + } else { + false // Return false if the node does not exist + } + } + GraphQuery::Component(ComponentReference::Cpe(component_cpe)) => { + if let Some(node) = graph.node_weight(i) { + node.cpe.contains(component_cpe) + } else { + false // Return false if the node does not exist + } + } + GraphQuery::Query(query) => graph + .node_weight(i) + .map(|node| { + query.apply(&HashMap::from([ + ("sbom_id", Value::String(&node.sbom_id)), + ("node_id", Value::String(&node.node_id)), + ("name", Value::String(&node.name)), + ("version", Value::String(&node.version)), + ])) + }) + .unwrap_or(false), + } + }) + .for_each(|node_index| { + if !visited.contains(&node_index) { + visited.insert(node_index); + + if let Some(find_match_package_node) = graph.node_weight(node_index) { + log::debug!("matched!"); + components.push(AncestorSummary { + sbom_id: find_match_package_node.sbom_id.to_string(), + node_id: find_match_package_node.node_id.to_string(), + purl: find_match_package_node.purl.clone(), + cpe: find_match_package_node.cpe.clone(), + name: find_match_package_node.name.to_string(), + version: find_match_package_node.version.to_string(), + published: find_match_package_node.published.to_string(), + document_id: find_match_package_node.document_id.to_string(), + product_name: find_match_package_node.product_name.to_string(), + product_version: find_match_package_node + .product_version + .to_string(), + ancestors: ancestor_nodes(graph, node_index), + }); + } + } + }); + } + } + + drop(graph_read_guard); + + components + } + + pub async fn query_deps_graph( + &self, + query: impl Into>, + distinct_sbom_ids: Vec, + ) -> Vec { + let query = query.into(); + + let mut components = Vec::new(); + + // RwLock for reading hashmap + let graph_read_guard = self.graph.read(); + for distinct_sbom_id in &distinct_sbom_ids { + if let Some(graph) = graph_read_guard.get(distinct_sbom_id.to_string().as_str()) { + if is_cyclic_directed(graph) { + log::warn!( + "analysis graph of sbom {} has circular references!", + distinct_sbom_id + ); + } + + let mut visited = HashSet::new(); + + // Iterate over matching node indices and process them directly + graph + .node_indices() + .filter(|&i| { + match &query { + GraphQuery::Component(ComponentReference::Name(component_name)) => { + graph + .node_weight(i) + .map(|node| node.name.eq(component_name)) + .unwrap_or(false) + } + GraphQuery::Component(ComponentReference::Purl(component_purl)) => { + if let Some(node) = graph.node_weight(i) { + node.purl.contains(component_purl) + } else { + false // Return false if the node does not exist + } + } + GraphQuery::Component(ComponentReference::Cpe(component_cpe)) => { + if let Some(node) = graph.node_weight(i) { + node.cpe.contains(component_cpe) + } else { + false // Return false if the node does not exist + } + } + GraphQuery::Query(query) => graph + .node_weight(i) + .map(|node| { + query.apply(&HashMap::from([ + ("sbom_id", Value::String(&node.sbom_id)), + ("node_id", Value::String(&node.node_id)), + ("name", Value::String(&node.name)), + ("version", Value::String(&node.version)), + ])) + }) + .unwrap_or(false), + } + }) + .for_each(|node_index| { + if !visited.contains(&node_index) { + visited.insert(node_index); + + if let Some(find_match_package_node) = graph.node_weight(node_index) { + log::debug!("matched!"); + components.push(DepSummary { + sbom_id: find_match_package_node.sbom_id.to_string(), + node_id: find_match_package_node.node_id.to_string(), + purl: find_match_package_node.purl.clone(), + cpe: find_match_package_node.cpe.clone(), + name: find_match_package_node.name.to_string(), + version: find_match_package_node.version.to_string(), + published: find_match_package_node.published.to_string(), + document_id: find_match_package_node.document_id.to_string(), + product_name: find_match_package_node.product_name.to_string(), + product_version: find_match_package_node + .product_version + .to_string(), + deps: dep_nodes(graph, node_index, &mut HashSet::new()), + }); + } + } + }); + } + } + + drop(graph_read_guard); + + components + } + + pub async fn retrieve_all_sbom_roots_by_name( + &self, + sbom_id: Uuid, + component_name: String, + connection: &C, + ) -> Result, Error> { + // This function searches for a component(s) by name in a specific sbom, then returns that components + // root components. + + let distinct_sbom_ids = vec![sbom_id.to_string()]; + self.load_graphs(connection, &distinct_sbom_ids).await?; + + let components = self.query_ancestor_graph( + GraphQuery::Component(ComponentReference::Name(&component_name)), + distinct_sbom_ids, + ); + + let mut root_components = Vec::new(); + for component in components { + if let Some(last_ancestor) = component.ancestors.last() { + if !root_components.contains(last_ancestor) { + // we want distinct list + root_components.push(last_ancestor.clone()); + } + } + } + + Ok(root_components) + } + + #[instrument(skip(self, connection), err)] + pub async fn retrieve_root_components( + &self, + query: impl Into> + Debug, + paginated: Paginated, + connection: &C, + ) -> Result, Error> { + let query = query.into(); + + let distinct_sbom_ids = self.load_graphs_query(connection, query).await?; + let components = self.query_ancestor_graph(query, distinct_sbom_ids); + + Ok(paginated.paginate_array(&components)) + } + + #[instrument(skip(self, connection), err)] + pub async fn retrieve_deps( + &self, + query: impl Into> + Debug, + paginated: Paginated, + connection: &C, + ) -> Result, Error> { + let query = query.into(); + + let distinct_sbom_ids = self.load_graphs_query(connection, query).await?; + let components = self.query_deps_graph(query, distinct_sbom_ids).await; + + Ok(paginated.paginate_array(&components)) + } + + /// Take a [`GraphQuery`] and load all required SBOMs + async fn load_graphs_query( + &self, + connection: &C, + query: GraphQuery<'_>, + ) -> Result, Error> { + let search_sbom_subquery = match query { + GraphQuery::Component(ComponentReference::Name(name)) => sbom_node::Entity::find() + .filter(sbom_node::Column::Name.eq(name)) + .select_only() + .column(sbom_node::Column::SbomId) + .distinct() + .into_query(), + GraphQuery::Component(ComponentReference::Purl(purl)) => sbom_node::Entity::find() + .join(JoinType::Join, sbom_node::Relation::Package.def()) + .join(JoinType::Join, sbom_package::Relation::Purl.def()) + .filter(sbom_package_purl_ref::Column::QualifiedPurlId.eq(purl.qualifier_uuid())) + .select_only() + .column(sbom_node::Column::SbomId) + .distinct() + .into_query(), + GraphQuery::Component(ComponentReference::Cpe(cpe)) => sbom_node::Entity::find() + .join(JoinType::Join, sbom_node::Relation::Package.def()) + .join(JoinType::Join, sbom_package::Relation::Cpe.def()) + .filter(sbom_package_cpe_ref::Column::CpeId.eq(cpe.uuid())) + .select_only() + .column(sbom_node::Column::SbomId) + .distinct() + .into_query(), + GraphQuery::Query(query) => sbom_node::Entity::find() + .filtering(query.clone())? + .select_only() + .column(sbom_node::Column::SbomId) + .distinct() + .into_query(), + }; + + self.load_graphs_subquery(connection, search_sbom_subquery) + .await + } + + /// Take a select for sboms, and ensure they are loaded and return their IDs. + async fn load_graphs_subquery( + &self, + connection: &C, + subquery: SelectStatement, + ) -> Result, Error> { + let distinct_sbom_ids: Vec = sbom::Entity::find() + .filter(sbom::Column::SbomId.in_subquery(subquery)) + .select() + .order_by(sbom::Column::DocumentId, Order::Asc) + .order_by(sbom::Column::Published, Order::Desc) + .all(connection) + .await? + .into_iter() + .map(|record| record.sbom_id.to_string()) // Assuming sbom_id is of type String + .collect(); + + self.load_graphs(connection, &distinct_sbom_ids).await?; + + Ok(distinct_sbom_ids) + } +} diff --git a/modules/analysis/src/service/query.rs b/modules/analysis/src/service/query.rs new file mode 100644 index 000000000..f592b3862 --- /dev/null +++ b/modules/analysis/src/service/query.rs @@ -0,0 +1,75 @@ +use trustify_common::db::query::Query; +use trustify_common::{cpe::Cpe, purl::Purl}; + +#[derive(Copy, Clone, Debug)] +pub enum ComponentReference<'a> { + Name(&'a str), + Purl(&'a Purl), + Cpe(&'a Cpe), +} + +impl<'a> From<&'a str> for ComponentReference<'a> { + fn from(value: &'a str) -> Self { + Self::Name(value) + } +} + +impl<'a> From<&'a String> for ComponentReference<'a> { + fn from(value: &'a String) -> Self { + Self::Name(value) + } +} + +impl<'a> From<&'a Cpe> for ComponentReference<'a> { + fn from(value: &'a Cpe) -> Self { + Self::Cpe(value) + } +} + +impl<'a> From<&'a Purl> for ComponentReference<'a> { + fn from(value: &'a Purl) -> Self { + Self::Purl(value) + } +} + +#[derive(Copy, Clone, Debug)] +pub enum GraphQuery<'a> { + Component(ComponentReference<'a>), + Query(&'a Query), +} + +impl<'a> From> for GraphQuery<'a> { + fn from(reference: ComponentReference<'a>) -> Self { + Self::Component(reference) + } +} + +impl<'a> From<&'a str> for GraphQuery<'a> { + fn from(value: &'a str) -> Self { + Self::Component(ComponentReference::Name(value)) + } +} + +impl<'a> From<&'a String> for GraphQuery<'a> { + fn from(value: &'a String) -> Self { + Self::Component(ComponentReference::Name(value)) + } +} + +impl<'a> From<&'a Cpe> for GraphQuery<'a> { + fn from(value: &'a Cpe) -> Self { + Self::Component(ComponentReference::Cpe(value)) + } +} + +impl<'a> From<&'a Purl> for GraphQuery<'a> { + fn from(value: &'a Purl) -> Self { + Self::Component(ComponentReference::Purl(value)) + } +} + +impl<'a> From<&'a Query> for GraphQuery<'a> { + fn from(query: &'a Query) -> Self { + Self::Query(query) + } +} diff --git a/modules/analysis/src/service/test.rs b/modules/analysis/src/service/test.rs new file mode 100644 index 000000000..d6facd859 --- /dev/null +++ b/modules/analysis/src/service/test.rs @@ -0,0 +1,410 @@ +use super::*; + +use std::str::FromStr; +use test_context::test_context; +use test_log::test; +use trustify_common::{cpe::Cpe, db::query::Query, model::Paginated, purl::Purl}; +use trustify_test_context::TrustifyContext; + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_analysis_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json", "spdx/simple.json"]) + .await?; //double ingestion intended + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_root_components(&Query::q("DD"), Paginated::default(), &ctx.db) + .await?; + + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .purl, + vec![Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src")?] + ); + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .node_id, + "SPDXRef-AA".to_string() + ); + assert_eq!(analysis_graph.total, 1); + + // ensure we set implicit relationship on component with no defined relationships + let analysis_graph = service + .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .await?; + assert_eq!(analysis_graph.total, 1); + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_analysis_cyclonedx_service( + ctx: &TrustifyContext, +) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["cyclonedx/simple.json", "cyclonedx/simple.json"]) + .await?; //double ingestion intended + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_root_components(&Query::q("DD"), Paginated::default(), &ctx.db) + .await?; + + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .purl, + vec![Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src")?] + ); + let node = analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap(); + assert_eq!(node.node_id, "aa".to_string()); + assert_eq!(node.name, "AA".to_string()); + assert_eq!(analysis_graph.total, 1); + + // ensure we set implicit relationship on component with no defined relationships + let analysis_graph = service + .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .await?; + assert_eq!(analysis_graph.total, 1); + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_by_name_analysis_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json"]).await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_root_components("B", Paginated::default(), &ctx.db) + .await?; + + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .purl, + vec![Purl::from_str("pkg:rpm/redhat/A@0.0.0?arch=src")?] + ); + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .node_id, + "SPDXRef-A".to_string() + ); + assert_eq!(analysis_graph.total, 1); + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_by_purl_analysis_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json"]).await?; + + let service = AnalysisService::new(); + + let component_purl: Purl = Purl::from_str("pkg:rpm/redhat/B@0.0.0").map_err(Error::Purl)?; + + let analysis_graph = service + .retrieve_root_components(&component_purl, Paginated::default(), &ctx.db) + .await?; + + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .purl, + vec![Purl::from_str("pkg:rpm/redhat/A@0.0.0?arch=src")?] + ); + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .node_id, + "SPDXRef-A".to_string() + ); + assert_eq!(analysis_graph.total, 1); + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_quarkus_analysis_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents([ + "spdx/quarkus-bom-3.2.11.Final-redhat-00001.json", + "spdx/quarkus-bom-3.2.12.Final-redhat-00002.json", + ]) + .await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_root_components(&Query::q("spymemcached"), Paginated::default(), &ctx.db) + .await?; + + assert_eq!( + analysis_graph.items.last().unwrap().ancestors.last().unwrap().purl, + vec![Purl::from_str("pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.12.Final-redhat-00002?type=pom&repository_url=https%3a%2f%2fmaven.repository.redhat.com%2fga%2f")?] + ); + assert_eq!( + analysis_graph + .items + .last() + .unwrap() + .ancestors + .last() + .unwrap() + .node_id, + "SPDXRef-e24fec28-1001-499c-827f-2e2e5f2671b5".to_string() + ); + + assert_eq!(analysis_graph.total, 2); + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_status_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json"]).await?; + + let service = AnalysisService::new(); + let _load_all_graphs = service.load_all_graphs(&ctx.db).await; + let analysis_status = service.status(&ctx.db).await?; + + assert_eq!(analysis_status.sbom_count, 1); + assert_eq!(analysis_status.graph_count, 1); + + let _clear_all_graphs = service.clear_all_graphs(); + + ctx.ingest_documents([ + "spdx/quarkus-bom-3.2.11.Final-redhat-00001.json", + "spdx/quarkus-bom-3.2.12.Final-redhat-00002.json", + ]) + .await?; + + let analysis_status = service.status(&ctx.db).await?; + + assert_eq!(analysis_status.sbom_count, 3); + assert_eq!(analysis_status.graph_count, 0); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json"]).await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_deps(&Query::q("AA"), Paginated::default(), &ctx.db) + .await?; + + assert_eq!(analysis_graph.total, 1); + + // ensure we set implicit relationship on component with no defined relationships + let analysis_graph = service + .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .await?; + assert_eq!(analysis_graph.total, 1); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_deps_cyclonedx_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["cyclonedx/simple.json"]).await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_deps(&Query::q("AA"), Paginated::default(), &ctx.db) + .await?; + + assert_eq!(analysis_graph.total, 1); + + // ensure we set implicit relationship on component with no defined relationships + let analysis_graph = service + .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .await?; + assert_eq!(analysis_graph.total, 1); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_by_name_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json"]).await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_deps("A", Paginated::default(), &ctx.db) + .await?; + + assert_eq!(analysis_graph.items.len(), 1); + assert_eq!(analysis_graph.total, 1); + + assert_eq!( + analysis_graph.items[0].purl, + vec![Purl::from_str("pkg:rpm/redhat/A@0.0.0?arch=src")?] + ); + assert_eq!( + analysis_graph.items[0].cpe, + vec![Cpe::from_str("cpe:/a:redhat:simple:1::el9")?] + ); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_simple_by_purl_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/simple.json"]).await?; + + let service = AnalysisService::new(); + + let component_purl: Purl = + Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src").map_err(Error::Purl)?; + + let analysis_graph = service + .retrieve_deps(&component_purl, Paginated::default(), &ctx.db) + .await?; + + assert_eq!( + analysis_graph.items[0].purl, + vec![Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src")?] + ); + + assert_eq!(analysis_graph.total, 1); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_quarkus_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents([ + "spdx/quarkus-bom-3.2.11.Final-redhat-00001.json", + "spdx/quarkus-bom-3.2.12.Final-redhat-00002.json", + ]) + .await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_deps(&Query::q("spymemcached"), Paginated::default(), &ctx.db) + .await?; + + assert_eq!(analysis_graph.total, 2); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_circular_deps_cyclonedx_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["cyclonedx/cyclonedx-circular.json"]) + .await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_deps("junit-bom", Paginated::default(), &ctx.db) + .await?; + + assert_eq!(analysis_graph.total, 1); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_circular_deps_spdx_service(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/loop.json"]).await?; + + let service = AnalysisService::new(); + + let analysis_graph = service + .retrieve_deps("A", Paginated::default(), &ctx.db) + .await?; + + assert_eq!(analysis_graph.total, 1); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_retrieve_all_sbom_roots_by_name1(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + ctx.ingest_documents(["spdx/quarkus-bom-3.2.11.Final-redhat-00001.json"]) + .await?; + + let service = AnalysisService::new(); + let component_name = "quarkus-vertx-http".to_string(); + + let analysis_graph = service + .retrieve_root_components(&Query::q(&component_name), Paginated::default(), &ctx.db) + .await?; + + let sbom_id = analysis_graph + .items + .last() + .unwrap() + .sbom_id + .parse::()?; + + let roots = service + .retrieve_all_sbom_roots_by_name(sbom_id, component_name, &ctx.db) + .await?; + + assert_eq!(roots.last().unwrap().name, "quarkus-bom"); + + Ok(()) +} diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index ac2b13818..ca1038fcc 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -10,6 +10,7 @@ trustify-auth = { workspace = true } trustify-common = { workspace = true } trustify-cvss = { workspace = true } trustify-entity = { workspace = true } +trustify-module-analysis = { workspace = true } trustify-module-ingestor = { workspace = true } trustify-module-storage = { workspace = true } diff --git a/modules/fundamental/src/ai/service/tools/package_info.rs b/modules/fundamental/src/ai/service/tools/package_info.rs index 11d7c3e51..9e3be47ce 100644 --- a/modules/fundamental/src/ai/service/tools/package_info.rs +++ b/modules/fundamental/src/ai/service/tools/package_info.rs @@ -129,7 +129,7 @@ Input: The package name, its Identifier URI, or UUID. let sboms = sbom_service .find_related_sboms( - SbomExternalPackageReference::Purl(item.head.purl.clone()), + SbomExternalPackageReference::Purl(&item.head.purl), Default::default(), Default::default(), db, @@ -279,10 +279,10 @@ mod tests { assert_tool_contains( tool.clone(), - "pkg:maven/org.jboss.logging/commons-logging-jboss-logging@1.0.0.Final-redhat-1?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "pkg:maven/org.jboss.logging/commons-logging-jboss-logging@1.0.0.Final-redhat-1?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", r#" { - "identifier": "pkg:maven/org.jboss.logging/commons-logging-jboss-logging@1.0.0.Final-redhat-1?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "identifier": "pkg:maven/org.jboss.logging/commons-logging-jboss-logging@1.0.0.Final-redhat-1?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "commons-logging-jboss-logging", "version": "1.0.0.Final-redhat-1", @@ -304,7 +304,7 @@ mod tests { "commons-logging-jboss-logging", r#" { - "identifier": "pkg:maven/org.jboss.logging/commons-logging-jboss-logging@1.0.0.Final-redhat-1?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "identifier": "pkg:maven/org.jboss.logging/commons-logging-jboss-logging@1.0.0.Final-redhat-1?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "commons-logging-jboss-logging", "version": "1.0.0.Final-redhat-1", @@ -330,25 +330,25 @@ There are multiple that match: { "items": [ { - "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb-common-deployment@2.13.8.Final-redhat-00004?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb-common-deployment@2.13.8.Final-redhat-00004?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "quarkus-resteasy-reactive-jsonb-common-deployment", "version": "2.13.8.Final-redhat-00004" }, { - "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb@2.13.8.Final-redhat-00004?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb@2.13.8.Final-redhat-00004?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "quarkus-resteasy-reactive-jsonb", "version": "2.13.8.Final-redhat-00004" }, { - "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb-common@2.13.8.Final-redhat-00004?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb-common@2.13.8.Final-redhat-00004?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "quarkus-resteasy-reactive-jsonb-common", "version": "2.13.8.Final-redhat-00004" }, { - "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb-deployment@2.13.8.Final-redhat-00004?repository_url=https://maven.repository.redhat.com/ga/&type=jar", + "identifier": "pkg:maven/io.quarkus/quarkus-resteasy-reactive-jsonb-deployment@2.13.8.Final-redhat-00004?repository_url=https%3A%2F%2Fmaven%2Erepository%2Eredhat%2Ecom%2Fga%2F&type=jar", "uuid": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "quarkus-resteasy-reactive-jsonb-deployment", "version": "2.13.8.Final-redhat-00004" diff --git a/modules/fundamental/src/endpoints.rs b/modules/fundamental/src/endpoints.rs index 3531f90a7..02efc5db8 100644 --- a/modules/fundamental/src/endpoints.rs +++ b/modules/fundamental/src/endpoints.rs @@ -1,5 +1,6 @@ use actix_web::web; use trustify_common::db::Database; +use trustify_module_analysis::service::AnalysisService; use trustify_module_ingestor::graph::Graph; use trustify_module_ingestor::service::IngestorService; use trustify_module_storage::service::dispatch::DispatchBackend; @@ -16,8 +17,9 @@ pub fn configure( config: Config, db: Database, storage: impl Into, + analysis: AnalysisService, ) { - let ingestor_service = IngestorService::new(Graph::new(db.clone()), storage); + let ingestor_service = IngestorService::new(Graph::new(db.clone()), storage, Some(analysis)); svc.app_data(web::Data::new(ingestor_service)); crate::advisory::endpoints::configure(svc, db.clone(), config.advisory_upload_limit); diff --git a/modules/fundamental/src/sbom/endpoints/mod.rs b/modules/fundamental/src/sbom/endpoints/mod.rs index 267ea6b7d..b50872856 100644 --- a/modules/fundamental/src/sbom/endpoints/mod.rs +++ b/modules/fundamental/src/sbom/endpoints/mod.rs @@ -1,43 +1,38 @@ mod config; mod label; +mod query; #[cfg(test)] mod test; -use crate::sbom::model::SbomExternalPackageReference; +pub use query::*; + use crate::{ purl::service::PurlService, sbom::{ model::{ - details::SbomAdvisory, SbomPackage, SbomPackageReference, SbomPackageRelation, - SbomSummary, Which, + details::SbomAdvisory, SbomExternalPackageReference, SbomNodeReference, SbomPackage, + SbomPackageRelation, SbomSummary, Which, }, service::SbomService, }, Error::{self, Internal}, }; -use actix_http::body::BoxBody; -use actix_web::{delete, get, http::header, post, web, HttpResponse, Responder, ResponseError}; +use actix_web::{delete, get, http::header, post, web, HttpResponse, Responder}; use config::Config; use futures_util::TryStreamExt; use sea_orm::{prelude::Uuid, TransactionTrait}; -use std::{ - fmt::{Display, Formatter}, - str::FromStr, -}; +use std::str::FromStr; use trustify_auth::{ all, authenticator::user::UserInformation, authorizer::{Authorizer, Require}, CreateSbom, DeleteSbom, Permission, ReadAdvisory, ReadSbom, }; -use trustify_common::cpe::Cpe; use trustify_common::{ db::{query::Query, Database}, decompress::decompress_async, - error::ErrorInformation, id::Id, model::{BinaryData, Paginated, PaginatedResults}, - purl::Purl, }; use trustify_entity::{labels::Labels, relationship::Relationship}; use trustify_module_ingestor::{ @@ -102,72 +97,6 @@ pub async fn all( Ok(HttpResponse::Ok().json(result)) } -#[derive(Clone, Debug, serde::Deserialize, utoipa::IntoParams, utoipa::ToSchema)] -struct AllRelatedQuery { - /// Find by PURL - #[serde(default)] - pub purl: Option, - /// Find by CPE - #[serde(default)] - pub cpe: Option, - /// Find by an ID of a package - #[serde(default)] - pub id: Option, -} - -#[derive(Debug)] -pub struct AllRelatedQueryParseError(AllRelatedQuery); - -impl Display for AllRelatedQueryParseError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Requires either `purl`, `cpe`, or `id` (got - purl: {:?}, cpe: {:?}, id: {:?})", - self.0.purl, self.0.cpe, self.0.id - ) - } -} - -impl ResponseError for AllRelatedQueryParseError { - fn error_response(&self) -> HttpResponse { - HttpResponse::BadRequest().json(ErrorInformation { - error: "IdOrPurl".into(), - message: "Requires either `purl`, `cpe`, or `id`".to_string(), - details: Some(format!( - "Received - PURL: {:?}, CPE: {:?}, ID: {:?}", - self.0.purl, self.0.cpe, self.0.id - )), - }) - } -} - -impl TryFrom for SbomExternalPackageReference { - type Error = AllRelatedQueryParseError; - - fn try_from(value: AllRelatedQuery) -> Result { - Ok(match value { - AllRelatedQuery { - purl: Some(purl), - cpe: None, - id: None, - } => SbomExternalPackageReference::Purl(purl), - AllRelatedQuery { - purl: None, - cpe: Some(cpe), - id: None, - } => SbomExternalPackageReference::Cpe(cpe), - AllRelatedQuery { - purl: None, - cpe: None, - id: Some(id), - } => SbomExternalPackageReference::Id(id), - _ => { - return Err(AllRelatedQueryParseError(value)); - } - }) - } -} - /// Find all SBOMs containing the provided package. /// /// The package can be provided either via a PURL or using the ID of a package as returned by @@ -178,7 +107,7 @@ impl TryFrom for SbomExternalPackageReference { params( Query, Paginated, - AllRelatedQuery, + ExternalReferenceQuery, ), responses( (status = 200, description = "Matching SBOMs", body = PaginatedResults), @@ -190,13 +119,13 @@ pub async fn all_related( db: web::Data, web::Query(search): web::Query, web::Query(paginated): web::Query, - web::Query(all_related): web::Query, + web::Query(all_related): web::Query, authorizer: web::Data, user: UserInformation, ) -> actix_web::Result { authorizer.require(&user, Permission::ReadSbom)?; - let id = all_related.try_into()?; + let id = (&all_related).try_into()?; let result = sbom .find_related_sboms(id, paginated, search, db.as_ref()) @@ -213,7 +142,7 @@ pub async fn all_related( tag = "sbom", operation_id = "countRelatedSboms", params( - AllRelatedQuery, + ExternalReferenceQuery, ), responses( (status = 200, description = "Number of matching SBOMs per package", body = Vec), @@ -223,12 +152,12 @@ pub async fn all_related( pub async fn count_related( sbom: web::Data, db: web::Data, - web::Json(ids): web::Json>, + web::Json(ids): web::Json>, _: Require, ) -> actix_web::Result { let ids = ids - .into_iter() - .map(Uuid::try_from) + .iter() + .map(SbomExternalPackageReference::try_from) .collect::, _>>()?; let result = sbom.count_related_sboms(ids, db.as_ref()).await?; @@ -406,8 +335,8 @@ pub async fn related( paginated, related.which, match &related.reference { - None => SbomPackageReference::All, - Some(id) => SbomPackageReference::Package(id), + None => SbomNodeReference::All, + Some(id) => SbomNodeReference::Package(id), }, related.relationship, db.as_ref(), diff --git a/modules/fundamental/src/sbom/endpoints/query.rs b/modules/fundamental/src/sbom/endpoints/query.rs new file mode 100644 index 000000000..6862d1ed8 --- /dev/null +++ b/modules/fundamental/src/sbom/endpoints/query.rs @@ -0,0 +1,61 @@ +use crate::sbom::model::SbomExternalPackageReference; +use actix_http::body::BoxBody; +use actix_web::{HttpResponse, ResponseError}; +use std::fmt::{Display, Formatter}; +use trustify_common::{cpe::Cpe, error::ErrorInformation, purl::Purl}; + +#[derive(Clone, Debug, serde::Deserialize, utoipa::IntoParams, utoipa::ToSchema)] +pub struct ExternalReferenceQuery { + /// Find by PURL + #[serde(default)] + pub purl: Option, + /// Find by CPE + #[serde(default)] + pub cpe: Option, +} + +#[derive(Debug)] +pub struct ExternalReferenceQueryParseError(ExternalReferenceQuery); + +impl Display for ExternalReferenceQueryParseError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Requires either `purl` or `cpe` (got - purl: {:?}, cpe: {:?})", + self.0.purl, self.0.cpe + ) + } +} + +impl ResponseError for ExternalReferenceQueryParseError { + fn error_response(&self) -> HttpResponse { + HttpResponse::BadRequest().json(ErrorInformation { + error: "CpeOrPurl".into(), + message: "Requires either `purl` or `cpe`".to_string(), + details: Some(format!( + "Received - PURL: {:?}, CPE: {:?}", + self.0.purl, self.0.cpe + )), + }) + } +} + +impl<'a> TryFrom<&'a ExternalReferenceQuery> for SbomExternalPackageReference<'a> { + type Error = ExternalReferenceQueryParseError; + + fn try_from(value: &'a ExternalReferenceQuery) -> Result { + Ok(match value { + ExternalReferenceQuery { + purl: Some(purl), + cpe: None, + } => SbomExternalPackageReference::Purl(purl), + ExternalReferenceQuery { + purl: None, + cpe: Some(cpe), + } => SbomExternalPackageReference::Cpe(cpe), + _ => { + return Err(ExternalReferenceQueryParseError(value.clone())); + } + }) + } +} diff --git a/modules/fundamental/src/sbom/model/mod.rs b/modules/fundamental/src/sbom/model/mod.rs index 60dd12d74..427b662e3 100644 --- a/modules/fundamental/src/sbom/model/mod.rs +++ b/modules/fundamental/src/sbom/model/mod.rs @@ -8,9 +8,7 @@ use async_graphql::SimpleObject; use sea_orm::{prelude::Uuid, ConnectionTrait, ModelTrait, PaginatorTrait}; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -use trustify_common::cpe::Cpe; -use trustify_common::model::Paginated; -use trustify_common::purl::Purl; +use trustify_common::{cpe::Cpe, model::Paginated, purl::Purl}; use trustify_entity::{ labels::Labels, relationship::Relationship, sbom, sbom_node, sbom_package, source_document, }; @@ -111,28 +109,46 @@ pub struct SbomPackage { pub cpe: Vec, } -// TODO: think about a way to add CPE and PURLs too -#[derive(Clone, Eq, PartialEq, Debug)] +#[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum SbomPackageReference<'a> { + Internal(&'a str), + External(SbomExternalPackageReference<'a>), +} + +impl<'a> From> for SbomPackageReference<'a> { + fn from(value: SbomExternalPackageReference<'a>) -> Self { + Self::External(value) + } +} + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum SbomExternalPackageReference<'a> { + Purl(&'a Purl), + Cpe(&'a Cpe), +} + +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum SbomNodeReference<'a> { /// Reference all packages of the SBOM. All, /// Reference a package inside an SBOM, by its node id. + // TODO: replace with `SbomPackageReference` Package(&'a str), } -impl<'a> From<&'a str> for SbomPackageReference<'a> { +impl<'a> From<&'a str> for SbomNodeReference<'a> { fn from(value: &'a str) -> Self { Self::Package(value) } } -impl From<()> for SbomPackageReference<'_> { +impl From<()> for SbomNodeReference<'_> { fn from(_value: ()) -> Self { Self::All } } -impl<'a> From<&'a SbomPackage> for SbomPackageReference<'a> { +impl<'a> From<&'a SbomPackage> for SbomNodeReference<'a> { fn from(value: &'a SbomPackage) -> Self { Self::Package(&value.id) } @@ -153,15 +169,3 @@ pub enum Which { /// Target side Right, } - -#[derive(Clone, Eq, PartialEq, Debug)] -pub enum SbomExternalPackageReference { - /// The ID of the package/component. - /// - /// This is actually not internal, but external. - Id(String), - /// The PackageURL of the package/component. - Purl(Purl), - /// The CPE of the package/component. - Cpe(Cpe), -} diff --git a/modules/fundamental/src/sbom/service/sbom.rs b/modules/fundamental/src/sbom/service/sbom.rs index e4abb9b79..9afaebe58 100644 --- a/modules/fundamental/src/sbom/service/sbom.rs +++ b/modules/fundamental/src/sbom/service/sbom.rs @@ -1,10 +1,9 @@ use super::SbomService; -use crate::sbom::model::SbomExternalPackageReference; use crate::{ purl::model::summary::purl::PurlSummary, sbom::model::{ - details::SbomDetails, SbomPackage, SbomPackageReference, SbomPackageRelation, SbomSummary, - Which, + details::SbomDetails, SbomExternalPackageReference, SbomNodeReference, SbomPackage, + SbomPackageRelation, SbomSummary, Which, }, Error, }; @@ -63,7 +62,6 @@ impl SbomService { &self, id: Id, statuses: Vec, - connection: &C, ) -> Result, Error> { Ok(match self.fetch_sbom(id, connection).await? { @@ -211,7 +209,7 @@ impl SbomService { Default::default(), paginated, Which::Right, - SbomPackageReference::All, + SbomNodeReference::All, Some(Relationship::DescribedBy), db, ) @@ -222,30 +220,76 @@ impl SbomService { #[instrument(skip(self, connection), err(level=tracing::Level::INFO))] pub async fn count_related_sboms( &self, - qualified_package_ids: Vec, + references: Vec>, connection: &C, ) -> Result, Error> { - let query = sbom::Entity::find() - .join(JoinType::Join, sbom::Relation::Packages.def()) - .join(JoinType::Join, sbom_package::Relation::Purl.def()) - .filter( - sbom_package_purl_ref::Column::QualifiedPurlId.is_in(qualified_package_ids.clone()), - ) - .group_by(sbom_package_purl_ref::Column::QualifiedPurlId) - .select_only() - .column(sbom_package_purl_ref::Column::QualifiedPurlId) - .column_as(sbom_package::Column::SbomId.count(), "count") - .into_tuple::<(Uuid, i64)>() - .all(connection) - .await?; + #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] + enum Id { + Cpe(Uuid), + Purl(Uuid), + } - // turn result into a map + let ids = references + .iter() + .map(|r| match r { + SbomExternalPackageReference::Cpe(c) => Id::Cpe(c.uuid()), + SbomExternalPackageReference::Purl(p) => Id::Purl(p.qualifier_uuid()), + }) + .collect::>(); - let counts_map = query.into_iter().collect::>(); + let mut counts_map = HashMap::new(); + + let cpes = ids + .iter() + .filter_map(|id| match id { + Id::Cpe(id) => Some(*id), + _ => None, + }) + .collect::>(); + + counts_map.extend( + sbom::Entity::find() + .join(JoinType::Join, sbom::Relation::Packages.def()) + .join(JoinType::Join, sbom_package::Relation::Cpe.def()) + .filter(sbom_package_cpe_ref::Column::CpeId.is_in(cpes)) + .group_by(sbom_package_cpe_ref::Column::CpeId) + .select_only() + .column(sbom_package_cpe_ref::Column::CpeId) + .column_as(sbom_package::Column::SbomId.count(), "count") + .into_tuple::<(Uuid, i64)>() + .all(connection) + .await? + .into_iter() + .map(|(id, count)| (Id::Cpe(id), count)), + ); + + let purls = ids + .iter() + .filter_map(|id| match id { + Id::Purl(id) => Some(*id), + _ => None, + }) + .collect::>(); + + counts_map.extend( + sbom::Entity::find() + .join(JoinType::Join, sbom::Relation::Packages.def()) + .join(JoinType::Join, sbom_package::Relation::Purl.def()) + .filter(sbom_package_purl_ref::Column::QualifiedPurlId.is_in(purls)) + .group_by(sbom_package_purl_ref::Column::QualifiedPurlId) + .select_only() + .column(sbom_package_purl_ref::Column::QualifiedPurlId) + .column_as(sbom_package::Column::SbomId.count(), "count") + .into_tuple::<(Uuid, i64)>() + .all(connection) + .await? + .into_iter() + .map(|(id, count)| (Id::Purl(id), count)), + ); // now use the inbound order and retrieve results in that order - let result: Vec = qualified_package_ids + let result: Vec = ids .into_iter() .map(|id| counts_map.get(&id).copied().unwrap_or_default()) .collect(); @@ -258,20 +302,17 @@ impl SbomService { #[instrument(skip(self, connection), err(level=tracing::Level::INFO))] pub async fn find_related_sboms( &self, - external_package_ref: SbomExternalPackageReference, + package_ref: SbomExternalPackageReference<'_>, paginated: Paginated, query: Query, connection: &C, ) -> Result, Error> { let select = sbom::Entity::find().join(JoinType::Join, sbom::Relation::Packages.def()); - let select = match external_package_ref { + let select = match package_ref { SbomExternalPackageReference::Purl(purl) => select .join(JoinType::Join, sbom_package::Relation::Purl.def()) .filter(sbom_package_purl_ref::Column::QualifiedPurlId.eq(purl.qualifier_uuid())), - SbomExternalPackageReference::Id(id) => { - select.filter(sbom_package::Column::NodeId.eq(id)) - } SbomExternalPackageReference::Cpe(cpe) => select .join(JoinType::Join, sbom_package::Relation::Cpe.def()) .filter(sbom_package_cpe_ref::Column::CpeId.eq(cpe.uuid())), @@ -306,12 +347,10 @@ impl SbomService { search: Query, paginated: Paginated, which: Which, - reference: impl Into> + Debug, + reference: impl Into> + Debug, relationship: Option, db: &C, ) -> Result, Error> { - // let db = self.db.connection(connection); - // which way log::debug!("Which: {which:?}"); @@ -356,11 +395,11 @@ impl SbomService { // filter for reference query = match reference.into() { - SbomPackageReference::All => { + SbomNodeReference::All => { // sbom - add join to sbom table query.join(JoinType::Join, sbom_node::Relation::Sbom.def()) } - SbomPackageReference::Package(node_id) => { + SbomNodeReference::Package(node_id) => { // package - set node id filter query.filter(filter.eq(node_id)) } @@ -411,7 +450,7 @@ impl SbomService { &self, sbom_id: Uuid, relationship: impl Into>, - pkg: impl Into> + Debug, + pkg: impl Into> + Debug, tx: &C, ) -> Result, Error> { let result = self @@ -614,32 +653,6 @@ struct PurlDto { qualifiers: Qualifiers, } -/* -impl From for Purl { - fn from(value: PurlDto) -> Self { - let PurlDto { - r#type, - name, - namespace, - version, - qualifiers, - } = value; - Self { - ty: r#type, - name, - namespace, - version: if version.is_empty() { - None - } else { - Some(version) - }, - qualifiers: qualifiers.0, - } - } -} - - */ - #[derive(Debug)] pub struct QueryCatcher { pub advisory: advisory::Model, diff --git a/modules/fundamental/src/sbom/service/test.rs b/modules/fundamental/src/sbom/service/test.rs index 246376041..ad810f582 100644 --- a/modules/fundamental/src/sbom/service/test.rs +++ b/modules/fundamental/src/sbom/service/test.rs @@ -1,9 +1,9 @@ -use crate::sbom::service::SbomService; +use crate::{sbom::model::SbomExternalPackageReference, sbom::service::SbomService}; use std::str::FromStr; use test_context::test_context; use test_log::test; -use trustify_common::id::Id; -use trustify_common::purl::Purl; +use trustify_common::cpe::Cpe; +use trustify_common::{id::Id, purl::Purl}; use trustify_test_context::TrustifyContext; #[test_context(TrustifyContext)] @@ -53,21 +53,29 @@ async fn count_sboms(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let service = SbomService::new(ctx.db.clone()); - let neither = Purl::from_str("pkg:maven/io.smallrye/smallrye-graphql@0.0.0.redhat-00000?repository_url=https://maven.repository.redhat.com/ga/&type=jar")?; - let both = Purl::from_str("pkg:maven/io.smallrye/smallrye-graphql@2.2.3.redhat-00001?repository_url=https://maven.repository.redhat.com/ga/&type=jar")?; - let one = Purl::from_str("pkg:maven/io.quarkus/quarkus-kubernetes-service-binding-deployment@3.2.12.Final-redhat-00001?repository_url=https://maven.repository.redhat.com/ga/&type=jar")?; + let neither_purl = Purl::from_str("pkg:maven/io.smallrye/smallrye-graphql@0.0.0.redhat-00000?repository_url=https://maven.repository.redhat.com/ga/&type=jar")?; + let both_purl = Purl::from_str("pkg:maven/io.smallrye/smallrye-graphql@2.2.3.redhat-00001?repository_url=https://maven.repository.redhat.com/ga/&type=jar")?; + let one_purl = Purl::from_str("pkg:maven/io.quarkus/quarkus-kubernetes-service-binding-deployment@3.2.12.Final-redhat-00001?repository_url=https://maven.repository.redhat.com/ga/&type=jar")?; + + let neither_cpe = Cpe::from_str("cpe:/a:redhat:quarkus:0.0::el8")?; + let both_cpe = Cpe::from_str("cpe:/a:redhat:quarkus:3.2::el8")?; + + assert_ne!(neither_cpe.uuid(), both_cpe.uuid()); + let counts = service .count_related_sboms( vec![ - neither.qualifier_uuid(), - both.qualifier_uuid(), - one.qualifier_uuid(), + SbomExternalPackageReference::Cpe(&neither_cpe), + SbomExternalPackageReference::Cpe(&both_cpe), + SbomExternalPackageReference::Purl(&neither_purl), + SbomExternalPackageReference::Purl(&both_purl), + SbomExternalPackageReference::Purl(&one_purl), ], &ctx.db, ) .await?; - assert_eq!(counts, vec![0, 2, 1]); + assert_eq!(counts, vec![0, 2, 0, 2, 1]); Ok(()) } diff --git a/modules/fundamental/src/test/common.rs b/modules/fundamental/src/test/common.rs index cc430a48f..5ecee76ae 100644 --- a/modules/fundamental/src/test/common.rs +++ b/modules/fundamental/src/test/common.rs @@ -1,3 +1,4 @@ +use trustify_module_analysis::service::AnalysisService; use trustify_test_context::{ call::{self, CallService}, TrustifyContext, @@ -11,5 +12,6 @@ async fn caller_with( ctx: &TrustifyContext, config: Config, ) -> anyhow::Result { - call::caller(|svc| configure(svc, config, ctx.db.clone(), ctx.storage.clone())).await + let analysis = AnalysisService::new(); + call::caller(|svc| configure(svc, config, ctx.db.clone(), ctx.storage.clone(), analysis)).await } diff --git a/modules/fundamental/tests/sbom/reingest.rs b/modules/fundamental/tests/sbom/reingest.rs index 399c0e4d6..8b03edf3f 100644 --- a/modules/fundamental/tests/sbom/reingest.rs +++ b/modules/fundamental/tests/sbom/reingest.rs @@ -7,6 +7,7 @@ use tracing::instrument; use trustify_common::db::query::Query; use trustify_common::model::Paginated; use trustify_common::purl::Purl; +use trustify_module_fundamental::sbom::model::SbomExternalPackageReference; use trustify_module_fundamental::sbom::{model::details::SbomDetails, service::SbomService}; use trustify_module_ingestor::service::Format; use trustify_test_context::{document_bytes, TrustifyContext}; @@ -86,7 +87,7 @@ async fn quarkus(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let sboms = sbom .find_related_sboms( - Purl::from_str(purl).expect("must parse").qualifier_uuid(), + SbomExternalPackageReference::Purl(&Purl::from_str(purl).expect("must parse")), Paginated::default(), Query::default(), &ctx.db, diff --git a/modules/fundamental/tests/sbom/spdx/corner_cases.rs b/modules/fundamental/tests/sbom/spdx/corner_cases.rs index aa2da457f..686840567 100644 --- a/modules/fundamental/tests/sbom/spdx/corner_cases.rs +++ b/modules/fundamental/tests/sbom/spdx/corner_cases.rs @@ -9,7 +9,7 @@ use test_context::test_context; use test_log::test; use trustify_common::{id::Id, purl::Purl}; use trustify_entity::relationship::Relationship; -use trustify_module_fundamental::{sbom::model::SbomPackageReference, sbom::service::SbomService}; +use trustify_module_fundamental::{sbom::model::SbomNodeReference, sbom::service::SbomService}; use trustify_module_ingestor::graph::{ purl::qualified_package::QualifiedPackageContext, sbom::SbomContext, }; @@ -63,7 +63,7 @@ async fn infinite_loop(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!(packages.total, 1); let packages = service - .related_packages(id, None, SbomPackageReference::All, &ctx.db) + .related_packages(id, None, SbomNodeReference::All, &ctx.db) .await?; log::info!("Packages: {packages:#?}"); @@ -99,7 +99,7 @@ async fn double_ref(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!(packages.len(), 3); let packages = service - .related_packages(id, None, SbomPackageReference::All, &ctx.db) + .related_packages(id, None, SbomNodeReference::All, &ctx.db) .await?; log::info!("Packages: {packages:#?}"); @@ -135,7 +135,7 @@ async fn self_ref(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!(packages.len(), 0); let packages = service - .related_packages(id, None, SbomPackageReference::All, &ctx.db) + .related_packages(id, None, SbomNodeReference::All, &ctx.db) .await?; log::info!("Packages: {packages:#?}"); @@ -171,7 +171,7 @@ async fn self_ref_package(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!(packages.len(), 1); let packages = service - .related_packages(id, None, SbomPackageReference::All, &ctx.db) + .related_packages(id, None, SbomNodeReference::All, &ctx.db) .await?; log::info!("Packages: {packages:#?}"); @@ -179,12 +179,7 @@ async fn self_ref_package(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!(packages.len(), 1); let packages = service - .related_packages( - id, - None, - SbomPackageReference::Package("SPDXRef-A"), - &ctx.db, - ) + .related_packages(id, None, SbomNodeReference::Package("SPDXRef-A"), &ctx.db) .await?; log::info!("Packages: {packages:#?}"); diff --git a/modules/importer/Cargo.toml b/modules/importer/Cargo.toml index c5af6ee46..5848cd811 100644 --- a/modules/importer/Cargo.toml +++ b/modules/importer/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true trustify-auth = { workspace = true } trustify-common = { workspace = true } trustify-entity = { workspace = true } +trustify-module-analysis = { workspace = true } trustify-module-ingestor = { workspace = true } trustify-module-storage = { workspace = true } diff --git a/modules/importer/src/runner/clearly_defined/mod.rs b/modules/importer/src/runner/clearly_defined/mod.rs index 1ff3b713f..f42393416 100644 --- a/modules/importer/src/runner/clearly_defined/mod.rs +++ b/modules/importer/src/runner/clearly_defined/mod.rs @@ -19,7 +19,11 @@ impl super::ImportRunner { clearly_defined: ClearlyDefinedImporter, continuation: serde_json::Value, ) -> Result { - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let report = Arc::new(Mutex::new(ReportBuilder::new())); let continuation = serde_json::from_value(continuation).unwrap_or_default(); diff --git a/modules/importer/src/runner/clearly_defined_curation/mod.rs b/modules/importer/src/runner/clearly_defined_curation/mod.rs index 57911c718..a5ffab280 100644 --- a/modules/importer/src/runner/clearly_defined_curation/mod.rs +++ b/modules/importer/src/runner/clearly_defined_curation/mod.rs @@ -77,7 +77,11 @@ impl super::ImportRunner { clearly_defined: ClearlyDefinedCurationImporter, continuation: serde_json::Value, ) -> Result { - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let report = Arc::new(Mutex::new(ReportBuilder::new())); let continuation = serde_json::from_value(continuation).unwrap_or_default(); diff --git a/modules/importer/src/runner/csaf/mod.rs b/modules/importer/src/runner/csaf/mod.rs index addc29fde..7f0982856 100644 --- a/modules/importer/src/runner/csaf/mod.rs +++ b/modules/importer/src/runner/csaf/mod.rs @@ -64,7 +64,11 @@ impl super::ImportRunner { // storage (called by validator) - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let storage = storage::StorageVisitor { context, ingestor, diff --git a/modules/importer/src/runner/cve/mod.rs b/modules/importer/src/runner/cve/mod.rs index fa94c89ea..f87ab06e2 100644 --- a/modules/importer/src/runner/cve/mod.rs +++ b/modules/importer/src/runner/cve/mod.rs @@ -77,7 +77,11 @@ impl super::ImportRunner { cve: CveImporter, continuation: serde_json::Value, ) -> Result { - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let report = Arc::new(Mutex::new(ReportBuilder::new())); let continuation = serde_json::from_value(continuation).unwrap_or_default(); diff --git a/modules/importer/src/runner/cwe/mod.rs b/modules/importer/src/runner/cwe/mod.rs index 4b3684d76..3bc85c460 100644 --- a/modules/importer/src/runner/cwe/mod.rs +++ b/modules/importer/src/runner/cwe/mod.rs @@ -20,7 +20,11 @@ impl super::ImportRunner { cwe_catalog: CweImporter, continuation: serde_json::Value, ) -> Result { - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let report = Arc::new(Mutex::new(ReportBuilder::new())); let continuation = serde_json::from_value(continuation).unwrap_or_default(); diff --git a/modules/importer/src/runner/mod.rs b/modules/importer/src/runner/mod.rs index 4f916a457..011ecd0a5 100644 --- a/modules/importer/src/runner/mod.rs +++ b/modules/importer/src/runner/mod.rs @@ -20,12 +20,14 @@ use std::path::PathBuf; use time::OffsetDateTime; use tracing::instrument; use trustify_common::db::Database; +use trustify_module_analysis::service::AnalysisService; use trustify_module_storage::service::dispatch::DispatchBackend; pub struct ImportRunner { pub db: Database, pub storage: DispatchBackend, pub working_dir: Option, + pub analysis: Option, } impl ImportRunner { diff --git a/modules/importer/src/runner/osv/mod.rs b/modules/importer/src/runner/osv/mod.rs index cc157ad81..63e64e8a2 100644 --- a/modules/importer/src/runner/osv/mod.rs +++ b/modules/importer/src/runner/osv/mod.rs @@ -104,7 +104,11 @@ impl super::ImportRunner { osv: OsvImporter, continuation: serde_json::Value, ) -> Result { - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let report = Arc::new(Mutex::new(ReportBuilder::new())); let continuation = serde_json::from_value(continuation).unwrap_or_default(); diff --git a/modules/importer/src/runner/sbom/mod.rs b/modules/importer/src/runner/sbom/mod.rs index 4ed9109ae..8fa64e899 100644 --- a/modules/importer/src/runner/sbom/mod.rs +++ b/modules/importer/src/runner/sbom/mod.rs @@ -65,7 +65,11 @@ impl super::ImportRunner { // storage (called by validator) - let ingestor = IngestorService::new(Graph::new(self.db.clone()), self.storage.clone()); + let ingestor = IngestorService::new( + Graph::new(self.db.clone()), + self.storage.clone(), + self.analysis.clone(), + ); let storage = storage::StorageVisitor { context, source, diff --git a/modules/importer/src/server/mod.rs b/modules/importer/src/server/mod.rs index 0096757a0..d05e03566 100644 --- a/modules/importer/src/server/mod.rs +++ b/modules/importer/src/server/mod.rs @@ -15,6 +15,7 @@ use time::OffsetDateTime; use tokio::time::MissedTickBehavior; use tracing::instrument; use trustify_common::db::Database; +use trustify_module_analysis::service::AnalysisService; use trustify_module_storage::service::dispatch::DispatchBackend; /// run the importer loop @@ -22,11 +23,13 @@ pub async fn importer( db: Database, storage: DispatchBackend, working_dir: Option, + analysis: Option, ) -> anyhow::Result<()> { Server { db, storage, working_dir, + analysis, } .run() .await @@ -52,6 +55,7 @@ struct Server { db: Database, storage: DispatchBackend, working_dir: Option, + analysis: Option, } impl Server { @@ -90,6 +94,7 @@ impl Server { db: self.db.clone(), storage: self.storage.clone(), working_dir: self.working_dir.clone(), + analysis: self.analysis.clone(), }; let (last_error, report, continuation) = match runner diff --git a/modules/ingestor/src/endpoints.rs b/modules/ingestor/src/endpoints.rs index 566be81ce..041ba0482 100644 --- a/modules/ingestor/src/endpoints.rs +++ b/modules/ingestor/src/endpoints.rs @@ -6,6 +6,7 @@ use actix_web::{post, web, HttpResponse, Responder}; use trustify_auth::{authorizer::Require, UploadDataset}; use trustify_common::{db::Database, model::BinaryData}; use trustify_entity::labels::Labels; +use trustify_module_analysis::service::AnalysisService; use trustify_module_storage::service::dispatch::DispatchBackend; use utoipa::IntoParams; @@ -15,8 +16,9 @@ pub fn configure( config: Config, db: Database, storage: impl Into, + analysis: Option, ) { - let ingestor_service = IngestorService::new(Graph::new(db), storage); + let ingestor_service = IngestorService::new(Graph::new(db), storage, analysis); svc.app_data(web::Data::new(ingestor_service)) .app_data(web::Data::new(config)) diff --git a/modules/ingestor/src/service/mod.rs b/modules/ingestor/src/service/mod.rs index d557a5469..260fc13e7 100644 --- a/modules/ingestor/src/service/mod.rs +++ b/modules/ingestor/src/service/mod.rs @@ -143,13 +143,19 @@ impl ResponseError for Error { pub struct IngestorService { graph: Graph, storage: DispatchBackend, + analysis: Option, } impl IngestorService { - pub fn new(graph: Graph, storage: impl Into) -> Self { + pub fn new( + graph: Graph, + storage: impl Into, + analysis: Option, + ) -> Self { Self { graph, storage: storage.into(), + analysis, } } @@ -196,31 +202,33 @@ impl IngestorService { .load(&self.graph, labels.into(), issuer, &result.digests, bytes) .await?; - match fmt { - Format::SPDX | Format::CycloneDX => { - let analysis_service = AnalysisService::new(); - if result.id.to_string().starts_with("urn:uuid:") { - match analysis_service // TODO: today we chop off 'urn:uuid:' prefix using .split_off on result.id - .load_graphs( - vec![result.id.to_string().split_off("urn:uuid:".len())], - &self.graph.db, - ) - .await - { - Ok(_) => log::debug!( - "Analysis graph for sbom: {} loaded successfully.", - result.id.value() - ), - Err(e) => log::warn!( - "Error loading sbom {} into analysis graph : {}", - result.id.value(), - e - ), + if let Some(analysis) = &self.analysis { + match fmt { + Format::SPDX | Format::CycloneDX => { + if result.id.to_string().starts_with("urn:uuid:") { + match analysis + .load_graphs( + &self.graph.db, + // TODO: today we chop off 'urn:uuid:' prefix using .split_off on result.id + &vec![result.id.to_string().split_off("urn:uuid:".len())], + ) + .await + { + Ok(_) => log::debug!( + "Analysis graph for sbom: {} loaded successfully.", + result.id.value() + ), + Err(e) => log::warn!( + "Error loading sbom {} into analysis graph : {}", + result.id.value(), + e + ), + } } } - } - _ => {} - }; + _ => {} + }; + } let duration = Instant::now() - start; log::debug!( diff --git a/modules/ingestor/src/service/sbom/clearly_defined.rs b/modules/ingestor/src/service/sbom/clearly_defined.rs index 021daa173..881e0468b 100644 --- a/modules/ingestor/src/service/sbom/clearly_defined.rs +++ b/modules/ingestor/src/service/sbom/clearly_defined.rs @@ -175,7 +175,7 @@ mod test { #[test(tokio::test)] async fn ingest_clearly_defined(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let graph = Graph::new(ctx.db.clone()); - let ingestor = IngestorService::new(graph, ctx.storage.clone()); + let ingestor = IngestorService::new(graph, ctx.storage.clone(), Default::default()); let data = document_bytes("clearly-defined/aspnet.mvc-4.0.40804.json").await?; diff --git a/modules/ingestor/src/service/sbom/clearly_defined_curation.rs b/modules/ingestor/src/service/sbom/clearly_defined_curation.rs index f1c272e82..60c1f15b2 100644 --- a/modules/ingestor/src/service/sbom/clearly_defined_curation.rs +++ b/modules/ingestor/src/service/sbom/clearly_defined_curation.rs @@ -62,7 +62,7 @@ mod test { #[test(tokio::test)] async fn ingest_clearly_defined(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let graph = Graph::new(ctx.db.clone()); - let ingestor = IngestorService::new(graph, ctx.storage.clone()); + let ingestor = IngestorService::new(graph, ctx.storage.clone(), Default::default()); let data = document_bytes("clearly-defined/chrono.yaml").await?; diff --git a/modules/ingestor/src/service/sbom/cyclonedx.rs b/modules/ingestor/src/service/sbom/cyclonedx.rs index b4d93134a..9c5e64b65 100644 --- a/modules/ingestor/src/service/sbom/cyclonedx.rs +++ b/modules/ingestor/src/service/sbom/cyclonedx.rs @@ -83,7 +83,7 @@ mod test { let graph = Graph::new(db.clone()); let data = document_bytes("zookeeper-3.9.2-cyclonedx.json").await?; - let ingestor = IngestorService::new(graph, ctx.storage.clone()); + let ingestor = IngestorService::new(graph, ctx.storage.clone(), Default::default()); ingestor .ingest(&data, Format::CycloneDX, ("source", "test"), None) diff --git a/modules/ingestor/src/service/sbom/spdx.rs b/modules/ingestor/src/service/sbom/spdx.rs index 7b6d55396..d767743c0 100644 --- a/modules/ingestor/src/service/sbom/spdx.rs +++ b/modules/ingestor/src/service/sbom/spdx.rs @@ -83,7 +83,7 @@ mod test { let graph = Graph::new(ctx.db.clone()); let data = document_bytes("ubi9-9.2-755.1697625012.json").await?; - let ingestor = IngestorService::new(graph, ctx.storage.clone()); + let ingestor = IngestorService::new(graph, ctx.storage.clone(), Default::default()); ingestor .ingest(&data, Format::SPDX, ("source", "test"), None) diff --git a/modules/ingestor/tests/common.rs b/modules/ingestor/tests/common.rs index 809193166..d3f2798b2 100644 --- a/modules/ingestor/tests/common.rs +++ b/modules/ingestor/tests/common.rs @@ -1,3 +1,4 @@ +use trustify_module_analysis::service::AnalysisService; use trustify_module_ingestor::endpoints::{configure, Config}; use trustify_test_context::{ call::{self, CallService}, @@ -8,5 +9,15 @@ pub async fn caller_with( ctx: &TrustifyContext, config: Config, ) -> anyhow::Result { - call::caller(|svc| configure(svc, config, ctx.db.clone(), ctx.storage.clone())).await + let analysis = AnalysisService::new(); + call::caller(|svc| { + configure( + svc, + config, + ctx.db.clone(), + ctx.storage.clone(), + Some(analysis), + ) + }) + .await } diff --git a/openapi.yaml b/openapi.yaml index e87fc40b5..ac5cc60c3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1626,15 +1626,14 @@ paths: oneOf: - type: 'null' - $ref: '#/components/schemas/Purl' - - name: id + - name: cpe in: query - description: Find by an ID of a package + description: Find by CPE required: false schema: - type: - - string - - 'null' - format: uuid + oneOf: + - type: 'null' + - $ref: '#/components/schemas/Cpe' responses: '200': description: Matching SBOMs @@ -1660,22 +1659,21 @@ paths: oneOf: - type: 'null' - $ref: '#/components/schemas/Purl' - - name: id + - name: cpe in: path - description: Find by an ID of a package + description: Find by CPE required: true schema: - type: - - string - - 'null' - format: uuid + oneOf: + - type: 'null' + - $ref: '#/components/schemas/Cpe' requestBody: content: application/json: schema: type: array items: - $ref: '#/components/schemas/AllRelatedQuery' + $ref: '#/components/schemas/ExternalReferenceQuery' required: true responses: '200': @@ -2369,20 +2367,6 @@ components: name: type: string parameters: {} - AllRelatedQuery: - type: object - properties: - id: - type: - - string - - 'null' - format: uuid - description: Find by an ID of a package - purl: - oneOf: - - type: 'null' - - $ref: '#/components/schemas/Purl' - description: Find by PURL AnalysisStatus: type: object required: @@ -2392,9 +2376,13 @@ components: graph_count: type: integer format: int32 + description: The number of graphs loaded in memory + minimum: 0 sbom_count: type: integer format: int32 + description: The number of SBOMs found in the database + minimum: 0 AncNode: type: object required: @@ -2402,15 +2390,22 @@ components: - node_id - relationship - purl + - cpe - name - version properties: + cpe: + type: array + items: + $ref: '#/components/schemas/Cpe' name: type: string node_id: type: string purl: - type: string + type: array + items: + $ref: '#/components/schemas/Purl' relationship: type: string sbom_id: @@ -2423,6 +2418,7 @@ components: - sbom_id - node_id - purl + - cpe - name - version - published @@ -2435,6 +2431,10 @@ components: type: array items: $ref: '#/components/schemas/AncNode' + cpe: + type: array + items: + $ref: '#/components/schemas/Cpe' document_id: type: string name: @@ -2448,7 +2448,9 @@ components: published: type: string purl: - type: string + type: array + items: + $ref: '#/components/schemas/Purl' sbom_id: type: string version: @@ -2602,6 +2604,9 @@ components: updated_at: type: string format: date-time + Cpe: + type: string + format: uri CsafImporter: allOf: - $ref: '#/components/schemas/CommonImporter' @@ -2658,10 +2663,15 @@ components: - node_id - relationship - purl + - cpe - name - version - deps properties: + cpe: + type: array + items: + $ref: '#/components/schemas/Cpe' deps: type: array items: @@ -2671,7 +2681,9 @@ components: node_id: type: string purl: - type: string + type: array + items: + $ref: '#/components/schemas/Purl' relationship: type: string sbom_id: @@ -2684,6 +2696,7 @@ components: - sbom_id - node_id - purl + - cpe - name - version - published @@ -2692,6 +2705,10 @@ components: - product_version - deps properties: + cpe: + type: array + items: + $ref: '#/components/schemas/Cpe' deps: type: array items: @@ -2709,11 +2726,26 @@ components: published: type: string purl: - type: string + type: array + items: + $ref: '#/components/schemas/Purl' sbom_id: type: string version: type: string + ExternalReferenceQuery: + type: object + properties: + cpe: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/Cpe' + description: Find by CPE + purl: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/Purl' + description: Find by PURL Id: type: string description: A hash/digest prefixed with its type. diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 695247c87..d2dbacca4 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "garage-door")] +use crate::embedded_oidc; use crate::{endpoints, sample_data}; use actix_web::{ body::MessageBody, @@ -32,6 +34,7 @@ use trustify_infrastructure::{ tracing::Tracing, Infrastructure, InfrastructureConfig, InitContext, Metrics, }; +use trustify_module_analysis::service::AnalysisService; use trustify_module_graphql::RootQuery; use trustify_module_importer::server::importer; use trustify_module_ingestor::graph::Graph; @@ -47,9 +50,6 @@ use utoipa::{ use utoipa_rapidoc::RapiDoc; use utoipa_redoc::{Redoc, Servable}; -#[cfg(feature = "garage-door")] -use crate::embedded_oidc; - /// Run the API server #[derive(clap::Args, Debug)] pub struct Run { @@ -413,6 +413,8 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi // register REST API & UI + let analysis = AnalysisService::new(); + svc.app_data(graph) .configure(|svc| { endpoints::configure(svc, auth.clone()); @@ -427,12 +429,14 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi ingestor, db.clone(), storage.clone(), + Some(analysis.clone()), ); trustify_module_fundamental::endpoints::configure( svc, fundamental, db.clone(), storage, + analysis, ); trustify_module_analysis::endpoints::configure(svc, db.clone()); trustify_module_user::endpoints::configure(svc, db.clone()); diff --git a/server/src/profile/importer.rs b/server/src/profile/importer.rs index 322c314d0..dae538ac5 100644 --- a/server/src/profile/importer.rs +++ b/server/src/profile/importer.rs @@ -133,7 +133,17 @@ impl InitData { let db = self.db; let storage = self.storage; - let importer = async { importer(db, storage, self.working_dir).await }.boxed_local(); + let importer = async { + importer( + db, + storage, + self.working_dir, + // Running the importer, we don't need an analysis graph update + None, + ) + .await + } + .boxed_local(); let tasks = vec![importer]; diff --git a/test-context/src/lib.rs b/test-context/src/lib.rs index bbab13151..596e0133c 100644 --- a/test-context/src/lib.rs +++ b/test-context/src/lib.rs @@ -43,7 +43,7 @@ impl TrustifyContext { .await .expect("initializing the storage backend"); let graph = Graph::new(db.clone()); - let ingestor = IngestorService::new(graph.clone(), storage.clone()); + let ingestor = IngestorService::new(graph.clone(), storage.clone(), Default::default()); let mem_limit_mb = env::var("MEM_LIMIT_MB") .unwrap_or("500".into()) .parse() diff --git a/xtask/src/dataset.rs b/xtask/src/dataset.rs index 66300bb93..723883f01 100644 --- a/xtask/src/dataset.rs +++ b/xtask/src/dataset.rs @@ -102,6 +102,8 @@ impl GenerateDump { db: db.clone(), storage: storage.into(), working_dir: self.working_dir.as_ref().map(|wd| wd.join("wd")), + // The xtask doesn't need the analysis graph + analysis: None, }; // ingest documents