diff --git a/Cargo.lock b/Cargo.lock index 23211ff7..23f2c3ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8466,6 +8466,7 @@ dependencies = [ "csaf", "hex", "humantime", + "itertools 0.13.0", "jsonpath-rust", "log", "packageurl", diff --git a/modules/analysis/Cargo.toml b/modules/analysis/Cargo.toml index 2198612e..f8309a77 100644 --- a/modules/analysis/Cargo.toml +++ b/modules/analysis/Cargo.toml @@ -35,6 +35,7 @@ bytesize = { workspace = true } chrono = { workspace = true } hex = { workspace = true } humantime = { workspace = true } +itertools = { workspace = true } jsonpath-rust = { workspace = true } log = { workspace = true } petgraph = { workspace = true } diff --git a/modules/analysis/src/endpoints/test.rs b/modules/analysis/src/endpoints/test.rs index e05a3c17..9e19f123 100644 --- a/modules/analysis/src/endpoints/test.rs +++ b/modules/analysis/src/endpoints/test.rs @@ -1,7 +1,9 @@ use crate::test::caller; use actix_http::Request; use actix_web::test::TestRequest; +use itertools::Itertools; use serde_json::{json, Value}; +use std::collections::HashMap; use test_context::test_context; use test_log::test; use trustify_test_context::{call::CallService, TrustifyContext}; @@ -179,27 +181,25 @@ async fn test_simple_dep_endpoint(ctx: &TrustifyContext) -> Result<(), anyhow::E let request: Request = TestRequest::get().uri(uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; - println!("Result: {response:#?}"); - assert_eq!( response["items"][0]["purl"], Value::from(["pkg:rpm/redhat/A@0.0.0?arch=src"]), ); - let purls: Value = response["items"][0]["deps"] + let purls = response["items"][0]["deps"] .as_array() .iter() .map(|deps| *deps) .flatten() .flat_map(|dep| dep["purl"].as_array()) .flatten() - .cloned() - .collect::>() - .into(); + .flat_map(|purl| purl.as_str().map(|s| s.to_string())) + .sorted() + .collect::>(); assert_eq!( purls, - json!(["pkg:rpm/redhat/B@0.0.0", "pkg:rpm/redhat/EE@0.0.0?arch=src"]) + &["pkg:rpm/redhat/B@0.0.0", "pkg:rpm/redhat/EE@0.0.0?arch=src"] ); assert_eq!(&response["total"], 2); @@ -379,7 +379,7 @@ async fn test_retrieve_query_params_endpoint(ctx: &TrustifyContext) -> Result<() let uri = format!("/api/v2/analysis/root-component?q=sbom_id={}", sbom_id); let request: Request = TestRequest::get().uri(uri.clone().as_str()).to_request(); let response: Value = app.call_and_read_body_json(request).await; - assert_eq!(&response["total"], 8); + assert_eq!(&response["total"], 9); // negative test let uri = @@ -397,6 +397,27 @@ async fn test_retrieve_query_params_endpoint(ctx: &TrustifyContext) -> Result<() Ok(()) } +fn count_deps(response: &Value, filter: F) -> HashMap<&str, usize> +where + F: Fn(&Value) -> bool, +{ + let mut num = HashMap::new(); + + for item in response["items"].as_array().unwrap() { + num.insert( + item["node_id"].as_str().unwrap(), + item["deps"] + .as_array() + .into_iter() + .flatten() + .filter(|f| filter(f)) + .count(), + ); + } + + num +} + #[test_context(TrustifyContext)] #[test(actix_web::test)] async fn issue_tc_2050(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { @@ -411,15 +432,8 @@ async fn issue_tc_2050(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let response: Value = app.call_and_read_body_json(request).await; log::debug!("{response:#?}"); - assert_eq!( - 35, - response["items"][0]["deps"] - .as_array() - .into_iter() - .flatten() - .filter(|m| m["relationship"] == "GeneratedFrom") - .count() - ); + let num = count_deps(&response, |m| m["relationship"] == "GeneratedFrom"); + assert_eq!(num["pkg:rpm/redhat/openssl@3.0.7-18.el9_2?arch=src"], 35); // Ensure binary rpm GeneratedFrom src rpm let x86 = "pkg:rpm/redhat/openssl@3.0.7-18.el9_2?arch=x86_64"; @@ -455,15 +469,9 @@ async fn issue_tc_2051(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let request: Request = TestRequest::get().uri(&uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; log::debug!("{response:#?}"); - assert_eq!( - 35, - response["items"][0]["deps"] - .as_array() - .into_iter() - .flatten() - .filter(|m| m["relationship"] == "GeneratedFrom") - .count() - ); + + let num = count_deps(&response, |m| m["relationship"] == "GeneratedFrom"); + assert_eq!(num["SPDXRef-SRPM"], 35); // Ensure binary rpm GeneratedFrom src rpm let x86 = "pkg:rpm/redhat/openssl@3.0.7-18.el9_2?arch=x86_64"; @@ -499,7 +507,9 @@ async fn issue_tc_2052(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let request: Request = TestRequest::get().uri(&uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; log::debug!("{response:#?}"); - assert_eq!(1, response["items"][0]["deps"].as_array().unwrap().len()); + + let num = count_deps(&response, |_| true); + assert_eq!(num["pkg:oci/openshift-ose-console@sha256%3A94a0d7feec34600a858c8e383ee0e8d5f4a077f6bbc327dcad8762acfcf40679"], 1); // Ensure child is variant of src let child = "pkg:oci/ose-console@sha256:c2d69e860b7457eb42f550ba2559a0452ec3e5c9ff6521d758c186266247678e?arch=s390x&os=linux&tag=v4.14.0-202412110104.p0.g350e1ea.assembly.stream.el8"; @@ -576,32 +586,38 @@ async fn issue_tc_2053(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { #[test_context(TrustifyContext)] #[test(actix_web::test)] async fn issue_tc_2054(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { - use std::str::FromStr; - use trustify_common::purl::Purl; - let app = caller(ctx).await?; ctx.ingest_documents(["cyclonedx/openssl-3.0.7-18.el9_2.cdx_1.6.sbom.json"]) .await?; let parent = "pkg:rpm/redhat/openssl@3.0.7-18.el9_2?arch=src"; - let child = "pkg:generic/openssl@3.0.7?download_url=https://pkgs.devel.redhat.com/repo/openssl/openssl-3.0.7-hobbled.tar.gz/sha512/1aea183b0b6650d9d5e7ba87b613bb1692c71720b0e75377b40db336b40bad780f7e8ae8dfb9f60841eeb4381f4b79c4c5043210c96e7cb51f90791b80c8285e/openssl-3.0.7-hobbled.tar.gz&checksum=SHA-512:1aea183b0b6650d9d5e7ba87b613bb1692c71720b0e75377b40db336b40bad780f7e8ae8dfb9f60841eeb4381f4b79c4c5043210c96e7cb51f90791b80c8285e"; + let child = "pkg:generic/openssl@3.0.7?checksum=SHA-512:1aea183b0b6650d9d5e7ba87b613bb1692c71720b0e75377b40db336b40bad780f7e8ae8dfb9f60841eeb4381f4b79c4c5043210c96e7cb51f90791b80c8285e&download_url=https://pkgs.devel.redhat.com/repo/openssl/openssl-3.0.7-hobbled.tar.gz/sha512/1aea183b0b6650d9d5e7ba87b613bb1692c71720b0e75377b40db336b40bad780f7e8ae8dfb9f60841eeb4381f4b79c4c5043210c96e7cb51f90791b80c8285e/openssl-3.0.7-hobbled.tar.gz"; // Ensure parent has deps that include the child let uri = format!("/api/v2/analysis/dep/{}", urlencoding::encode(parent)); let request: Request = TestRequest::get().uri(&uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; log::debug!("{response:#?}"); - let deps: Vec<_> = response["items"][0]["deps"] + + // get all PURLs of 'AncestorOf' for all dependencies of the parent + let deps: Vec<_> = response["items"] .as_array() .into_iter() .flatten() + // we're only looking for the parent node + .filter(|m| m["node_id"] == parent) + // flatten all dependencies of that parent node + .flat_map(|m| m["deps"].as_array().into_iter().flatten()) + // filter out all non-AncestorOf dependencies .filter(|m| m["relationship"] == "AncestorOf") .collect(); + + // check if there is one dependency of type 'AncestorOf' in the parent package dependencies assert_eq!(1, deps.len()); - assert_eq!( - Purl::from_str(child)?, - Purl::from_str(deps[0]["purl"][0].as_str().unwrap())? - ); + // that dependency must have a single purl + assert_eq!(1, deps[0]["purl"].as_array().unwrap().len()); + // that purl must be the child purl + assert_eq!(child, deps[0]["purl"][0]); // Ensure child has ancestors that include the parent let uri = format!( diff --git a/modules/analysis/src/service/load.rs b/modules/analysis/src/service/load.rs index 8e8e5ee7..69df80e9 100644 --- a/modules/analysis/src/service/load.rs +++ b/modules/analysis/src/service/load.rs @@ -243,6 +243,8 @@ impl AnalysisService { product_version: package.product_version.clone().unwrap_or_default(), }); + log::debug!("Inserting - id: {}, index: {index:?}", entry.key()); + entry.insert(index); } Entry::Occupied(_) => {} @@ -259,10 +261,11 @@ impl AnalysisService { } }; + // the nodes describing the document let mut describedby_node_id: Vec = Default::default(); for edge in edges { - // remove all node IDs we somehow connected + log::debug!("Adding edge {:?}", edge); // insert edge into the graph match ( @@ -271,9 +274,10 @@ impl AnalysisService { ) { (Some(left), Some(right)) => { if edge.relationship == Relationship::DescribedBy { - describedby_node_id.push(*right); + describedby_node_id.push(*left); } + // remove all node IDs we somehow connected detected_nodes.remove(&edge.left_node_id); detected_nodes.remove(&edge.right_node_id); @@ -283,13 +287,17 @@ impl AnalysisService { } } + log::debug!("Describing nodes: {describedby_node_id:?}"); + log::debug!("Unconnected nodes: {detected_nodes:?}"); + if !describedby_node_id.is_empty() { - // search of unconnected nodes and create dummy relationships + // search of unconnected nodes and create undefined relationships // all nodes not removed are unconnected for id in detected_nodes { let Some(id) = nodes.get(&id) else { continue }; // add "undefined" relationship for from in &describedby_node_id { + log::debug!("Creating undefined relationship - left: {id:?}, right: {from:?}"); g.add_edge(*id, *from, Relationship::Undefined); } } diff --git a/modules/analysis/src/service/test.rs b/modules/analysis/src/service/test.rs index 3d0f9f51..b3b35697 100644 --- a/modules/analysis/src/service/test.rs +++ b/modules/analysis/src/service/test.rs @@ -1,6 +1,6 @@ use super::*; -use std::str::FromStr; -use std::time::SystemTime; +use crate::test::*; +use std::{str::FromStr, time::SystemTime}; use test_context::test_context; use test_log::test; use trustify_common::{ @@ -107,29 +107,30 @@ async fn test_simple_by_name_analysis_service(ctx: &TrustifyContext) -> Result<( .retrieve_root_components("B", Paginated::default(), &ctx.db) .await?; - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - vec![Purl::from_str("pkg:rpm/redhat/A@0.0.0?arch=src")?] - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-A".to_string() - ); + assert_ancestors(&analysis_graph.items, |ancestors| { + assert_eq!( + ancestors, + &[&[ + Node { + id: "SPDXRef-A", + name: "A", + version: "1", + cpes: &["cpe:/a:redhat:simple:1:*:el9:*"], + purls: &["pkg:rpm/redhat/A@0.0.0?arch=src"], + }, + Node { + id: "SPDXRef-DOCUMENT", + name: "simple", + version: "", + cpes: &[], + purls: &[], + }, + ]] + ); + }); + assert_eq!(analysis_graph.total, 1); + Ok(()) } @@ -146,28 +147,28 @@ async fn test_simple_by_purl_analysis_service(ctx: &TrustifyContext) -> Result<( .retrieve_root_components(&component_purl, Paginated::default(), &ctx.db) .await?; - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - vec![Purl::from_str("pkg:rpm/redhat/A@0.0.0?arch=src")?] - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-A".to_string() - ); + assert_ancestors(&analysis_graph.items, |ancestors| { + assert_eq!( + ancestors, + [[ + Node { + id: "SPDXRef-A", + name: "A", + version: "1", + purls: &["pkg:rpm/redhat/A@0.0.0?arch=src"], + cpes: &["cpe:/a:redhat:simple:1:*:el9:*"], + }, + Node { + id: "SPDXRef-DOCUMENT", + name: "simple", + version: "", + cpes: &[], + purls: &[], + } + ]] + ); + }); + assert_eq!(analysis_graph.total, 1); Ok(()) } @@ -187,23 +188,36 @@ async fn test_quarkus_analysis_service(ctx: &TrustifyContext) -> Result<(), anyh .retrieve_root_components(&Query::q("spymemcached"), Paginated::default(), &ctx.db) .await?; - assert_eq!( - analysis_graph.items.last().unwrap().ancestors.last().unwrap().purl, - vec![Purl::from_str("pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.12.Final-redhat-00002?type=pom&repository_url=https%3a%2f%2fmaven.repository.redhat.com%2fga%2f")?] - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-e24fec28-1001-499c-827f-2e2e5f2671b5".to_string() - ); + assert_ancestors(&analysis_graph.items, |ancestors| { + assert!( + matches!(ancestors, [ + [..], + [ + Node { + id: "SPDXRef-DOCUMENT", + name: "quarkus-bom-3.2.12.Final-redhat-00002", + version: "", + .. + }, + Node { + id: "SPDXRef-e24fec28-1001-499c-827f-2e2e5f2671b5", + name: "quarkus-bom", + version: "3.2.12.Final-redhat-00002", + cpes: [ + "cpe:/a:redhat:quarkus:3.2:*:el8:*", + ], + purls: [ + "pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.12.Final-redhat-00002?repository_url=https://maven.repository.redhat.com/ga/&type=pom" + ], + }, + ] + ]), + "must match: {ancestors:#?}" + ); + }); assert_eq!(analysis_graph.total, 2); + Ok(()) } @@ -384,7 +398,7 @@ async fn test_circular_deps_spdx_service(ctx: &TrustifyContext) -> Result<(), an #[test_context(TrustifyContext)] #[test(tokio::test)] -async fn test_retrieve_all_sbom_roots_by_name1(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { +async fn test_retrieve_all_sbom_roots_by_name(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { ctx.ingest_documents(["spdx/quarkus-bom-3.2.11.Final-redhat-00001.json"]) .await?; @@ -395,6 +409,8 @@ async fn test_retrieve_all_sbom_roots_by_name1(ctx: &TrustifyContext) -> Result< .retrieve_root_components(&Query::q(&component_name), Paginated::default(), &ctx.db) .await?; + log::debug!("Result: {analysis_graph:#?}"); + let sbom_id = analysis_graph .items .last() @@ -406,7 +422,10 @@ async fn test_retrieve_all_sbom_roots_by_name1(ctx: &TrustifyContext) -> Result< .retrieve_all_sbom_roots_by_name(sbom_id, component_name, &ctx.db) .await?; - assert_eq!(roots.last().unwrap().name, "quarkus-bom"); + assert_eq!( + roots.last().unwrap().name, + "quarkus-bom-3.2.11.Final-redhat-00001" + ); Ok(()) } diff --git a/modules/analysis/src/test.rs b/modules/analysis/src/test.rs index 0dc5f857..74893bb4 100644 --- a/modules/analysis/src/test.rs +++ b/modules/analysis/src/test.rs @@ -1,4 +1,8 @@ -use crate::endpoints::configure; +use crate::{ + endpoints::configure, + model::{AncNode, AncestorSummary}, +}; +use itertools::Itertools; use trustify_test_context::{ call::{self, CallService}, TrustifyContext, @@ -7,3 +11,98 @@ use trustify_test_context::{ pub async fn caller(ctx: &TrustifyContext) -> anyhow::Result { call::caller(|svc| configure(svc, ctx.db.clone())).await } + +#[derive(PartialEq, Eq, Debug, Copy, Clone)] +pub struct Node<'a> { + pub id: &'a str, + pub name: &'a str, + pub version: &'a str, + + pub cpes: &'a [&'a str], + pub purls: &'a [&'a str], +} + +#[derive(PartialEq, Eq, Debug, Clone)] +struct OwnedNode<'a> { + pub id: &'a str, + pub name: &'a str, + pub version: &'a str, + + pub cpes: Vec, + pub purls: Vec, +} + +#[derive(PartialEq, Eq, Debug, Clone)] +struct RefNode<'a> { + pub id: &'a str, + pub name: &'a str, + pub version: &'a str, + + pub cpes: Vec<&'a str>, + pub purls: Vec<&'a str>, +} + +impl<'a> From<&'a AncNode> for OwnedNode<'a> { + fn from(value: &'a AncNode) -> Self { + Self { + id: &value.node_id, + name: &value.name, + version: &value.version, + cpes: value.cpe.iter().map(ToString::to_string).collect(), + purls: value.purl.iter().map(ToString::to_string).collect(), + } + } +} + +pub fn assert_ancestors(ancestors: &[AncestorSummary], f: F) +where + F: for<'a> FnOnce(&'a [&'a [Node]]), +{ + let ancestors = ancestors + .iter() + .sorted_by_key(|a| &a.node_id) + .map(|item| { + item.ancestors + .iter() + .map(|anc| OwnedNode::from(anc)) + .sorted_by_key(|n| n.id.to_string()) + .collect::>() + }) + .collect::>(); + + let ancestors = ancestors + .iter() + .map(|a| { + a.iter() + .map(|node| RefNode { + id: node.id, + name: node.name, + version: node.version, + cpes: node.cpes.iter().map(|s| s.as_str()).collect(), + purls: node.purls.iter().map(|s| s.as_str()).collect(), + }) + .collect::>() + }) + .collect::>(); + + let ancestors = ancestors + .iter() + .map(|a| { + a.iter() + .map(|node| Node { + id: node.id, + name: node.name, + version: node.version, + cpes: node.cpes.as_slice(), + purls: node.purls.as_slice(), + }) + .collect::>() + }) + .collect::>(); + + let ancestors = ancestors.iter().map(|a| a.as_slice()).collect::>(); + + log::debug!("Ancestors: {ancestors:#?}"); + + f(ancestors.as_slice()) +}