From 7d6576d38f56dcbac77e741b2e74ad5b658ac076 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 13:04:44 +0000 Subject: [PATCH 1/5] Implement Graph struct This is the main graph implemented in Rust - but it isn't plumbed into Python yet. --- rust/Cargo.lock | 46 + rust/Cargo.toml | 2 + rust/src/graph.rs | 2928 +++++++++++++++++++++++++++++++++++++++++++++ rust/src/lib.rs | 1 + 4 files changed, 2977 insertions(+) create mode 100644 rust/src/graph.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index faeb0261..c9b0a5fe 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6,7 +6,9 @@ version = 3 name = "_rustgrimp" version = "0.1.0" dependencies = [ + "bimap", "log", + "petgraph", "pyo3", "pyo3-log", "rayon", @@ -25,6 +27,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "cfg-if" version = "1.0.0" @@ -62,12 +70,40 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.5" @@ -107,6 +143,16 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "portable-atomic" version = "1.6.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index fca4e896..3ab4be53 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -12,6 +12,8 @@ log = "0.4.19" pyo3-log = "0.11.0" serde_json = "1.0.103" rayon = "1.10" +petgraph = "0.6.5" +bimap = "0.6.3" [dependencies.pyo3] version = "0.22.4" diff --git a/rust/src/graph.rs b/rust/src/graph.rs new file mode 100644 index 00000000..0de65ec3 --- /dev/null +++ b/rust/src/graph.rs @@ -0,0 +1,2928 @@ +/* +Also, sensible behaviour when passing modules that don't exist in the graph. +*/ +#![allow(dead_code)] + +use bimap::BiMap; +use log::info; +use petgraph::algo::astar; +use petgraph::graph::EdgeIndex; +use petgraph::stable_graph::{NodeIndex, StableGraph}; +use petgraph::visit::{Bfs, Walker}; +use petgraph::Direction; +use rayon::prelude::*; +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::time::Instant; + +/// A group of layers at the same level in the layering. +#[derive(PartialEq, Eq, Hash, Debug)] +pub struct Level { + pub layers: Vec, + pub independent: bool, +} + +// Delimiter for Python modules. +const DELIMITER: char = '.'; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Module { + pub name: String, +} + +impl fmt::Display for Module { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.name) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ModuleNotPresent { + pub module: Module, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct NoSuchContainer { + pub container: String, +} + +pub struct ModulesHaveSharedDescendants {} + +impl fmt::Display for ModuleNotPresent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "\"{}\" not present in the graph", self.module.name) + } +} + +impl Module { + pub fn new(name: String) -> Module { + Module { name } + } + + // Returns whether the module is a root-level package. + pub fn is_root(&self) -> bool { + !self.name.contains(DELIMITER) + } + + // Create a Module that is the parent of the passed Module. + // + // Panics if the child is a root Module. + pub fn new_parent(child: &Module) -> Module { + let parent_name = match child.name.rsplit_once(DELIMITER) { + Some((base, _)) => base.to_string(), + None => panic!("{} is a root level package", child.name), + }; + + Module::new(parent_name) + } + + // Return whether this module is a descendant of the supplied one, based on the name. + pub fn is_descendant_of(&self, module: &Module) -> bool { + let candidate = format!("{}{}", module.name, DELIMITER); + self.name.starts_with(&candidate) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DetailedImport { + pub importer: Module, + pub imported: Module, + pub line_number: usize, + pub line_contents: String, +} + +#[derive(Default, Clone)] +pub struct Graph { + // Bidirectional lookup between Module and NodeIndex. + hierarchy_module_indices: BiMap, + hierarchy: StableGraph, + imports_module_indices: BiMap, + imports: StableGraph, + squashed_modules: HashSet, + // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. + invisible_modules: HashSet, + detailed_imports_map: HashMap<(Module, Module), HashSet>, +} + +#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] +pub struct Route { + pub heads: Vec, + pub middle: Vec, + pub tails: Vec, +} + +#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] +pub struct PackageDependency { + pub importer: Module, + pub imported: Module, + pub routes: Vec, +} + +fn _module_from_layer(layer: &str, container: &Option) -> Module { + let module_name = match container { + Some(container) => format!("{}{}{}", container.name, DELIMITER, layer), + None => layer.to_string(), + }; + Module::new(module_name) +} + +fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { + let route_count = match dependency_or_none { + Some(dependency) => dependency.routes.len(), + None => 0, + }; + let pluralized = if route_count == 1 { "" } else { "s" }; + info!( + "Found {} illegal route{} in {}s.", + route_count, pluralized, duration_in_s + ); +} + +impl Graph { + pub fn pretty_str(&self) -> String { + let mut hierarchy: Vec = vec![]; + let mut imports: Vec = vec![]; + + let hierarchy_module_indices: Vec<_> = self.hierarchy_module_indices.iter().collect(); + + for (parent_module, parent_index) in hierarchy_module_indices { + for child_index in self.hierarchy.neighbors(*parent_index) { + let child_module = self + .hierarchy_module_indices + .get_by_right(&child_index) + .unwrap(); + let parent_module_str = match self.invisible_modules.contains(&parent_module) { + true => format!("({})", parent_module.name), + false => parent_module.name.to_string(), + }; + let child_module_str = match self.invisible_modules.contains(&child_module) { + true => format!("({})", child_module.name), + false => child_module.name.to_string(), + }; + hierarchy.push(format!(" {} -> {}", parent_module_str, child_module_str)); + } + } + + let imports_module_indices: Vec<_> = self.imports_module_indices.iter().collect(); + + for (from_module, from_index) in imports_module_indices { + for to_index in self.imports.neighbors(*from_index) { + let to_module = self.imports_module_indices.get_by_right(&to_index).unwrap(); + imports.push(format!(" {} -> {}", from_module.name, to_module.name)); + } + } + // Assemble String. + let mut pretty = String::new(); + pretty.push_str("hierarchy:\n"); + hierarchy.sort(); + pretty.push_str(&hierarchy.join("\n")); + pretty.push_str("\nimports:\n"); + imports.sort(); + pretty.push_str(&imports.join("\n")); + pretty.push('\n'); + pretty + } + + pub fn add_module(&mut self, module: Module) { + // If this module is already in the graph, but invisible, just make it visible. + if self.invisible_modules.contains(&module) { + self.invisible_modules.remove(&module); + return; + } + // If this module is already in the graph, don't do anything. + if self.hierarchy_module_indices.get_by_left(&module).is_some() { + return; + } + + let module_index = self.hierarchy.add_node(module.clone()); + self.hierarchy_module_indices + .insert(module.clone(), module_index); + + // Add this module to the hierarchy. + if !module.is_root() { + let parent = Module::new_parent(&module); + + let parent_index = match self.hierarchy_module_indices.get_by_left(&parent) { + Some(index) => index, + None => { + // If the parent isn't already in the graph, add it, but as an invisible module. + self.add_module(parent.clone()); + self.invisible_modules.insert(parent.clone()); + self.hierarchy_module_indices.get_by_left(&parent).unwrap() + } + }; + self.hierarchy.add_edge(*parent_index, module_index, ()); + } + } + + pub fn add_squashed_module(&mut self, module: Module) { + self.add_module(module.clone()); + self.squashed_modules.insert(module); + } + + pub fn remove_module(&mut self, module: &Module) { + // Remove imports by module. + let imported_modules: Vec = self + .find_modules_directly_imported_by(module) + .iter() + .map(|m| (*m).clone()) + .collect(); + for imported_module in imported_modules { + self.remove_import(&module, &imported_module); + } + + // Remove imports of module. + let importer_modules: Vec = self + .find_modules_that_directly_import(module) + .iter() + .map(|m| (*m).clone()) + .collect(); + for importer_module in importer_modules { + self.remove_import(&importer_module, &module); + } + + // Remove module from hierarchy. + if let Some(hierarchy_index) = self.hierarchy_module_indices.get_by_left(module) { + // TODO should we check for children before removing? + // Maybe should just make invisible instead? + self.hierarchy.remove_node(*hierarchy_index); + self.hierarchy_module_indices.remove_by_left(module); + }; + } + + pub fn get_modules(&self) -> HashSet<&Module> { + self.hierarchy_module_indices + .left_values() + .filter(|module| !self.invisible_modules.contains(module)) + .collect() + } + + pub fn count_imports(&self) -> usize { + self.imports.edge_count() + } + + pub fn get_import_details( + &self, + importer: &Module, + imported: &Module, + ) -> HashSet { + let key = (importer.clone(), imported.clone()); + match self.detailed_imports_map.get(&key) { + Some(import_details) => import_details.clone(), + None => HashSet::new(), + } + } + + pub fn find_children(&self, module: &Module) -> HashSet<&Module> { + if self.invisible_modules.contains(module) { + return HashSet::new(); + } + let module_index = match self.hierarchy_module_indices.get_by_left(module) { + Some(index) => index, + // Module does not exist. + // TODO: should this return a result, to handle if module is not in graph? + None => return HashSet::new(), + }; + self.hierarchy + .neighbors(*module_index) + .map(|index| self.hierarchy_module_indices.get_by_right(&index).unwrap()) + .filter(|module| !self.invisible_modules.contains(module)) + .collect() + } + + pub fn find_descendants(&self, module: &Module) -> Result, ModuleNotPresent> { + let module_index = match self.hierarchy_module_indices.get_by_left(module) { + Some(index) => index, + None => { + return Err(ModuleNotPresent { + module: module.clone(), + }) + } + }; + Ok(Bfs::new(&self.hierarchy, *module_index) + .iter(&self.hierarchy) + .filter(|index| index != module_index) // Don't include the supplied module. + .map(|index| self.hierarchy_module_indices.get_by_right(&index).unwrap()) // This panics sometimes. + .filter(|module| !self.invisible_modules.contains(module)) + .collect()) + } + + pub fn add_import(&mut self, importer: &Module, imported: &Module) { + // Don't bother doing anything if it's already in the graph. + if self.direct_import_exists(&importer, &imported, false) { + return; + } + + self.add_module_if_not_in_hierarchy(importer); + self.add_module_if_not_in_hierarchy(imported); + + let importer_index: NodeIndex = match self.imports_module_indices.get_by_left(importer) { + Some(index) => *index, + None => { + let index = self.imports.add_node(importer.clone()); + self.imports_module_indices.insert(importer.clone(), index); + index + } + }; + let imported_index: NodeIndex = match self.imports_module_indices.get_by_left(imported) { + Some(index) => *index, + None => { + let index = self.imports.add_node(imported.clone()); + self.imports_module_indices.insert(imported.clone(), index); + index + } + }; + + self.imports.add_edge(importer_index, imported_index, ()); + // println!( + // "Added {:?} {:?} -> {:?} {:?}, edge count now {:?}", + // importer, + // importer_index, + // imported, + // imported_index, + // self.imports.edge_count() + // ); + } + + pub fn add_detailed_import(&mut self, import: &DetailedImport) { + let key = (import.importer.clone(), import.imported.clone()); + self.detailed_imports_map + .entry(key) + .or_insert_with(HashSet::new) + .insert(import.clone()); + self.add_import(&import.importer, &import.imported); + } + + pub fn remove_import(&mut self, importer: &Module, imported: &Module) { + let importer_index: NodeIndex = match self.imports_module_indices.get_by_left(importer) { + Some(index) => *index, + None => return, + }; + let imported_index: NodeIndex = match self.imports_module_indices.get_by_left(imported) { + Some(index) => *index, + None => return, + }; + let edge_index: EdgeIndex = match self.imports.find_edge(importer_index, imported_index) { + Some(index) => index, + None => return, + }; + + self.imports.remove_edge(edge_index); + + // There might be other imports to / from the modules, so don't + // remove from the indices. (TODO: does it matter if we don't clean these up + // if there are no more imports?) + // self.imports_module_indices.remove_by_left(importer); + // self.imports_module_indices.remove_by_left(importer); + + let key = (importer.clone(), imported.clone()); + + self.detailed_imports_map.remove(&key); + self.imports.remove_edge(edge_index); + } + + // Note: this will panic if importer and imported are in the same package. + #[allow(unused_variables)] + pub fn direct_import_exists( + &self, + importer: &Module, + imported: &Module, + as_packages: bool, + ) -> bool { + let graph_to_use: &Graph; + let mut graph_copy: Graph; + + if as_packages { + graph_copy = self.clone(); + graph_copy.squash_module(importer); + graph_copy.squash_module(imported); + graph_to_use = &graph_copy; + } else { + graph_to_use = self; + } + + // The modules may appear in the hierarchy, but have no imports, so we + // return false unless they're both in there. + let importer_index = match graph_to_use.imports_module_indices.get_by_left(importer) { + Some(importer_index) => *importer_index, + None => return false, + }; + let imported_index = match graph_to_use.imports_module_indices.get_by_left(imported) { + Some(imported_index) => *imported_index, + None => return false, + }; + + graph_to_use + .imports + .contains_edge(importer_index, imported_index) + } + + pub fn find_modules_that_directly_import(&self, imported: &Module) -> HashSet<&Module> { + let imported_index = match self.imports_module_indices.get_by_left(imported) { + Some(imported_index) => *imported_index, + None => return HashSet::new(), + }; + let importer_indices: HashSet = self + .imports + .neighbors_directed(imported_index, Direction::Incoming) + .collect(); + + let importers: HashSet<&Module> = importer_indices + .iter() + .map(|importer_index| { + self.imports_module_indices + .get_by_right(importer_index) + .unwrap() + }) + .collect(); + importers + } + + pub fn find_modules_directly_imported_by(&self, importer: &Module) -> HashSet<&Module> { + let importer_index = match self.imports_module_indices.get_by_left(importer) { + Some(importer_index) => *importer_index, + None => return HashSet::new(), + }; + let imported_indices: HashSet = self + .imports + .neighbors_directed(importer_index, Direction::Outgoing) + .collect(); + + let importeds: HashSet<&Module> = imported_indices + .iter() + .map(|imported_index| { + self.imports_module_indices + .get_by_right(imported_index) + .unwrap() + }) + .collect(); + importeds + } + + pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { + let mut upstream_modules = HashSet::new(); + + let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + if as_package { + let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + modules_to_check.extend(descendants.into_iter()); + }; + + for module_to_check in modules_to_check.iter() { + let module_index = match self.imports_module_indices.get_by_left(module_to_check) { + Some(index) => *index, + None => continue, + }; + upstream_modules.extend( + Bfs::new(&self.imports, module_index) + .iter(&self.imports) + .map(|index| self.imports_module_indices.get_by_right(&index).unwrap()) + // Exclude any modules that we are checking. + .filter(|downstream_module| !modules_to_check.contains(downstream_module)), + ); + } + + upstream_modules + } + + pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { + let mut downstream_modules = HashSet::new(); + + let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + if as_package { + let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + modules_to_check.extend(descendants.into_iter()); + }; + + for module_to_check in modules_to_check.iter() { + let module_index = match self.imports_module_indices.get_by_left(module_to_check) { + Some(index) => *index, + None => continue, + }; + + // Reverse all the edges in the graph and then do what we do in find_upstream_modules. + // Is there a way of doing this without the clone? + let mut reversed_graph = self.imports.clone(); + reversed_graph.reverse(); + + downstream_modules.extend( + Bfs::new(&reversed_graph, module_index) + .iter(&reversed_graph) + .map(|index| self.imports_module_indices.get_by_right(&index).unwrap()) + // Exclude any modules that we are checking. + .filter(|downstream_module| !modules_to_check.contains(downstream_module)), + ) + } + + downstream_modules + } + + pub fn find_shortest_chain( + &self, + importer: &Module, + imported: &Module, + ) -> Option> { + let importer_index = match self.imports_module_indices.get_by_left(importer) { + Some(index) => *index, + None => return None, // Importer has no imports to or from. + }; + let imported_index = match self.imports_module_indices.get_by_left(imported) { + Some(index) => *index, + None => return None, // Imported has no imports to or from. + }; + let path_to_imported = match astar( + &self.imports, + importer_index, + |finish| finish == imported_index, + |_e| 1, + |_| 0, + ) { + Some(path_tuple) => path_tuple.1, + None => return None, // No chain to the imported. + }; + + let mut chain: Vec<&Module> = vec![]; + for link_index in path_to_imported { + let module = self + .imports_module_indices + .get_by_right(&link_index) + .unwrap(); + chain.push(module); + } + Some(chain) + } + + // https://github.com/seddonym/grimp/blob/master/src/grimp/adaptors/graph.py#L290 + pub fn find_shortest_chains( + &self, + importer: &Module, + imported: &Module, + as_packages: bool, + ) -> Result>, String> { + let mut chains = HashSet::new(); + let mut temp_graph = self.clone(); + + let mut downstream_modules: HashSet = HashSet::from([importer.clone()]); + let mut upstream_modules: HashSet = HashSet::from([imported.clone()]); + + // TODO don't do this if module is squashed? + if as_packages { + for descendant in self.find_descendants(importer).unwrap() { + downstream_modules.insert(descendant.clone()); + } + for descendant in self.find_descendants(imported).unwrap() { + upstream_modules.insert(descendant.clone()); + } + if upstream_modules + .intersection(&downstream_modules) + .next() + .is_some() + { + return Err("Modules have shared descendants.".to_string()); + } + } + + // Remove imports within the packages. + let mut imports_to_remove: Vec<(Module, Module)> = vec![]; + for upstream_module in &upstream_modules { + for imported_module in temp_graph.find_modules_directly_imported_by(&upstream_module) { + if upstream_modules.contains(&imported_module) { + imports_to_remove.push((upstream_module.clone(), imported_module.clone())); + } + } + } + for downstream_module in &downstream_modules { + for imported_module in temp_graph.find_modules_directly_imported_by(&downstream_module) + { + if downstream_modules.contains(&imported_module) { + imports_to_remove.push((downstream_module.clone(), imported_module.clone())); + } + } + } + for (importer_to_remove, imported_to_remove) in imports_to_remove { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + + // Keep track of imports into/out of upstream/downstream packages, and remove them. + let mut map_of_imports: HashMap> = HashMap::new(); + for module in upstream_modules.union(&downstream_modules) { + let mut imports_to_or_from_module = HashSet::new(); + for imported_module in temp_graph.find_modules_directly_imported_by(&module) { + imports_to_or_from_module.insert((module.clone(), imported_module.clone())); + } + for importer_module in temp_graph.find_modules_that_directly_import(&module) { + imports_to_or_from_module.insert((importer_module.clone(), module.clone())); + } + map_of_imports.insert(module.clone(), imports_to_or_from_module); + } + for imports in map_of_imports.values() { + for (importer_to_remove, imported_to_remove) in imports { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + } + + for importer_module in &downstream_modules { + // Reveal imports to/from importer module. + for (importer_to_add, imported_to_add) in &map_of_imports[&importer_module] { + temp_graph.add_import(&importer_to_add, &imported_to_add); + } + for imported_module in &upstream_modules { + // Reveal imports to/from imported module. + for (importer_to_add, imported_to_add) in &map_of_imports[&imported_module] { + temp_graph.add_import(&importer_to_add, &imported_to_add); + } + if let Some(chain) = + temp_graph.find_shortest_chain(importer_module, imported_module) + { + chains.insert(chain.iter().cloned().map(|module| module.clone()).collect()); + } + // Remove imports relating to imported module again. + for (importer_to_remove, imported_to_remove) in &map_of_imports[&imported_module] { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + } + // Remove imports relating to importer module again. + for (importer_to_remove, imported_to_remove) in &map_of_imports[&importer_module] { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + } + Ok(chains) + } + + #[allow(unused_variables)] + pub fn chain_exists(&self, importer: &Module, imported: &Module, as_packages: bool) -> bool { + // TODO should this return a Result, so we can handle the situation the importer / imported + // having shared descendants when as_packages=true? + let mut temp_graph; + let graph = match as_packages { + true => { + temp_graph = self.clone(); + temp_graph.squash_module(importer); + temp_graph.squash_module(imported); + &temp_graph + } + false => self, + }; + graph.find_shortest_chain(importer, imported).is_some() + } + + #[allow(unused_variables)] + pub fn find_illegal_dependencies_for_layers( + &self, + levels: Vec, + containers: HashSet, + ) -> Result, NoSuchContainer> { + // Check that containers exist. + let modules = self.get_modules(); + for container in containers.iter() { + let container_module = Module::new(container.clone()); + if !modules.contains(&container_module) { + return Err(NoSuchContainer { + container: container.clone(), + }); + } + } + + let all_layers: Vec = levels + .iter() + .flat_map(|level| level.layers.iter()) + .map(|module_name| module_name.to_string()) + .collect(); + + let perms = self._generate_module_permutations(&levels, &containers); + + let mut dependencies: Vec = self + ._generate_module_permutations(&levels, &containers) + //.into_iter() + .into_par_iter() + .filter_map(|(higher_layer_package, lower_layer_package, container)| { + // TODO: it's inefficient to do this for sibling layers, as we don't need + // to clone and trim the graph for identical pairs. + info!( + "Searching for import chains from {} to {}...", + lower_layer_package, higher_layer_package + ); + let now = Instant::now(); + let dependency_or_none = self._search_for_package_dependency( + &higher_layer_package, + &lower_layer_package, + &all_layers, + &container, + ); + _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); + dependency_or_none + }) + .collect(); + + dependencies.sort(); + + Ok(dependencies) + } + + // Return every permutation of modules that exist in the graph + /// in which the second should not import the first. + /// The third item in the tuple is the relevant container, if used. + fn _generate_module_permutations( + &self, + levels: &Vec, + containers: &HashSet, + ) -> Vec<(Module, Module, Option)> { + let mut permutations: Vec<(Module, Module, Option)> = vec![]; + + let quasi_containers: Vec> = if containers.is_empty() { + vec![None] + } else { + containers + .iter() + .map(|i| Some(Module::new(i.to_string()))) + .collect() + }; + let all_modules = self.get_modules(); + + for quasi_container in quasi_containers { + for (index, higher_level) in levels.iter().enumerate() { + for higher_layer in &higher_level.layers { + let higher_layer_module = _module_from_layer(&higher_layer, &quasi_container); + if !all_modules.contains(&higher_layer_module) { + continue; + } + + // Build the layers that mustn't import this higher layer. + // That includes: + // * lower layers. + // * sibling layers, if the layer is independent. + let mut layers_forbidden_to_import_higher_layer: Vec = vec![]; + + // Independence + if higher_level.independent { + for potential_sibling_layer in &higher_level.layers { + let sibling_module = + _module_from_layer(&potential_sibling_layer, &quasi_container); + if sibling_module != higher_layer_module + && all_modules.contains(&sibling_module) + { + layers_forbidden_to_import_higher_layer.push(sibling_module); + } + } + } + + for lower_level in &levels[index + 1..] { + for lower_layer in &lower_level.layers { + let lower_layer_module = + _module_from_layer(&lower_layer, &quasi_container); + if all_modules.contains(&lower_layer_module) { + layers_forbidden_to_import_higher_layer.push(lower_layer_module); + } + } + } + + // Add to permutations. + for forbidden in layers_forbidden_to_import_higher_layer { + permutations.push(( + higher_layer_module.clone(), + forbidden.clone(), + quasi_container.clone(), + )); + } + } + } + } + + permutations + } + + fn _search_for_package_dependency( + &self, + higher_layer_package: &Module, + lower_layer_package: &Module, + layers: &Vec, + container: &Option, + ) -> Option { + let mut temp_graph = self.clone(); + + // Remove other layers. + let mut modules_to_remove: Vec = vec![]; + for layer in layers { + let layer_module = _module_from_layer(&layer, &container); + if layer_module != *higher_layer_package && layer_module != *lower_layer_package { + // Remove this subpackage. + match temp_graph.find_descendants(&layer_module) { + Ok(descendants) => { + for descendant in descendants { + modules_to_remove.push(descendant.clone()) + } + } + Err(_) => (), // ModuleNotPresent. + } + modules_to_remove.push(layer_module.clone()); + } + } + for module_to_remove in modules_to_remove.clone() { + temp_graph.remove_module(&module_to_remove); + } + + let mut routes: Vec = vec![]; + + // Direct routes. + // TODO: do we need to pop the imports? + // The indirect routes should cope without removing them? + let direct_links = + temp_graph._pop_direct_imports(lower_layer_package, higher_layer_package); + for (importer, imported) in direct_links { + routes.push(Route { + heads: vec![importer], + middle: vec![], + tails: vec![imported], + }); + } + + // Indirect routes. + for indirect_route in + temp_graph._find_indirect_routes(lower_layer_package, higher_layer_package) + { + routes.push(indirect_route); + } + + if routes.is_empty() { + None + } else { + Some(PackageDependency { + importer: lower_layer_package.clone(), + imported: higher_layer_package.clone(), + routes, + }) + } + } + + fn _find_indirect_routes( + &self, + importer_package: &Module, + imported_package: &Module, + ) -> Vec { + let mut routes = vec![]; + + let mut temp_graph = self.clone(); + temp_graph.squash_module(importer_package); + temp_graph.squash_module(imported_package); + + // Find middles. + let mut middles: Vec> = vec![]; + for chain in temp_graph._pop_shortest_chains(importer_package, imported_package) { + // Remove first and last element. + let mut middle: Vec = vec![]; + let chain_length = chain.len(); + for (index, module) in chain.iter().enumerate() { + if index != 0 && index != chain_length - 1 { + middle.push(module.clone()); + } + } + middles.push(middle); + } + + // Set up importer/imported package contents. + let mut importer_modules: HashSet<&Module> = HashSet::from([importer_package]); + importer_modules.extend(self.find_descendants(&importer_package).unwrap()); + let mut imported_modules: HashSet<&Module> = HashSet::from([imported_package]); + imported_modules.extend(self.find_descendants(&imported_package).unwrap()); + + // Build routes from middles. + for middle in middles { + // Construct heads. + let mut heads: Vec = vec![]; + let first_imported_module = &middle[0]; + for candidate_head in self.find_modules_that_directly_import(&first_imported_module) { + if importer_modules.contains(candidate_head) { + heads.push(candidate_head.clone()); + } + } + + // Construct tails. + let mut tails: Vec = vec![]; + let last_importer_module = &middle[middle.len() - 1]; + for candidate_tail in self.find_modules_directly_imported_by(&last_importer_module) { + if imported_modules.contains(candidate_tail) { + tails.push(candidate_tail.clone()); + } + } + + routes.push(Route { + heads, + middle, + tails, + }) + } + + routes + } + + fn _pop_shortest_chains(&mut self, importer: &Module, imported: &Module) -> Vec> { + let mut chains = vec![]; + + loop { + // TODO - defend against infinite loops somehow. + + let found_chain: Vec; + { + let chain = self.find_shortest_chain(importer, imported); + + if chain.is_none() { + break; + } + + found_chain = chain.unwrap().into_iter().cloned().collect(); + } + // Remove chain. + for i in 0..found_chain.len() - 1 { + self.remove_import(&found_chain[i], &found_chain[i + 1]); + } + chains.push(found_chain); + } + chains + } + + /// Remove the direct imports, returning them as (importer, imported) tuples. + fn _pop_direct_imports( + &mut self, + lower_layer_module: &Module, + higher_layer_module: &Module, + ) -> HashSet<(Module, Module)> { + let mut imports = HashSet::new(); + + let mut lower_layer_modules = HashSet::from([lower_layer_module.clone()]); + for descendant in self + .find_descendants(lower_layer_module) + .unwrap() + .iter() + .cloned() + { + lower_layer_modules.insert(descendant.clone()); + } + + let mut higher_layer_modules = HashSet::from([higher_layer_module.clone()]); + for descendant in self + .find_descendants(higher_layer_module) + .unwrap() + .iter() + .cloned() + { + higher_layer_modules.insert(descendant.clone()); + } + + for lower_layer_module in lower_layer_modules { + for imported_module in self.find_modules_directly_imported_by(&lower_layer_module) { + if higher_layer_modules.contains(imported_module) { + imports.insert((lower_layer_module.clone(), imported_module.clone())); + } + } + } + + // Remove imports. + for (importer, imported) in &imports { + self.remove_import(&importer, &imported) + } + + imports + } + + #[allow(unused_variables)] + pub fn squash_module(&mut self, module: &Module) { + // Get descendants and their imports. + let descendants: Vec = self + .find_descendants(module) + .unwrap() + .into_iter() + .cloned() + .collect(); + let modules_imported_by_descendants: Vec = descendants + .iter() + .flat_map(|descendant| { + self.find_modules_directly_imported_by(descendant) + .into_iter() + .cloned() + }) + .collect(); + let modules_that_import_descendants: Vec = descendants + .iter() + .flat_map(|descendant| { + self.find_modules_that_directly_import(descendant) + .into_iter() + .cloned() + }) + .collect(); + + // Remove any descendants. + for descendant in descendants { + self.remove_module(&descendant); + } + + // Add descendants and imports to parent module. + for imported in modules_imported_by_descendants { + self.add_import(module, &imported); + } + + for importer in modules_that_import_descendants { + self.add_import(&importer, module); + } + + self.squashed_modules.insert(module.clone()); + } + + pub fn is_module_squashed(&self, module: &Module) -> bool { + self.squashed_modules.contains(module) + } + + fn add_module_if_not_in_hierarchy(&mut self, module: &Module) { + if self.hierarchy_module_indices.get_by_left(module).is_none() { + self.add_module(module.clone()); + }; + if self.invisible_modules.contains(&module) { + self.invisible_modules.remove(&module); + }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn modules_when_empty() { + let graph = Graph::default(); + + assert_eq!(graph.get_modules(), HashSet::new()); + } + + #[test] + fn module_is_value_object() { + assert_eq!( + Module::new("mypackage".to_string()), + Module::new("mypackage".to_string()) + ); + } + + #[test] + fn add_module() { + let mypackage = Module::new("mypackage".to_string()); + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + + let result = graph.get_modules(); + + assert_eq!(result, HashSet::from([&mypackage])); + } + + #[test] + fn add_module_doesnt_add_parent() { + let mypackage = Module::new("mypackage.foo".to_string()); + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + + let result = graph.get_modules(); + + assert_eq!(result, HashSet::from([&mypackage])); + } + + #[test] + fn add_modules() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + + let result = graph.get_modules(); + + assert_eq!(result, HashSet::from([&mypackage, &mypackage_foo])); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.foo +imports: + +" + .trim_start() + ); + } + + #[test] + fn remove_nonexistent_module() { + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mut graph = Graph::default(); + // Add mypackage but not mypackage.foo. + graph.add_module(mypackage.clone()); + + graph.remove_module(&mypackage_foo); + + let result = graph.get_modules(); + assert_eq!(result, HashSet::from([&mypackage])); + } + + #[test] + fn remove_existing_module_without_imports() { + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + + graph.remove_module(&mypackage_foo); + + let result = graph.get_modules(); + assert_eq!( + result, + HashSet::from([ + &mypackage, + &mypackage_foo_alpha, // To be consistent with previous versions of Grimp. + ]) + ); + } + + #[test] + fn remove_existing_module_with_imports() { + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_import(&importer, &mypackage_foo); + graph.add_import(&mypackage_foo, &imported); + + graph.remove_module(&mypackage_foo); + + let result = graph.get_modules(); + assert_eq!( + result, + HashSet::from([&mypackage, &mypackage_foo_alpha, &importer, &imported]) + ); + assert_eq!( + graph.direct_import_exists(&importer, &mypackage_foo, false), + false + ); + assert_eq!( + graph.direct_import_exists(&mypackage_foo, &imported, false), + false + ); + } + + #[test] + fn remove_importer_module_removes_import_details() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_detailed_import(&DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 99, + line_contents: "-".to_string(), + }); + + graph.remove_module(&importer); + + assert_eq!( + graph.get_import_details(&importer, &imported), + HashSet::new() + ); + } + + #[test] + fn remove_imported_module_removes_import_details() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_detailed_import(&DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 99, + line_contents: "-".to_string(), + }); + + graph.remove_module(&imported); + + assert_eq!( + graph.get_import_details(&importer, &imported), + HashSet::new() + ); + } + + #[test] + fn remove_import_that_exists() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_import(&importer, &imported); + + graph.remove_import(&importer, &imported); + + // The import has gone... + assert_eq!( + graph.direct_import_exists(&importer, &imported, false), + false + ); + // ...but the modules are still there. + assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + } + + #[test] + fn remove_import_does_nothing_if_import_doesnt_exist() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_module(importer.clone()); + graph.add_module(imported.clone()); + + graph.remove_import(&importer, &imported); + + // The modules are still there. + assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + } + + #[test] + fn remove_import_does_nothing_if_modules_dont_exist() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + + graph.remove_import(&importer, &imported); + } + + #[test] + fn remove_import_doesnt_affect_other_imports_from_same_modules() { + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let yellow = Module::new("yellow".to_string()); + let red = Module::new("red".to_string()); + let mut graph = Graph::default(); + graph.add_import(&blue, &green); + graph.add_import(&blue, &yellow); + graph.add_import(&red, &blue); + + graph.remove_import(&blue, &green); + + // The other imports are still there. + assert_eq!(graph.direct_import_exists(&blue, &yellow, false), true); + assert_eq!(graph.direct_import_exists(&red, &blue, false), true); + } + + #[test] + #[should_panic(expected = "rootpackage is a root level package")] + fn new_parent_root_module() { + let root = Module::new("rootpackage".to_string()); + + Module::new_parent(&root); + } + + #[test] + fn is_root_true() { + let root = Module::new("rootpackage".to_string()); + + assert!(root.is_root()); + } + + #[test] + fn is_descendant_of_true_for_child() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar = Module::new("mypackage.foo.bar".to_string()); + + assert!(foo_bar.is_descendant_of(&foo)); + } + + #[test] + fn is_descendant_of_false_for_parent() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar = Module::new("mypackage.foo.bar".to_string()); + + assert_eq!(foo.is_descendant_of(&foo_bar), false); + } + + #[test] + fn is_descendant_of_true_for_grandchild() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar_baz = Module::new("mypackage.foo.bar.baz".to_string()); + + assert!(foo_bar_baz.is_descendant_of(&foo)); + } + + #[test] + fn is_descendant_of_false_for_grandparent() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar_baz = Module::new("mypackage.foo.bar.baz".to_string()); + + assert_eq!(foo.is_descendant_of(&foo_bar_baz), false); + } + + #[test] + fn is_root_false() { + let non_root = Module::new("rootpackage.blue".to_string()); + + assert_eq!(non_root.is_root(), false); + } + + #[test] + fn find_children_no_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + + assert_eq!(graph.find_children(&mypackage_foo), HashSet::new()); + } + + #[test] + fn find_children_one_result() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + + assert_eq!( + graph.find_children(&mypackage), + HashSet::from([&mypackage_foo, &mypackage_bar]) + ); + } + + #[test] + fn find_children_multiple_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + + assert_eq!( + graph.find_children(&mypackage), + HashSet::from([&mypackage_foo, &mypackage_bar]) + ); + } + + #[test] + fn find_children_returns_empty_set_with_nonexistent_module() { + let mut graph = Graph::default(); + // Note: mypackage is not in the graph. + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + + assert_eq!( + graph.find_children(&Module::new("mypackage".to_string())), + HashSet::new() + ); + } + + #[test] + fn find_descendants_no_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + + assert_eq!(graph.find_descendants(&mypackage_bar), Ok(HashSet::new())); + } + + #[test] + fn find_descendants_module_not_in_graph() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + graph.add_module(blue.clone()); + + assert_eq!( + graph.find_descendants(&green), + Err(ModuleNotPresent { + module: green.clone() + }) + ); + } + + #[test] + fn find_descendants_multiple_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + + assert_eq!( + graph.find_descendants(&mypackage_foo), + Ok(HashSet::from([ + &mypackage_foo_alpha, + &mypackage_foo_alpha_blue, + &mypackage_foo_alpha_green, + &mypackage_foo_beta + ])) + ); + } + + #[test] + fn find_descendants_with_gap() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + // mypackage.foo.blue is not added. + let mypackage_foo_blue_alpha = Module::new("mypackage.foo.blue.alpha".to_string()); + let mypackage_foo_blue_alpha_one = Module::new("mypackage.foo.blue.alpha.one".to_string()); + let mypackage_foo_blue_alpha_two = Module::new("mypackage.foo.blue.alpha.two".to_string()); + let mypackage_foo_blue_beta_three = + Module::new("mypackage.foo.blue.beta.three".to_string()); + let mypackage_bar_green_alpha = Module::new("mypackage.bar.green.alpha".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_blue_alpha.clone()); + graph.add_module(mypackage_foo_blue_alpha_one.clone()); + graph.add_module(mypackage_foo_blue_alpha_two.clone()); + graph.add_module(mypackage_foo_blue_beta_three.clone()); + graph.add_module(mypackage_bar_green_alpha.clone()); + + assert_eq!( + graph.find_descendants(&mypackage_foo), + // mypackage.foo.blue is not included. + Ok(HashSet::from([ + &mypackage_foo_blue_alpha, + &mypackage_foo_blue_alpha_one, + &mypackage_foo_blue_alpha_two, + &mypackage_foo_blue_beta_three, + ])) + ); + } + + #[test] + fn find_descendants_added_in_different_order() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_foo_blue_alpha = Module::new("mypackage.foo.blue.alpha".to_string()); + let mypackage_foo_blue_alpha_one = Module::new("mypackage.foo.blue.alpha.one".to_string()); + let mypackage_foo_blue_alpha_two = Module::new("mypackage.foo.blue.alpha.two".to_string()); + let mypackage_foo_blue_beta_three = + Module::new("mypackage.foo.blue.beta.three".to_string()); + let mypackage_bar_green_alpha = Module::new("mypackage.bar.green.alpha".to_string()); + let mypackage_foo_blue = Module::new("mypackage.foo.blue".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_blue_alpha.clone()); + graph.add_module(mypackage_foo_blue_alpha_one.clone()); + graph.add_module(mypackage_foo_blue_alpha_two.clone()); + graph.add_module(mypackage_foo_blue_beta_three.clone()); + graph.add_module(mypackage_bar_green_alpha.clone()); + // Add the middle one at the end. + graph.add_module(mypackage_foo_blue.clone()); + + assert_eq!( + graph.find_descendants(&mypackage_foo), + Ok(HashSet::from([ + &mypackage_foo_blue, // Should be included. + &mypackage_foo_blue_alpha, + &mypackage_foo_blue_alpha_one, + &mypackage_foo_blue_alpha_two, + &mypackage_foo_blue_beta_three, + ])) + ); + } + + #[test] + fn direct_import_exists_returns_true() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); + } + + #[test] + fn add_detailed_import_adds_import() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + let import = DetailedImport { + importer: blue.clone(), + imported: green.clone(), + line_number: 11, + line_contents: "-".to_string(), + }; + + graph.add_detailed_import(&import); + + assert_eq!(graph.direct_import_exists(&blue, &green, false), true); + } + + #[test] + fn direct_import_exists_returns_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert!(!graph.direct_import_exists(&mypackage_bar, &mypackage_foo, false)); + } + + #[test] + fn direct_import_exists_returns_false_root_to_child() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha); + + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.bar + mypackage -> mypackage.foo + mypackage.foo -> mypackage.foo.alpha +imports: + mypackage.bar -> mypackage.foo.alpha +" + .trim_start() + ); + assert!(!graph.direct_import_exists(&mypackage_bar, &mypackage_foo, false)); + } + + #[test] + fn add_import_with_non_existent_importer_adds_that_module() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage_bar.clone()); + + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert_eq!( + graph.get_modules(), + HashSet::from([&mypackage_bar, &mypackage_foo]) + ); + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + (mypackage) -> mypackage.bar + (mypackage) -> mypackage.foo +imports: + mypackage.foo -> mypackage.bar +" + .trim_start() + ); + } + + #[test] + fn add_import_with_non_existent_imported_adds_that_module() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage_foo.clone()); + + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert_eq!( + graph.get_modules(), + HashSet::from([&mypackage_bar, &mypackage_foo]) + ); + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + (mypackage) -> mypackage.bar + (mypackage) -> mypackage.foo +imports: + mypackage.foo -> mypackage.bar +" + .trim_start() + ); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + // Add an import in the other direction. + graph.add_import(&mypackage_bar, &mypackage_foo); + + assert!(!graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_true_between_roots() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_true_root_to_child() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha); + + assert!(graph.direct_import_exists(&mypackage_bar, &mypackage_foo, true)); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_true_child_to_root() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_foo_alpha, &mypackage_bar); + + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); + } + + #[test] + #[should_panic] + fn direct_import_exists_within_package_panics() { + let mut graph = Graph::default(); + let ancestor = Module::new("mypackage.foo".to_string()); + let descendant = Module::new("mypackage.foo.blue.alpha".to_string()); + graph.add_import(&ancestor, &descendant); + + graph.direct_import_exists(&ancestor, &descendant, true); + } + + #[test] + fn find_modules_that_directly_import() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + let anotherpackage = Module::new("anotherpackage".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_foo_alpha, &mypackage_bar); + graph.add_import(&anotherpackage, &mypackage_bar); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha_green); + + let result = graph.find_modules_that_directly_import(&mypackage_bar); + + assert_eq!( + result, + HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + ) + } + + #[test] + fn find_modules_that_directly_import_after_removal() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let yellow = Module::new("yellow".to_string()); + graph.add_import(&green, &blue); + graph.add_import(&yellow, &blue); + + graph.remove_import(&green, &blue); + let result = graph.find_modules_that_directly_import(&blue); + + assert_eq!(result, HashSet::from([&yellow])) + } + + #[test] + fn find_modules_directly_imported_by() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + let anotherpackage = Module::new("anotherpackage".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha); + graph.add_import(&mypackage_bar, &anotherpackage); + graph.add_import(&mypackage_foo_alpha_green, &mypackage_bar); + + let result = graph.find_modules_directly_imported_by(&mypackage_bar); + + assert_eq!( + result, + HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + ) + } + + #[test] + fn find_modules_directly_imported_by_after_removal() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let yellow = Module::new("yellow".to_string()); + graph.add_import(&blue, &green); + graph.add_import(&blue, &yellow); + + graph.remove_import(&blue, &green); + let result = graph.find_modules_directly_imported_by(&blue); + + assert_eq!(result, HashSet::from([&yellow])) + } + + #[test] + fn squash_module_descendants() { + let mut graph = Graph::default(); + // Module we're going to squash. + let mypackage = Module::new("mypackage".to_string()); + let mypackage_blue = Module::new("mypackage.blue".to_string()); + let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let mypackage_blue_alpha_foo = Module::new("mypackage.blue.alpha.foo".to_string()); + let mypackage_blue_beta = Module::new("mypackage.blue.beta".to_string()); + // Other modules. + let mypackage_green = Module::new("mypackage.green".to_string()); + let mypackage_red = Module::new("mypackage.red".to_string()); + let mypackage_orange = Module::new("mypackage.orange".to_string()); + let mypackage_yellow = Module::new("mypackage.yellow".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_blue.clone()); + // Module's descendants importing other modules. + graph.add_import(&mypackage_blue_alpha, &mypackage_green); + graph.add_import(&mypackage_blue_alpha, &mypackage_red); + graph.add_import(&mypackage_blue_alpha_foo, &mypackage_yellow); + graph.add_import(&mypackage_blue_beta, &mypackage_orange); + // Other modules importing squashed module's descendants. + graph.add_import(&mypackage_red, &mypackage_blue_alpha); + graph.add_import(&mypackage_yellow, &mypackage_blue_alpha); + graph.add_import(&mypackage_orange, &mypackage_blue_alpha_foo); + graph.add_import(&mypackage_green, &mypackage_blue_beta); + // Unrelated imports. + graph.add_import(&mypackage_green, &mypackage_orange); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.blue + mypackage -> mypackage.green + mypackage -> mypackage.orange + mypackage -> mypackage.red + mypackage -> mypackage.yellow + mypackage.blue -> mypackage.blue.alpha + mypackage.blue -> mypackage.blue.beta + mypackage.blue.alpha -> mypackage.blue.alpha.foo +imports: + mypackage.blue.alpha -> mypackage.green + mypackage.blue.alpha -> mypackage.red + mypackage.blue.alpha.foo -> mypackage.yellow + mypackage.blue.beta -> mypackage.orange + mypackage.green -> mypackage.blue.beta + mypackage.green -> mypackage.orange + mypackage.orange -> mypackage.blue.alpha.foo + mypackage.red -> mypackage.blue.alpha + mypackage.yellow -> mypackage.blue.alpha +" + .trim_start() + ); + + graph.squash_module(&mypackage_blue); + + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.blue + mypackage -> mypackage.green + mypackage -> mypackage.orange + mypackage -> mypackage.red + mypackage -> mypackage.yellow +imports: + mypackage.blue -> mypackage.green + mypackage.blue -> mypackage.orange + mypackage.blue -> mypackage.red + mypackage.blue -> mypackage.yellow + mypackage.green -> mypackage.blue + mypackage.green -> mypackage.orange + mypackage.orange -> mypackage.blue + mypackage.red -> mypackage.blue + mypackage.yellow -> mypackage.blue +" + .trim_start() + ); + } + + #[test] + fn squash_module_no_descendants() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_blue = Module::new("mypackage.blue".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_blue.clone()); + + graph.squash_module(&mypackage_blue); + + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.blue +imports: + +" + .trim_start() + ); + } + + #[test] + fn find_count_imports_empty_graph() { + let graph = Graph::default(); + + let result = graph.count_imports(); + + assert_eq!(result, 0); + } + + #[test] + fn find_count_imports_modules_but_no_imports() { + let mut graph = Graph::default(); + graph.add_module(Module::new("mypackage.foo".to_string())); + graph.add_module(Module::new("mypackage.bar".to_string())); + + let result = graph.count_imports(); + + assert_eq!(result, 0); + } + + #[test] + fn find_count_imports_some_imports() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_baz = Module::new("mypackage.baz".to_string()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + graph.add_import(&mypackage_foo, &mypackage_baz); + + let result = graph.count_imports(); + + assert_eq!(result, 2); + } + + #[test] + fn find_count_imports_treats_two_imports_between_same_modules_as_one() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + graph.add_import(&mypackage_foo, &mypackage_bar); + + let result = graph.count_imports(); + + assert_eq!(result, 1); + } + + #[test] + fn is_module_squashed_when_not_squashed() { + let mut graph = Graph::default(); + // Module we're going to squash. + let mypackage_blue = Module::new("mypackage.blue".to_string()); + let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + // Other module. + let mypackage_green = Module::new("mypackage.green".to_string()); + graph.add_module(mypackage_blue.clone()); + graph.add_module(mypackage_blue_alpha.clone()); + graph.add_module(mypackage_green.clone()); + graph.squash_module(&mypackage_blue); + + let result = graph.is_module_squashed(&mypackage_green); + + assert!(!result); + } + + #[test] + fn is_module_squashed_when_squashed() { + let mut graph = Graph::default(); + // Module we're going to squash. + let mypackage_blue = Module::new("mypackage.blue".to_string()); + let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + // Other module. + let mypackage_green = Module::new("mypackage.green".to_string()); + graph.add_module(mypackage_blue.clone()); + graph.add_module(mypackage_blue_alpha.clone()); + graph.add_module(mypackage_green.clone()); + graph.squash_module(&mypackage_blue); + + let result = graph.is_module_squashed(&mypackage_blue); + + assert!(result); + } + + #[test] + fn find_upstream_modules_when_there_are_some() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chain we care about. + graph.add_import(&blue, &green); + graph.add_import(&blue, &red); + graph.add_import(&green, &yellow); + graph.add_import(&yellow, &purple); + // Add an import to blue. + graph.add_import(&brown, &blue); + + let result = graph.find_upstream_modules(&blue, false); + + assert_eq!(result, HashSet::from([&green, &red, &yellow, &purple])) + } + + #[test] + fn find_upstream_modules_when_module_doesnt_exist() { + let graph = Graph::default(); + let blue = Module::new("mypackage.blue".to_string()); + + let result = graph.find_upstream_modules(&blue, false); + + assert_eq!(result, HashSet::new()) + } + + #[test] + fn find_upstream_modules_as_packages() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let alpha = Module::new("mypackage.blue.alpha".to_string()); + let beta = Module::new("mypackage.blue.beta".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(alpha.clone()); + graph.add_module(beta.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chains we care about. + graph.add_import(&blue, &green); + graph.add_import(&green, &yellow); + graph.add_import(&alpha, &purple); + graph.add_import(&purple, &brown); + // Despite being technically upstream, beta doesn't appear because it's + // in the same package. + graph.add_import(&purple, &beta); + + let result = graph.find_upstream_modules(&blue, true); + + assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + } + + #[test] + fn find_downstream_modules_when_there_are_some() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chain we care about. + graph.add_import(&blue, &green); + graph.add_import(&blue, &red); + graph.add_import(&green, &yellow); + graph.add_import(&yellow, &purple); + // Add an import from purple. + graph.add_import(&purple, &brown); + + let result = graph.find_downstream_modules(&purple, false); + + assert_eq!(result, HashSet::from([&yellow, &green, &blue])) + } + + #[test] + fn find_downstream_modules_when_module_doesnt_exist() { + let graph = Graph::default(); + let blue = Module::new("mypackage.blue".to_string()); + + let result = graph.find_downstream_modules(&blue, false); + + assert_eq!(result, HashSet::new()) + } + + #[test] + fn find_downstream_modules_as_packages() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let alpha = Module::new("mypackage.blue.alpha".to_string()); + let beta = Module::new("mypackage.blue.beta".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(alpha.clone()); + graph.add_module(beta.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chains we care about. + graph.add_import(&yellow, &green); + graph.add_import(&green, &blue); + graph.add_import(&brown, &purple); + graph.add_import(&purple, &alpha); + // Despite being technically downstream, beta doesn't appear because it's + // in the same package. + graph.add_import(&beta, &yellow); + + let result = graph.find_downstream_modules(&blue, true); + + assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + } + + // find_shortest_chain + #[test] + fn find_shortest_chain_none() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(purple.clone()); + // Add imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green); + + assert!(result.is_none()) + } + + #[test] + fn find_shortest_chain_one_step() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add the one-step chain. + graph.add_import(&blue, &green); + // Add a longer chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green).unwrap(); + + assert_eq!(result, vec![&blue, &green]) + } + + #[test] + fn find_shortest_chain_one_step_reverse() { + let mut graph = Graph::default(); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + // Add the one-step chain. + graph.add_import(&blue, &green); + + let result = graph.find_shortest_chain(&green, &blue); + + assert_eq!(result.is_none(), true); + } + + #[test] + fn find_shortest_chain_two_steps() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(orange.clone()); + graph.add_module(purple.clone()); + // Add the two-step chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green); + // Add a longer chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &orange); + graph.add_import(&orange, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green).unwrap(); + + assert_eq!(result, vec![&blue, &red, &green]) + } + + #[test] + fn find_shortest_chain_three_steps() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(orange.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + // Add the three-step chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &orange); + graph.add_import(&orange, &green); + // Add a longer chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &orange); + graph.add_import(&orange, &yellow); + graph.add_import(&yellow, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green).unwrap(); + + assert_eq!(result, vec![&blue, &red, &orange, &green]) + } + + // find_shortest_chains + + #[test] + fn find_shortest_chains_none() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(purple.clone()); + // Add imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::new())); + } + + #[test] + fn find_shortest_chains_between_passed_modules() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::from([vec![blue, red, green],]))); + } + + #[test] + fn find_shortest_chains_between_passed_module_and_child() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let green_alpha = Module::new("mypackage.green.alpha".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(green_alpha.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green_alpha); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::from([vec![blue, red, green_alpha]]))); + } + + #[test] + fn find_shortest_chains_between_passed_module_and_grandchild() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let green_alpha = Module::new("mypackage.green.alpha".to_string()); + let green_alpha_one = Module::new("mypackage.green.alpha.one".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(green_alpha.clone()); + graph.add_module(green_alpha_one.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green_alpha_one); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!( + result, + Ok(HashSet::from([vec![blue, red, green_alpha_one],])) + ) + } + + #[test] + fn find_shortest_chains_between_child_and_passed_module() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::from([vec![blue_alpha, red, green],]))); + } + + #[test] + fn find_shortest_chains_between_grandchild_and_passed_module() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!( + result, + Ok(HashSet::from([vec![blue_alpha_one, red, green],])) + ) + } + + #[test] + fn chain_exists_true_as_packages_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.chain_exists(&blue_alpha_one, &green, false); + + assert!(result); + } + + #[test] + fn chain_exists_false_as_packages_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + + let result = graph.chain_exists(&blue, &green, false); + + assert_eq!(result, false); + } + + #[test] + fn chain_exists_true_as_packages_true() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + + let result = graph.chain_exists(&blue, &green, true); + + assert_eq!(result, true); + } + + #[test] + fn chain_exists_false_as_packages_true() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + + let result = graph.chain_exists(&green, &blue, true); + + assert_eq!(result, false); + } + + #[test] + fn find_illegal_dependencies_for_layers_empty_everything() { + let graph = Graph::default(); + + let dependencies = graph.find_illegal_dependencies_for_layers(vec![], HashSet::new()); + + assert_eq!(dependencies, Ok(vec![])); + } + + #[test] + fn find_illegal_dependencies_for_layers_no_such_container() { + let graph = Graph::default(); + let container = "nonexistent_container".to_string(); + + let dependencies = + graph.find_illegal_dependencies_for_layers(vec![], HashSet::from([container.clone()])); + + assert_eq!( + dependencies, + Err(NoSuchContainer { + container: container + }) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_nonexistent_layers_no_container() { + let graph = Graph::default(); + let level = Level { + layers: vec!["nonexistent".to_string()], + independent: true, + }; + + let dependencies = graph.find_illegal_dependencies_for_layers(vec![level], HashSet::new()); + + assert_eq!(dependencies, Ok(vec![])); + } + + #[test] + fn find_illegal_dependencies_for_layers_nonexistent_layers_with_container() { + let mut graph = Graph::default(); + graph.add_module(Module::new("mypackage".to_string())); + let level = Level { + layers: vec!["nonexistent".to_string()], + independent: true, + }; + let container = "mypackage".to_string(); + + let dependencies = + graph.find_illegal_dependencies_for_layers(vec![level], HashSet::from([container])); + + assert_eq!(dependencies, Ok(vec![])); + } + + #[test] + fn find_illegal_dependencies_for_layers_no_container_direct_dependency() { + let mut graph = Graph::default(); + let high = Module::new("high".to_string()); + let low = Module::new("low".to_string()); + graph.add_import(&low, &high); + let levels = vec![ + Level { + layers: vec![high.name.clone()], + independent: true, + }, + Level { + layers: vec![low.name.clone()], + independent: true, + }, + ]; + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + + assert_eq!( + dependencies, + Ok(vec![PackageDependency { + importer: low.clone(), + imported: high.clone(), + routes: vec![Route { + heads: vec![low.clone()], + middle: vec![], + tails: vec![high.clone()], + }] + }]) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_no_container_indirect_dependency() { + let mut graph = Graph::default(); + let high = Module::new("high".to_string()); + let elsewhere = Module::new("elsewhere".to_string()); + let low = Module::new("low".to_string()); + graph.add_import(&low, &elsewhere); + graph.add_import(&elsewhere, &high); + let levels = vec![ + Level { + layers: vec![high.name.clone()], + independent: true, + }, + Level { + layers: vec![low.name.clone()], + independent: true, + }, + ]; + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + + assert_eq!( + dependencies, + Ok(vec![PackageDependency { + importer: low.clone(), + imported: high.clone(), + routes: vec![Route { + heads: vec![low.clone()], + middle: vec![elsewhere.clone()], + tails: vec![high.clone()], + }] + }]) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_containers() { + let mut graph = Graph::default(); + let blue_high = Module::new("blue.high".to_string()); + let blue_high_alpha = Module::new("blue.high.alpha".to_string()); + let blue_low = Module::new("blue.low".to_string()); + let blue_low_beta = Module::new("blue.low.beta".to_string()); + let green_high = Module::new("green.high".to_string()); + let green_high_gamma = Module::new("green.high.gamma".to_string()); + let green_low = Module::new("green.low".to_string()); + let green_low_delta = Module::new("green.low.delta".to_string()); + graph.add_module(Module::new("blue".to_string())); + graph.add_module(blue_high.clone()); + graph.add_module(blue_low.clone()); + graph.add_module(Module::new("green".to_string())); + graph.add_module(green_high.clone()); + graph.add_module(green_low.clone()); + graph.add_import(&blue_low_beta, &blue_high_alpha); + graph.add_import(&green_low_delta, &green_high_gamma); + + let levels = vec![ + Level { + layers: vec!["high".to_string()], + independent: true, + }, + Level { + layers: vec!["low".to_string()], + independent: true, + }, + ]; + let containers = HashSet::from(["blue".to_string(), "green".to_string()]); + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, containers); + + assert_eq!( + dependencies, + Ok(vec![ + PackageDependency { + importer: blue_low.clone(), + imported: blue_high.clone(), + routes: vec![Route { + heads: vec![blue_low_beta.clone()], + middle: vec![], + tails: vec![blue_high_alpha.clone()], + }] + }, + PackageDependency { + importer: green_low.clone(), + imported: green_high.clone(), + routes: vec![Route { + heads: vec![green_low_delta.clone()], + middle: vec![], + tails: vec![green_high_gamma.clone()], + }] + } + ]) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_independent() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let blue_alpha = Module::new("blue.alpha".to_string()); + let green_beta = Module::new("green.beta".to_string()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_import(&blue_alpha, &green_beta); + + let levels = vec![Level { + layers: vec![blue.name.clone(), green.name.clone()], + independent: true, + }]; + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + + assert_eq!( + dependencies, + Ok(vec![PackageDependency { + importer: blue.clone(), + imported: green.clone(), + routes: vec![Route { + heads: vec![blue_alpha.clone()], + middle: vec![], + tails: vec![green_beta.clone()], + }] + }]) + ); + } + + #[test] + fn get_import_details_no_modules() { + let graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::new()); + } + + #[test] + fn get_import_details_module_without_metadata() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + graph.add_import(&importer, &imported); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::new()); + } + + #[test] + fn get_import_details_module_one_result() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + let import = DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 5, + line_contents: "import bar".to_string(), + }; + let unrelated_import = DetailedImport { + importer: importer.clone(), + imported: Module::new("baz".to_string()), + line_number: 2, + line_contents: "-".to_string(), + }; + graph.add_detailed_import(&import); + graph.add_detailed_import(&unrelated_import); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::from([import])); + } + + #[test] + fn get_import_details_module_two_results() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let blue_to_green_a = DetailedImport { + importer: blue.clone(), + imported: green.clone(), + line_number: 5, + line_contents: "import green".to_string(), + }; + let blue_to_green_b = DetailedImport { + importer: blue.clone(), + imported: green.clone(), + line_number: 15, + line_contents: "import green".to_string(), + }; + graph.add_detailed_import(&blue_to_green_a); + graph.add_detailed_import(&blue_to_green_b); + + let result = graph.get_import_details(&blue, &green); + + assert_eq!(result, HashSet::from([blue_to_green_a, blue_to_green_b])); + } + + #[test] + fn get_import_details_after_removal() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + let import = DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 5, + line_contents: "import bar".to_string(), + }; + let unrelated_import = DetailedImport { + importer: importer.clone(), + imported: Module::new("baz".to_string()), + line_number: 2, + line_contents: "-".to_string(), + }; + graph.add_detailed_import(&import); + graph.add_detailed_import(&unrelated_import); + graph.remove_import(&import.importer, &import.imported); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::new()); + } + + #[test] + fn get_import_details_after_removal_of_unrelated_import() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + let import = DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 5, + line_contents: "import bar".to_string(), + }; + let unrelated_import = DetailedImport { + importer: importer.clone(), + imported: Module::new("baz".to_string()), + line_number: 2, + line_contents: "-".to_string(), + }; + graph.add_detailed_import(&import); + graph.add_detailed_import(&unrelated_import); + graph.remove_import(&unrelated_import.importer, &unrelated_import.imported); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::from([import])); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 73e7038b..cbc1a5ff 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -13,6 +13,7 @@ use pyo3::create_exception; use pyo3::prelude::*; use pyo3::types::{PyDict, PyFrozenSet, PySet, PyString, PyTuple}; use std::collections::{HashMap, HashSet}; +pub mod graph; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { From f5a2664a8c40328b7d7942362ce89c49d3ae80df Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 16:10:03 +0000 Subject: [PATCH 2/5] Make integration test use new API --- rust/tests/large.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 0fff1d65..1cdaf9ab 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,7 +1,6 @@ -use _rustgrimp::importgraph::ImportGraph; -use _rustgrimp::layers::{find_illegal_dependencies, Level}; +use _rustgrimp::graph::{Graph, Level, Module}; use serde_json::{Map, Value}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::fs; #[test] @@ -9,15 +8,19 @@ fn test_large_graph_deep_layers() { let data = fs::read_to_string("tests/large_graph.json").expect("Unable to read file"); let value: Value = serde_json::from_str(&data).unwrap(); let items: &Map = value.as_object().unwrap(); - let mut importeds_by_importer: HashMap<&str, HashSet<&str>> = HashMap::new(); + let mut graph = Graph::default(); for (importer, importeds_value) in items.iter() { - let mut importeds = HashSet::new(); for imported in importeds_value.as_array().unwrap() { - importeds.insert(imported.as_str().unwrap()); + graph.add_import( + &Module { + name: importer.clone(), + }, + &Module { + name: imported.as_str().unwrap().to_string(), + }, + ); } - importeds_by_importer.insert(importer, importeds); } - let graph = ImportGraph::new(importeds_by_importer); let deep_layers = vec![ "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.1991886645", @@ -39,7 +42,9 @@ fn test_large_graph_deep_layers() { .collect(); let containers = HashSet::new(); - let deps = find_illegal_dependencies(&graph, &levels, &containers); + let deps = graph + .find_illegal_dependencies_for_layers(levels, containers) + .unwrap(); assert_eq!(deps.len(), 8); } From 0f63e65470b6e64f3fb420b8e9b10704b093e69e Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 13:11:49 +0000 Subject: [PATCH 3/5] Use a Rust-backed Graph This adds a wrapper around the Rust-graph in lib.rs which handles the interface between Python and Rust. At the same time, it changes the Python ImportGraph so it uses Rust instead. --- rust/Cargo.lock | 103 ++++---- rust/Cargo.toml | 4 +- rust/src/graph.rs | 25 +- rust/src/lib.rs | 399 +++++++++++++++++++++++----- src/grimp/adaptors/graph.py | 501 ++++++++---------------------------- 5 files changed, 515 insertions(+), 517 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c9b0a5fe..37709188 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -23,9 +23,9 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bimap" @@ -41,9 +41,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -60,9 +60,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "either" @@ -84,9 +84,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -96,9 +96,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown", @@ -112,21 +112,27 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -139,9 +145,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "petgraph" @@ -155,24 +161,24 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "cfg-if", "indoc", @@ -188,9 +194,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -198,9 +204,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", @@ -208,9 +214,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac84e6eec1159bc2a575c9ae6723baa6ee9d45873e9bebad1e3ad7e8d28a443" +checksum = "be5bb22b77965a7b5394e9aae9897a0607b51df5167561ffc3b02643b4200bc7" dependencies = [ "arc-swap", "log", @@ -219,9 +225,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -231,9 +237,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ "heck", "proc-macro2", @@ -244,9 +250,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -279,18 +285,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -299,20 +305,21 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "syn" -version = "2.0.70" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -321,15 +328,15 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unindent" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3ab4be53..09e0e5ec 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,14 +9,14 @@ crate-type = ["cdylib", "rlib"] [dependencies] log = "0.4.19" -pyo3-log = "0.11.0" +pyo3-log = "0.12.1" serde_json = "1.0.103" rayon = "1.10" petgraph = "0.6.5" bimap = "0.6.3" [dependencies.pyo3] -version = "0.22.4" +version = "0.23.4" [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/src/graph.rs b/rust/src/graph.rs index 0de65ec3..ae9cf237 100644 --- a/rust/src/graph.rs +++ b/rust/src/graph.rs @@ -1,8 +1,3 @@ -/* -Also, sensible behaviour when passing modules that don't exist in the graph. -*/ -#![allow(dead_code)] - use bimap::BiMap; use log::info; use petgraph::algo::astar; @@ -382,7 +377,6 @@ impl Graph { } // Note: this will panic if importer and imported are in the same package. - #[allow(unused_variables)] pub fn direct_import_exists( &self, importer: &Module, @@ -552,7 +546,6 @@ impl Graph { Some(chain) } - // https://github.com/seddonym/grimp/blob/master/src/grimp/adaptors/graph.py#L290 pub fn find_shortest_chains( &self, importer: &Module, @@ -649,7 +642,6 @@ impl Graph { Ok(chains) } - #[allow(unused_variables)] pub fn chain_exists(&self, importer: &Module, imported: &Module, as_packages: bool) -> bool { // TODO should this return a Result, so we can handle the situation the importer / imported // having shared descendants when as_packages=true? @@ -666,7 +658,6 @@ impl Graph { graph.find_shortest_chain(importer, imported).is_some() } - #[allow(unused_variables)] pub fn find_illegal_dependencies_for_layers( &self, levels: Vec, @@ -689,8 +680,6 @@ impl Graph { .map(|module_name| module_name.to_string()) .collect(); - let perms = self._generate_module_permutations(&levels, &containers); - let mut dependencies: Vec = self ._generate_module_permutations(&levels, &containers) //.into_iter() @@ -984,7 +973,6 @@ impl Graph { imports } - #[allow(unused_variables)] pub fn squash_module(&mut self, module: &Module) { // Get descendants and their imports. let descendants: Vec = self @@ -1031,6 +1019,19 @@ impl Graph { self.squashed_modules.contains(module) } + /// Return the squashed module that is the nearest ancestor of the supplied module, + /// if such an ancestor exists. + pub fn find_ancestor_squashed_module(&self, module: &Module) -> Option { + if module.is_root() { + return None; + } + let parent = Module::new_parent(&module); + if self.is_module_squashed(&parent) { + return Some(parent); + } + self.find_ancestor_squashed_module(&parent) + } + fn add_module_if_not_in_hierarchy(&mut self, module: &Module) { if self.hierarchy_module_indices.get_by_left(module).is_none() { self.add_module(module.clone()); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index cbc1a5ff..112dbb0e 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,72 +1,342 @@ -mod containers; -// TODO make these private. -pub mod dependencies; -pub mod importgraph; -pub mod layers; - -use crate::dependencies::PackageDependency; -use containers::check_containers_exist; -use importgraph::ImportGraph; -use layers::Level; +pub mod graph; + +use crate::graph::{DetailedImport, Graph, Level, Module, PackageDependency}; use log::info; use pyo3::create_exception; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyFrozenSet, PySet, PyString, PyTuple}; -use std::collections::{HashMap, HashSet}; -pub mod graph; +use pyo3::types::{PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; +use std::collections::HashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { pyo3_log::init(); - m.add_function(wrap_pyfunction!(find_illegal_dependencies, m)?)?; - m.add("NoSuchContainer", py.get_type_bound::())?; + m.add_class::()?; + m.add("NoSuchContainer", py.get_type::())?; Ok(()) } create_exception!(_rustgrimp, NoSuchContainer, pyo3::exceptions::PyException); -#[pyfunction] -pub fn find_illegal_dependencies<'py>( - py: Python<'py>, - levels: &Bound<'py, PyTuple>, - containers: &Bound<'py, PySet>, - importeds_by_importer: &Bound<'py, PyDict>, -) -> PyResult> { - info!("Using Rust to find illegal dependencies."); +#[pyclass(name = "Graph")] +struct GraphWrapper { + _graph: Graph, +} + +/// Wrapper around the Graph struct that integrates with Python. +#[pymethods] +impl GraphWrapper { + #[new] + fn new() -> Self { + GraphWrapper { + _graph: Graph::default(), + } + } - let importeds_by_importer_strings: HashMap> = - importeds_by_importer.extract()?; - let importeds_by_importer_strs = strings_to_strs_hashmap(&importeds_by_importer_strings); + pub fn get_modules(&self) -> HashSet { + self._graph + .get_modules() + .iter() + .map(|module| module.name.clone()) + .collect() + } + + #[pyo3(signature = (module, is_squashed = false))] + pub fn add_module(&mut self, module: &str, is_squashed: bool) -> PyResult<()> { + let module_struct = Module::new(module.to_string()); - let graph = ImportGraph::new(importeds_by_importer_strs); - let levels_rust = rustify_levels(levels); - let containers_rust: HashSet = containers.extract()?; + if let Some(ancestor_squashed_module) = + self._graph.find_ancestor_squashed_module(&module_struct) + { + return Err(PyValueError::new_err(format!( + "Module is a descendant of squashed module {}.", + &ancestor_squashed_module.name + ))); + } - if let Err(err) = check_containers_exist(&graph, &containers_rust) { - return Err(NoSuchContainer::new_err(err)); + if self._graph.get_modules().contains(&module_struct) { + if self._graph.is_module_squashed(&module_struct) != is_squashed { + return Err(PyValueError::new_err( + "Cannot add a squashed module when it is already present in the graph \ + as an unsquashed module, or vice versa.", + )); + } + } + + match is_squashed { + false => self._graph.add_module(module_struct), + true => self._graph.add_squashed_module(module_struct), + }; + Ok(()) } - let dependencies = py.allow_threads(|| { - layers::find_illegal_dependencies(&graph, &levels_rust, &containers_rust) - }); + pub fn remove_module(&mut self, module: &str) { + self._graph.remove_module(&Module::new(module.to_string())); + } - convert_dependencies_to_python(py, dependencies, &graph) -} + pub fn squash_module(&mut self, module: &str) { + self._graph.squash_module(&Module::new(module.to_string())); + } + + pub fn is_module_squashed(&self, module: &str) -> bool { + self._graph + .is_module_squashed(&Module::new(module.to_string())) + } + + #[pyo3(signature = (*, importer, imported, line_number=None, line_contents=None))] + pub fn add_import( + &mut self, + importer: &str, + imported: &str, + line_number: Option, + line_contents: Option<&str>, + ) { + let importer = Module::new(importer.to_string()); + let imported = Module::new(imported.to_string()); + match (line_number, line_contents) { + (Some(line_number), Some(line_contents)) => { + self._graph.add_detailed_import(&DetailedImport { + importer: importer, + imported: imported, + line_number: line_number, + line_contents: line_contents.to_string(), + }); + } + (None, None) => { + self._graph.add_import(&importer, &imported); + } + _ => { + // TODO handle better. + panic!("Expected line_number and line_contents, or neither."); + } + } + } + + #[pyo3(signature = (*, importer, imported))] + pub fn remove_import(&mut self, importer: &str, imported: &str) { + self._graph.remove_import( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + ); + } + + pub fn count_imports(&self) -> usize { + self._graph.count_imports() + } + + pub fn find_children(&self, module: &str) -> HashSet { + self._graph + .find_children(&Module::new(module.to_string())) + .iter() + .map(|child| child.name.clone()) + .collect() + } + + pub fn find_descendants(&self, module: &str) -> HashSet { + self._graph + .find_descendants(&Module::new(module.to_string())) + .unwrap() + .iter() + .map(|descendant| descendant.name.clone()) + .collect() + } -fn strings_to_strs_hashmap<'a>( - string_map: &'a HashMap>, -) -> HashMap<&'a str, HashSet<&'a str>> { - let mut str_map: HashMap<&str, HashSet<&str>> = HashMap::new(); + #[pyo3(signature = (*, importer, imported, as_packages = false))] + pub fn direct_import_exists( + &self, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult { + if as_packages { + let importer_module = Module::new(importer.to_string()); + let imported_module = Module::new(imported.to_string()); + // Raise a ValueError if they are in the same package. + // (direct_import_exists) will panic if they are passed. + // TODO - this is a simpler check than Python, is it enough? + if importer_module.is_descendant_of(&imported_module) + || imported_module.is_descendant_of(&importer_module) + { + return Err(PyValueError::new_err("Modules have shared descendants.")); + } + } + + Ok(self._graph.direct_import_exists( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + as_packages, + )) + } + + pub fn find_modules_directly_imported_by(&self, module: &str) -> HashSet { + self._graph + .find_modules_directly_imported_by(&Module::new(module.to_string())) + .iter() + .map(|imported| imported.name.clone()) + .collect() + } + + pub fn find_modules_that_directly_import(&self, module: &str) -> HashSet { + self._graph + .find_modules_that_directly_import(&Module::new(module.to_string())) + .iter() + .map(|importer| importer.name.clone()) + .collect() + } + + #[pyo3(signature = (*, importer, imported))] + pub fn get_import_details<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + ) -> PyResult> { + let mut vector: Vec> = vec![]; + + let mut rust_import_details_vec: Vec = self + ._graph + .get_import_details( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + ) + .into_iter() + .collect(); + rust_import_details_vec.sort(); + + for detailed_import in rust_import_details_vec { + let pydict = PyDict::new(py); + pydict.set_item( + "importer".to_string(), + detailed_import.importer.name.clone(), + )?; + pydict.set_item( + "imported".to_string(), + detailed_import.imported.name.clone(), + )?; + pydict.set_item("line_number".to_string(), detailed_import.line_number)?; + pydict.set_item( + "line_contents".to_string(), + detailed_import.line_contents.clone(), + )?; + vector.push(pydict); + } + PyList::new(py, &vector) + } + + #[allow(unused_variables)] + #[pyo3(signature = (module, as_package=false))] + pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> HashSet { + // Turn the Modules to Strings. + self._graph + .find_downstream_modules(&Module::new(module.to_string()), as_package) + .iter() + .map(|downstream| downstream.name.clone()) + .collect() + } + + #[allow(unused_variables)] + #[pyo3(signature = (module, as_package=false))] + pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> HashSet { + self._graph + .find_upstream_modules(&Module::new(module.to_string()), as_package) + .iter() + .map(|upstream| upstream.name.clone()) + .collect() + } - for (key, set) in string_map { - let mut str_set: HashSet<&str> = HashSet::new(); - for item in set.iter() { - str_set.insert(item); + pub fn find_shortest_chain(&self, importer: &str, imported: &str) -> Option> { + let chain = self._graph.find_shortest_chain( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + )?; + + Some(chain.iter().map(|module| module.name.clone()).collect()) + } + + #[pyo3(signature = (importer, imported, as_packages=true))] + pub fn find_shortest_chains<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult> { + let rust_chains: HashSet> = self + ._graph + .find_shortest_chains( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + as_packages, + ) + .map_err(|string| PyValueError::new_err(string))?; + + let mut tuple_chains: Vec> = vec![]; + for rust_chain in rust_chains.iter() { + let module_names: Vec> = rust_chain + .iter() + .map(|module| PyString::new(py, &module.name)) + .collect(); + let tuple = PyTuple::new(py, &module_names)?; + tuple_chains.push(tuple); + } + PySet::new(py, &tuple_chains) + } + + #[pyo3(signature = (importer, imported, as_packages=false))] + pub fn chain_exists( + &self, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult { + if as_packages { + let importer_module = Module::new(importer.to_string()); + let imported_module = Module::new(imported.to_string()); + // Raise a ValueError if they are in the same package. + // TODO - this is a simpler check than Python, is it enough? + if importer_module.is_descendant_of(&imported_module) + || imported_module.is_descendant_of(&importer_module) + { + return Err(PyValueError::new_err("Modules have shared descendants.")); + } + } + Ok(self._graph.chain_exists( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + as_packages, + )) + } + + #[allow(unused_variables)] + #[pyo3(signature = (layers, containers))] + pub fn find_illegal_dependencies_for_layers<'py>( + &self, + py: Python<'py>, + layers: &Bound<'py, PyTuple>, + containers: HashSet, + ) -> PyResult> { + info!("Using Rust to find illegal dependencies."); + let levels = rustify_levels(layers); + + println!("\nIncoming {:?}, {:?}", levels, containers); + let dependencies = py.allow_threads(|| { + self._graph + .find_illegal_dependencies_for_layers(levels, containers) + }); + match dependencies { + Ok(dependencies) => _convert_dependencies_to_python(py, &dependencies), + Err(error) => Err(NoSuchContainer::new_err(format!( + "Container {} does not exist.", + error.container + ))), + } + } + pub fn clone(&self) -> GraphWrapper { + GraphWrapper { + _graph: self._graph.clone(), } - str_map.insert(key.as_str(), str_set); } - str_map } fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { @@ -94,47 +364,46 @@ fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { rust_levels } -fn convert_dependencies_to_python<'py>( +fn _convert_dependencies_to_python<'py>( py: Python<'py>, - dependencies: Vec, - graph: &ImportGraph, + dependencies: &Vec, ) -> PyResult> { let mut python_dependencies: Vec> = vec![]; for rust_dependency in dependencies { - let python_dependency = PyDict::new_bound(py); - python_dependency.set_item("imported", graph.names_by_id[&rust_dependency.imported])?; - python_dependency.set_item("importer", graph.names_by_id[&rust_dependency.importer])?; + let python_dependency = PyDict::new(py); + python_dependency.set_item("imported", &rust_dependency.imported.name)?; + python_dependency.set_item("importer", &rust_dependency.importer.name)?; let mut python_routes: Vec> = vec![]; - for rust_route in rust_dependency.routes { - let route = PyDict::new_bound(py); + for rust_route in &rust_dependency.routes { + let route = PyDict::new(py); let heads: Vec> = rust_route .heads .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) + .map(|module| PyString::new(py, &module.name)) .collect(); - route.set_item("heads", PyFrozenSet::new_bound(py, &heads)?)?; + route.set_item("heads", PyFrozenSet::new(py, &heads)?)?; let middle: Vec> = rust_route .middle .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) + .map(|module| PyString::new(py, &module.name)) .collect(); - route.set_item("middle", PyTuple::new_bound(py, &middle))?; + route.set_item("middle", PyTuple::new(py, &middle)?)?; let tails: Vec> = rust_route .tails .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) + .map(|module| PyString::new(py, &module.name)) .collect(); - route.set_item("tails", PyFrozenSet::new_bound(py, &tails)?)?; + route.set_item("tails", PyFrozenSet::new(py, &tails)?)?; python_routes.push(route); } - python_dependency.set_item("routes", PyTuple::new_bound(py, python_routes))?; + python_dependency.set_item("routes", PyTuple::new(py, python_routes)?)?; python_dependencies.push(python_dependency) } - Ok(PyTuple::new_bound(py, python_dependencies)) + PyTuple::new(py, python_dependencies) } #[cfg(test)] @@ -146,7 +415,7 @@ mod tests { macro_rules! pydict { ($py: ident, {$($k: expr => $v: expr),*, $(,)?}) => { { - let dict = PyDict::new_bound($py); + let dict = PyDict::new($py); $( dict.set_item($k, $v)?; )* @@ -173,7 +442,7 @@ mod tests { "layers" => HashSet::from(["low"]), }), ]; - let python_levels = PyTuple::new_bound(py, elements); + let python_levels = PyTuple::new(py, elements)?; let result = rustify_levels(&python_levels); @@ -222,7 +491,7 @@ mod tests { "layers" => HashSet::from(["low"]), }), ]; - let python_levels = PyTuple::new_bound(py, elements); + let python_levels = PyTuple::new(py, elements)?; let mut result = rustify_levels(&python_levels); diff --git a/src/grimp/adaptors/graph.py b/src/grimp/adaptors/graph.py index 0ea9d96a..2c497fae 100644 --- a/src/grimp/adaptors/graph.py +++ b/src/grimp/adaptors/graph.py @@ -1,125 +1,41 @@ from __future__ import annotations - -from copy import copy -from typing import Dict, List, Optional, Sequence, Set, Tuple, cast - -from grimp.algorithms.shortest_path import bidirectional_shortest_path +from typing import List, Optional, Sequence, Set, Tuple, TypedDict +from grimp.application.ports.graph import DetailedImport +from grimp.domain.analysis import PackageDependency, Route +from grimp.domain.valueobjects import Layer +from grimp import _rustgrimp as rust # type: ignore[attr-defined] +from grimp.exceptions import ModuleNotPresent, NoSuchContainer from grimp.application.ports import graph -from grimp.domain.analysis import PackageDependency -from grimp.domain.valueobjects import Module, Layer -from grimp.exceptions import ModuleNotPresent - -from . import _layers class ImportGraph(graph.ImportGraph): """ - Pure Python implementation of the ImportGraph. + Rust-backed implementation of the ImportGraph. """ def __init__(self) -> None: - # Maps all the modules directly imported by each key. - self._importeds_by_importer: Dict[str, Set[str]] = {} - # Maps all the modules that directly import each key. - self._importers_by_imported: Dict[str, Set[str]] = {} - - self._edge_count = 0 - - # Instantiate a dict that stores the details for all direct imports. - self._import_details: Dict[str, List[graph.DetailedImport]] = {} - self._squashed_modules: Set[str] = set() - - # Dunder methods - # -------------- - - def __deepcopy__(self, memodict: Dict) -> "ImportGraph": - new_graph = ImportGraph() - new_graph._importeds_by_importer = { - key: value.copy() for key, value in self._importeds_by_importer.items() - } - new_graph._importers_by_imported = { - key: value.copy() for key, value in self._importers_by_imported.items() - } - new_graph._edge_count = self._edge_count - - # Note: this copies the dictionaries containing each import detail - # by *reference*, so be careful about mutating the import details - # dictionaries internally. - new_graph._import_details = { - key: value.copy() for key, value in self._import_details.items() - } - - new_graph._squashed_modules = self._squashed_modules.copy() - - return new_graph - - # Mechanics - # --------- + super().__init__() + self._rustgraph = rust.Graph() @property def modules(self) -> Set[str]: - # Note: wrapping this in a set() makes it 10 times slower to build the graph! - # As a further optimisation, we use the _StringSet type alias instead of looking up Set[str] - # when casting. - return cast(_StringSet, self._importeds_by_importer.keys()) + return self._rustgraph.get_modules() def add_module(self, module: str, is_squashed: bool = False) -> None: - ancestor_squashed_module = self._find_ancestor_squashed_module(module) - if ancestor_squashed_module: - raise ValueError( - f"Module is a descendant of squashed module {ancestor_squashed_module}." - ) - - if module in self.modules: - if self.is_module_squashed(module) != is_squashed: - raise ValueError( - "Cannot add a squashed module when it is already present in the graph as " - "an unsquashed module, or vice versa." - ) - - self._importeds_by_importer.setdefault(module, set()) - self._importers_by_imported.setdefault(module, set()) - - if is_squashed: - self._mark_module_as_squashed(module) + self._rustgraph.add_module(module, is_squashed) def remove_module(self, module: str) -> None: - if module not in self.modules: - # TODO: rethink this behaviour. - return - - for imported in copy(self.find_modules_directly_imported_by(module)): - self.remove_import(importer=module, imported=imported) - for importer in copy(self.find_modules_that_directly_import(module)): - self.remove_import(importer=importer, imported=module) - del self._importeds_by_importer[module] - del self._importers_by_imported[module] + self._rustgraph.remove_module(module) def squash_module(self, module: str) -> None: - if self.is_module_squashed(module): - return - - squashed_root = module - descendants = self.find_descendants(squashed_root) - - # Add imports to/from the root. - for descendant in descendants: - for imported_module in self.find_modules_directly_imported_by(descendant): - self.add_import(importer=squashed_root, imported=imported_module) - for importing_module in self.find_modules_that_directly_import(descendant): - self.add_import(importer=importing_module, imported=squashed_root) - - # Now we've added imports to/from the root, we can delete the root's descendants. - for descendant in descendants: - self.remove_module(descendant) - - self._mark_module_as_squashed(squashed_root) + if module not in self.modules: + raise ModuleNotPresent(f'"{module}" not present in the graph.') + self._rustgraph.squash_module(module) def is_module_squashed(self, module: str) -> bool: if module not in self.modules: raise ModuleNotPresent(f'"{module}" not present in the graph.') - - return module in self._squashed_modules + return self._rustgraph.is_module_squashed(module) def add_import( self, @@ -129,344 +45,149 @@ def add_import( line_number: Optional[int] = None, line_contents: Optional[str] = None, ) -> None: - if any((line_number, line_contents)): - if not all((line_number, line_contents)): - raise ValueError( - "Line number and contents must be provided together, or not at all." - ) - self._import_details.setdefault(importer, []) - self._import_details[importer].append( - { - "importer": importer, - "imported": imported, - "line_number": cast(int, line_number), - "line_contents": cast(str, line_contents), - } - ) - - importer_map = self._importeds_by_importer.setdefault(importer, set()) - imported_map = self._importers_by_imported.setdefault(imported, set()) - if imported not in importer_map: - # (Alternatively could check importer in imported_map.) - importer_map.add(imported) - imported_map.add(importer) - self._edge_count += 1 - - # Also ensure they have entry in other maps. - self._importeds_by_importer.setdefault(imported, set()) - self._importers_by_imported.setdefault(importer, set()) + self._rustgraph.add_import( + importer=importer, + imported=imported, + line_number=line_number, + line_contents=line_contents, + ) def remove_import(self, *, importer: str, imported: str) -> None: - if imported in self._importeds_by_importer[importer]: - self._importeds_by_importer[importer].remove(imported) - self._importers_by_imported[imported].remove(importer) - self._edge_count -= 1 - - # Clean up import details. - if importer in self._import_details: - new_details = [ - details - for details in self._import_details[importer] - if details["imported"] != imported - ] - if new_details: - self._import_details[importer] = new_details - else: - del self._import_details[importer] + return self._rustgraph.remove_import(importer=importer, imported=imported) def count_imports(self) -> int: - return self._edge_count - - # Descendants - # ----------- + return self._rustgraph.count_imports() def find_children(self, module: str) -> Set[str]: # It doesn't make sense to find the children of a squashed module, as we don't store # the children in the graph. if self.is_module_squashed(module): raise ValueError("Cannot find children of a squashed module.") - - children = set() - for potential_child in self.modules: - if Module(potential_child).is_child_of(Module(module)): - children.add(potential_child) - return children + return self._rustgraph.find_children(module) def find_descendants(self, module: str) -> Set[str]: # It doesn't make sense to find the descendants of a squashed module, as we don't store # the descendants in the graph. if self.is_module_squashed(module): raise ValueError("Cannot find descendants of a squashed module.") - - descendants = set() - for potential_descendant in self.modules: - if Module(potential_descendant).is_descendant_of(Module(module)): - descendants.add(potential_descendant) - return descendants - - # Direct imports - # -------------- + return self._rustgraph.find_descendants(module) def direct_import_exists( self, *, importer: str, imported: str, as_packages: bool = False ) -> bool: - if not as_packages: - return imported in self.find_modules_directly_imported_by(importer) - - importer_modules = self._all_modules_in_package(importer) - imported_modules = self._all_modules_in_package(imported) - - if importer_modules & imported_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - # Return True as soon as we find a path between any of the modules in the subpackages. - for candidate_importer in importer_modules: - imported_by_importer = self.find_modules_directly_imported_by(candidate_importer) - for candidate_imported in imported_modules: - if candidate_imported in imported_by_importer: - return True - return False + return self._rustgraph.direct_import_exists( + importer=importer, imported=imported, as_packages=as_packages + ) def find_modules_directly_imported_by(self, module: str) -> Set[str]: - return self._importeds_by_importer[module] + return self._rustgraph.find_modules_directly_imported_by(module) def find_modules_that_directly_import(self, module: str) -> Set[str]: - return self._importers_by_imported[module] - - def get_import_details(self, *, importer: str, imported: str) -> List[graph.DetailedImport]: - import_details_for_importer = self._import_details.get(importer, []) - # Only include the details for the imported module. - # Note: we copy each details dictionary at this point, as our deepcopying - # only copies the dictionaries by reference. - return [i.copy() for i in import_details_for_importer if i["imported"] == imported] + if module in self._rustgraph.get_modules(): + # TODO panics if module isn't in modules. + return self._rustgraph.find_modules_that_directly_import(module) + return set() - # Indirect imports - # ---------------- + def get_import_details(self, *, importer: str, imported: str) -> List[DetailedImport]: + return self._rustgraph.get_import_details( + importer=importer, + imported=imported, + ) def find_downstream_modules(self, module: str, as_package: bool = False) -> Set[str]: - # TODO optimise for as_package. - if as_package: - source_modules = self._all_modules_in_package(module) - else: - source_modules = {module} - - downstream_modules = set() - - for candidate in filter(lambda m: m not in source_modules, self.modules): - for source_module in source_modules: - if self.chain_exists(importer=candidate, imported=source_module): - downstream_modules.add(candidate) - break - - return downstream_modules + return self._rustgraph.find_downstream_modules(module, as_package) def find_upstream_modules(self, module: str, as_package: bool = False) -> Set[str]: - # TODO optimise for as_package. - if as_package: - destination_modules = self._all_modules_in_package(module) - else: - destination_modules = {module} - - upstream_modules = set() - - for candidate in filter(lambda m: m not in destination_modules, self.modules): - for destination_module in destination_modules: - if self.chain_exists(importer=destination_module, imported=candidate): - upstream_modules.add(candidate) - break - - return upstream_modules + return self._rustgraph.find_upstream_modules(module, as_package) - def find_shortest_chain(self, importer: str, imported: str) -> Optional[Tuple[str, ...]]: + def find_shortest_chain(self, importer: str, imported: str) -> tuple[str, ...] | None: for module in (importer, imported): if module not in self.modules: raise ValueError(f"Module {module} is not present in the graph.") - return self._find_shortest_chain(importer=importer, imported=imported) + chain = self._rustgraph.find_shortest_chain(importer, imported) + return tuple(chain) if chain else None def find_shortest_chains( self, importer: str, imported: str, as_packages: bool = True ) -> Set[Tuple[str, ...]]: - """ - Find the shortest import chains that exist between the importer and imported, and - between any modules contained within them if as_packages is True. Only one chain per - upstream/downstream pair will be included. Any chains that are contained within other - chains in the result set will be excluded. - - The default behavior is to treat the import and imported as packages, however, if - as_packages is False, both the importer and imported will be treated as modules instead. - - Returns: - A set of tuples of strings. Each tuple is ordered from importer to imported modules. - """ - shortest_chains = set() - - upstream_modules = ( - {imported} if not as_packages else self._all_modules_in_package(imported) - ) - downstream_modules = ( - {importer} if not as_packages else self._all_modules_in_package(importer) - ) - - if upstream_modules & downstream_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - imports_between_modules = self._find_all_imports_between_modules( - upstream_modules - ) | self._find_all_imports_between_modules(downstream_modules) - self._hide_any_existing_imports(imports_between_modules) - - map_of_imports = {} - for module in upstream_modules | downstream_modules: - map_of_imports[module] = set( - (m, module) for m in self.find_modules_that_directly_import(module) - ) | set((module, m) for m in self.find_modules_directly_imported_by(module)) - for imports in map_of_imports.values(): - self._hide_any_existing_imports(imports) - - for upstream in upstream_modules: - imports_of_upstream_module = map_of_imports[upstream] - self._reveal_imports(imports_of_upstream_module) - for downstream in downstream_modules: - imports_by_downstream_module = map_of_imports[downstream] - self._reveal_imports(imports_by_downstream_module) - shortest_chain = self._find_shortest_chain(imported=upstream, importer=downstream) - if shortest_chain: - shortest_chains.add(shortest_chain) - self._hide_any_existing_imports(imports_by_downstream_module) - self._hide_any_existing_imports(imports_of_upstream_module) - - # Reveal all the hidden imports. - for imports in map_of_imports.values(): - self._reveal_imports(imports) - self._reveal_imports(imports_between_modules) - - return shortest_chains + return self._rustgraph.find_shortest_chains(importer, imported, as_packages) def chain_exists(self, importer: str, imported: str, as_packages: bool = False) -> bool: - if not as_packages: - return bool(self._find_shortest_chain(importer=importer, imported=imported)) - - upstream_modules = self._all_modules_in_package(imported) - downstream_modules = self._all_modules_in_package(importer) - - if upstream_modules & downstream_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - # Return True as soon as we find a path between any of the modules in the subpackages. - for upstream in upstream_modules: - for downstream in downstream_modules: - if self.chain_exists(imported=upstream, importer=downstream): - return True - - return False - - # High level analysis + return self._rustgraph.chain_exists(importer, imported, as_packages) def find_illegal_dependencies_for_layers( self, layers: Sequence[Layer | str | set[str]], containers: set[str] | None = None, ) -> set[PackageDependency]: - layers = _layers.parse_layers(layers) - return _layers.find_illegal_dependencies( - graph=self, layers=layers, containers=containers or set() - ) + layers = _parse_layers(layers) + try: + result = self._rustgraph.find_illegal_dependencies_for_layers( + layers=tuple( + {"layers": layer.module_tails, "independent": layer.independent} + for layer in layers + ), + containers=set(containers) if containers else set(), + ) + except rust.NoSuchContainer as e: + raise NoSuchContainer(str(e)) - # Private methods + return _dependencies_from_tuple(result) - def _find_ancestor_squashed_module(self, module: str) -> Optional[str]: - """ - Return the name of a squashed module that is an ancestor of the supplied module, or None - if no such module exists. - """ - try: - parent = Module(module).parent.name - except ValueError: - # The module has no more ancestors. - return None + # Dunder methods + # -------------- - if parent in self.modules and self.is_module_squashed(parent): - return parent - else: - return self._find_ancestor_squashed_module(parent) - - def _mark_module_as_squashed(self, module: str) -> None: - """ - Set a flag on a module in the graph that it is squashed. - """ - self._squashed_modules.add(module) - - def _all_modules_in_package(self, module: str) -> Set[str]: - """ - Return all the modules in the supplied module, including itself. - - If the module is squashed, it will be treated as a single module. - """ - importer_modules = {module} - if not self.is_module_squashed(module): - importer_modules |= self.find_descendants(module) - return importer_modules - - def _find_all_imports_between_modules(self, modules: Set[str]) -> Set[Tuple[str, str]]: - """ - Return all the imports between the supplied set of modules. - - Return: - Set of imports, in the form (importer, imported). - """ - imports = set() - for importer in modules: - for imported in self.find_modules_directly_imported_by(importer): - if imported in modules: - imports.add((importer, imported)) - return imports - - def _hide_any_existing_imports(self, imports: Set[Tuple[str, str]]) -> None: - """ - Temporarily remove the supplied direct imports from the graph. - - If an import is not in the graph, or already hidden, this will have no effect. - - Args: - imports: Set of direct imports, in the form (importer, imported). - """ - for importer, imported in tuple(imports): - if self.direct_import_exists(importer=importer, imported=imported): - # Low-level removal from import graph (but leaving other metadata in place). - self._importeds_by_importer[importer].remove(imported) - self._importers_by_imported[imported].remove(importer) - - def _reveal_imports(self, imports: Set[Tuple[str, str]]) -> None: - """ - Given a set of direct imports that were hidden by _hide_any_existing_imports, add them back. - - Args: - imports: Set of direct imports, in the form (importer, imported). - """ - for importer, imported in tuple(imports): - # Low-level addition to import graph. - self._importeds_by_importer[importer].add(imported) - self._importers_by_imported[imported].add(importer) - - def _find_shortest_chain(self, importer: str, imported: str) -> Optional[Tuple[str, ...]]: - # Similar to find_shortest_chain but without bothering to check if the modules are - # in the graph first. - return bidirectional_shortest_path( - importers_by_imported=self._importers_by_imported, - importeds_by_importer=self._importeds_by_importer, - importer=importer, - imported=imported, - ) + def __deepcopy__(self, memodict: dict) -> "ImportGraph": + new_graph = ImportGraph() + new_graph._rustgraph = self._rustgraph.clone() + return new_graph -_StringSet = Set[str] +class _RustRoute(TypedDict): + heads: frozenset[str] + middle: tuple[str, ...] + tails: frozenset[str] + + +class _RustPackageDependency(TypedDict): + importer: str + imported: str + routes: tuple[_RustRoute, ...] + + +def _parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]: + """ + Convert the passed raw `layers` into `Layer`s. + """ + out_layers = [] + for layer in layers: + if isinstance(layer, Layer): + out_layers.append(layer) + elif isinstance(layer, str): + out_layers.append(Layer(layer, independent=True)) + else: + out_layers.append(Layer(*tuple(layer), independent=True)) + return tuple(out_layers) + + +def _dependencies_from_tuple( + rust_package_dependency_tuple: tuple[_RustPackageDependency, ...] +) -> set[PackageDependency]: + return { + PackageDependency( + imported=dep_dict["imported"], + importer=dep_dict["importer"], + routes=frozenset( + { + Route( + heads=route_dict["heads"], + middle=route_dict["middle"], + tails=route_dict["tails"], + ) + for route_dict in dep_dict["routes"] + } + ), + ) + for dep_dict in rust_package_dependency_tuple + } From 895080d36ddd6b7ebdb79f2a8658500296f63ee3 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 13:14:01 +0000 Subject: [PATCH 4/5] Remove unused code --- rust/src/containers.rs | 14 - rust/src/dependencies.rs | 13 - rust/src/importgraph.rs | 435 --------------------- rust/src/layers.rs | 699 ---------------------------------- src/grimp/adaptors/_layers.py | 172 --------- 5 files changed, 1333 deletions(-) delete mode 100644 rust/src/containers.rs delete mode 100644 rust/src/dependencies.rs delete mode 100644 rust/src/importgraph.rs delete mode 100644 rust/src/layers.rs delete mode 100644 src/grimp/adaptors/_layers.py diff --git a/rust/src/containers.rs b/rust/src/containers.rs deleted file mode 100644 index 6f17116d..00000000 --- a/rust/src/containers.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::importgraph::ImportGraph; -use std::collections::HashSet; - -pub fn check_containers_exist<'a>( - graph: &'a ImportGraph, - containers: &'a HashSet, -) -> Result<(), String> { - for container in containers { - if !graph.contains_module(container) { - return Err(format!("Container {} does not exist.", container)); - } - } - Ok(()) -} diff --git a/rust/src/dependencies.rs b/rust/src/dependencies.rs deleted file mode 100644 index 5ed9da01..00000000 --- a/rust/src/dependencies.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Route { - pub heads: Vec, - pub middle: Vec, - pub tails: Vec, -} - -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct PackageDependency { - pub importer: u32, - pub imported: u32, - pub routes: Vec, -} diff --git a/rust/src/importgraph.rs b/rust/src/importgraph.rs deleted file mode 100644 index cf58aaf9..00000000 --- a/rust/src/importgraph.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::collections::hash_map::Entry::Vacant; -use std::collections::{HashMap, HashSet}; -use std::fmt; - -#[derive(Clone)] -pub struct ImportGraph<'a> { - pub names_by_id: HashMap, - pub ids_by_name: HashMap<&'a str, u32>, - pub importers_by_imported: HashMap>, - pub importeds_by_importer: HashMap>, -} - -impl<'a> ImportGraph<'a> { - pub fn new(importeds_by_importer: HashMap<&'a str, HashSet<&'a str>>) -> ImportGraph<'a> { - // Build the name/id lookup maps. - let mut names_by_id: HashMap = HashMap::new(); - let mut ids_by_name: HashMap<&'a str, u32> = HashMap::new(); - let mut current_id: u32 = 1; - for name in importeds_by_importer.keys() { - names_by_id.insert(current_id, name); - ids_by_name.insert(name, current_id); - current_id += 1; - } - - // Convert importeds_by_importer to id-based. - let mut importeds_by_importer_u32: HashMap> = HashMap::new(); - for (importer_str, importeds_strs) in importeds_by_importer.iter() { - let mut importeds_u32 = HashSet::new(); - for imported_str in importeds_strs { - importeds_u32.insert(*ids_by_name.get(imported_str).unwrap()); - } - - importeds_by_importer_u32 - .insert(*ids_by_name.get(importer_str).unwrap(), importeds_u32); - } - - let importers_by_imported_u32 = - ImportGraph::_build_importers_by_imported_u32(&importeds_by_importer_u32); - - ImportGraph { - names_by_id, - ids_by_name, - importers_by_imported: importers_by_imported_u32, - importeds_by_importer: importeds_by_importer_u32, - } - } - - fn _build_importers_by_imported_u32( - importeds_by_importer_u32: &HashMap>, - ) -> HashMap> { - // Build importers_by_imported from importeds_by_importer. - let mut importers_by_imported_u32: HashMap> = HashMap::new(); - for (importer, importeds) in importeds_by_importer_u32.iter() { - for imported in importeds { - let entry = importers_by_imported_u32.entry(*imported).or_default(); - entry.insert(*importer); - } - } - - // Check that there is an empty set for any remaining. - for importer in importeds_by_importer_u32.keys() { - importers_by_imported_u32.entry(*importer).or_default(); - } - importers_by_imported_u32 - } - - pub fn get_module_ids(&self) -> HashSet { - self.names_by_id.keys().copied().collect() - } - - pub fn contains_module(&self, module_name: &str) -> bool { - self.ids_by_name.contains_key(module_name) - } - - pub fn remove_import(&mut self, importer: &str, imported: &str) { - self.remove_import_ids(self.ids_by_name[importer], self.ids_by_name[imported]); - } - - pub fn add_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.insert(imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.insert(importer); - } - - pub fn remove_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.remove(&imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.remove(&importer); - } - - pub fn remove_module_by_id(&mut self, module_id: u32) { - let _module = self.names_by_id[&module_id]; - - let mut imports_to_remove = Vec::with_capacity(self.names_by_id.len()); - { - for imported_id in &self.importeds_by_importer[&module_id] { - imports_to_remove.push((module_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&module_id] { - imports_to_remove.push((*importer_id, module_id)); - } - } - - for (importer, imported) in imports_to_remove { - self.remove_import_ids(importer, imported); - } - - self.importeds_by_importer.remove(&module_id); - self.importers_by_imported.remove(&module_id); - } - - pub fn get_descendant_ids(&self, module_name: &str) -> Vec { - let mut descendant_ids = vec![]; - let namespace: String = format!("{}.", module_name); - for (candidate_name, candidate_id) in &self.ids_by_name { - if candidate_name.starts_with(&namespace) { - descendant_ids.push(*candidate_id); - } - } - descendant_ids - } - - pub fn remove_package(&mut self, module_name: &str) { - for descendant_id in self.get_descendant_ids(module_name) { - self.remove_module_by_id(descendant_id); - } - self.remove_module_by_id(self.ids_by_name[&module_name]); - } - - pub fn squash_module(&mut self, module_name: &str) { - let squashed_root_id = self.ids_by_name[module_name]; - let descendant_ids = &self.get_descendant_ids(module_name); - - // Assemble imports to add first, then add them in a second loop, - // to avoid needing to clone importeds_by_importer. - let mut imports_to_add = Vec::with_capacity(self.names_by_id.len()); - // Imports from the root. - { - for descendant_id in descendant_ids { - for imported_id in &self.importeds_by_importer[&descendant_id] { - imports_to_add.push((squashed_root_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&descendant_id] { - imports_to_add.push((*importer_id, squashed_root_id)); - } - } - } - - for (importer, imported) in imports_to_add { - self.add_import_ids(importer, imported); - } - - // Now we've added imports to/from the root, we can delete the root's descendants. - for descendant_id in descendant_ids { - self.remove_module_by_id(*descendant_id); - } - } - - pub fn pop_shortest_chains(&mut self, importer: &str, imported: &str) -> Vec> { - let mut chains = vec![]; - let importer_id = self.ids_by_name[&importer]; - let imported_id = self.ids_by_name[&imported]; - - while let Some(chain) = self.find_shortest_chain(importer_id, imported_id) { - // Remove chain - let _mods: Vec<&str> = chain.iter().map(|i| self.names_by_id[&i]).collect(); - for i in 0..chain.len() - 1 { - self.remove_import_ids(chain[i], chain[i + 1]); - } - chains.push(chain); - } - - chains - } - - pub fn find_shortest_chain(&self, importer_id: u32, imported_id: u32) -> Option> { - let results_or_none = self._search_for_path(importer_id, imported_id); - match results_or_none { - Some(results) => { - let (pred, succ, initial_w) = results; - - let mut w_or_none: Option = Some(initial_w); - // Transform results into vector. - let mut path: Vec = Vec::new(); - // From importer to w: - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = pred[&w]; - } - path.reverse(); - - // From w to imported: - w_or_none = succ[path.last().unwrap()]; - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = succ[&w]; - } - - Some(path) - } - None => None, - } - } - /// Performs a breadth first search from both source and target, meeting in the middle. - // - // Returns: - // (pred, succ, w) where - // - pred is a dictionary of predecessors from w to the source, and - // - succ is a dictionary of successors from w to the target. - // - fn _search_for_path( - &self, - importer: u32, - imported: u32, - ) -> Option<(HashMap>, HashMap>, u32)> { - if importer == imported { - Some(( - HashMap::from([(imported, None)]), - HashMap::from([(importer, None)]), - importer, - )) - } else { - let mut pred: HashMap> = HashMap::from([(importer, None)]); - let mut succ: HashMap> = HashMap::from([(imported, None)]); - - // Initialize fringes, start with forward. - let mut forward_fringe: Vec = Vec::from([importer]); - let mut reverse_fringe: Vec = Vec::from([imported]); - let mut this_level: Vec; - - while !forward_fringe.is_empty() && !reverse_fringe.is_empty() { - if forward_fringe.len() <= reverse_fringe.len() { - this_level = forward_fringe.to_vec(); - forward_fringe = Vec::new(); - for v in this_level { - for w in self.importeds_by_importer[&v].clone() { - pred.entry(w).or_insert_with(|| { - forward_fringe.push(w); - Some(v) - }); - if succ.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } else { - this_level = reverse_fringe.to_vec(); - reverse_fringe = Vec::new(); - for v in this_level { - for w in self.importers_by_imported[&v].clone() { - if let Vacant(e) = succ.entry(w) { - e.insert(Some(v)); - reverse_fringe.push(w); - } - if pred.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } - } - None - } - } -} - -impl fmt::Display for ImportGraph<'_> { - fn fmt(&self, dest: &mut fmt::Formatter) -> fmt::Result { - let mut strings = vec![]; - for (importer, importeds) in self.importeds_by_importer.iter() { - let mut string = format!("IMPORTER {}: ", self.names_by_id[&importer]); - for imported in importeds { - string.push_str(format!("{}, ", self.names_by_id[&imported]).as_str()); - } - strings.push(string); - } - strings.push(" ".to_string()); - for (imported, importers) in self.importers_by_imported.iter() { - let mut string = format!("IMPORTED {}: ", self.names_by_id[&imported]); - for importer in importers { - string.push_str(format!("{}, ", self.names_by_id[&importer]).as_str()); - } - strings.push(string); - } - write!(dest, "{}", strings.join("\n")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn _make_graph() -> ImportGraph<'static> { - ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["blue.alpha", "blue.beta", "green"])), - ("blue.alpha", HashSet::new()), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ])) - } - - #[test] - fn get_module_ids() { - let graph = _make_graph(); - - assert_eq!( - graph.get_module_ids(), - HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]) - ); - } - - #[test] - fn new_stores_importeds_by_importer_using_id() { - let graph = _make_graph(); - - let expected_importeds: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importeds_by_importer - .get(graph.ids_by_name.get("blue").unwrap()) - .unwrap(), - expected_importeds - ); - } - - #[test] - fn new_stores_importers_by_imported_using_id() { - let graph = _make_graph(); - - let expected_importers: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importers_by_imported - .get(graph.ids_by_name.get("blue.alpha").unwrap()) - .unwrap(), - expected_importers - ); - } - - #[test] - fn test_squash_module() { - let mut graph = ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["orange", "green"])), - ("blue.alpha", HashSet::from(["green.delta"])), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ("green.gamma", HashSet::new()), - ("green.delta", HashSet::new()), - ("orange", HashSet::new()), - ])); - - graph.squash_module("blue"); - - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["blue"]], - HashSet::from([ - graph.ids_by_name["orange"], - graph.ids_by_name["green"], - graph.ids_by_name["green.delta"], - ]) - ); - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["orange"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green.delta"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["blue"]], - HashSet::from([graph.ids_by_name["green"],]) - ); - } - - #[test] - fn test_find_shortest_chain() { - let blue = "blue"; - let green = "green"; - let yellow = "yellow"; - let blue_alpha = "blue.alpha"; - let blue_beta = "blue.beta"; - - let graph = ImportGraph::new(HashMap::from([ - (green, HashSet::from([blue])), - (blue_alpha, HashSet::from([blue])), - (yellow, HashSet::from([green])), - (blue_beta, HashSet::from([green])), - (blue, HashSet::new()), - ])); - - let path_or_none: Option> = - graph.find_shortest_chain(graph.ids_by_name[&yellow], graph.ids_by_name[&blue]); - - assert_eq!( - path_or_none, - Some(Vec::from([ - graph.ids_by_name[&yellow], - graph.ids_by_name[&green], - graph.ids_by_name[&blue] - ])) - ); - } -} diff --git a/rust/src/layers.rs b/rust/src/layers.rs deleted file mode 100644 index 0d5257fa..00000000 --- a/rust/src/layers.rs +++ /dev/null @@ -1,699 +0,0 @@ -use crate::dependencies::{PackageDependency, Route}; -use crate::importgraph::ImportGraph; - -use log::info; -use rayon::prelude::*; -use std::collections::HashSet; -use std::time::Instant; - -/// A group of layers at the same level in the layering. -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Level { - pub layers: Vec, - pub independent: bool, -} - -pub fn find_illegal_dependencies<'a>( - graph: &'a ImportGraph, - levels: &'a Vec, - containers: &'a HashSet, -) -> Vec { - let layers = _layers_from_levels(levels); - - _generate_module_permutations(graph, levels, containers) - .into_par_iter() - .filter_map(|(higher_layer_package, lower_layer_package, container)| { - // TODO: it's inefficient to do this for sibling layers, as we don't need - // to clone and trim the graph for identical pairs. - info!( - "Searching for import chains from {} to {}...", - lower_layer_package, higher_layer_package - ); - let now = Instant::now(); - let dependency_or_none = _search_for_package_dependency( - &higher_layer_package, - &lower_layer_package, - &layers, - &container, - graph, - ); - _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); - dependency_or_none - }) - .collect() -} - -/// Return every permutation of modules that exist in the graph -/// in which the second should not import the first. -fn _generate_module_permutations<'a>( - graph: &'a ImportGraph, - levels: &'a [Level], - containers: &'a HashSet, -) -> Vec<(String, String, Option)> { - let mut permutations: Vec<(String, String, Option)> = vec![]; - - let quasi_containers: Vec> = if containers.is_empty() { - vec![None] - } else { - containers.iter().map(|i| Some(i.to_string())).collect() - }; - for container in quasi_containers { - for (index, higher_level) in levels.iter().enumerate() { - for higher_layer in &higher_level.layers { - let higher_layer_module_name = _module_from_layer(higher_layer, &container); - if graph - .ids_by_name - .get(&higher_layer_module_name as &str) - .is_none() - { - continue; - } - - // Build the layers that mustn't import this higher layer. - // That includes: - // * lower layers. - // * sibling layers, if the layer is independent. - let mut layers_forbidden_to_import_higher_layer: Vec<&str> = vec![]; - - if higher_level.independent { - for potential_sibling_layer in &higher_level.layers { - if potential_sibling_layer != higher_layer { - // It's a sibling layer. - layers_forbidden_to_import_higher_layer.push(potential_sibling_layer); - } - } - } - - for lower_level in &levels[index + 1..] { - for lower_layer in &lower_level.layers { - layers_forbidden_to_import_higher_layer.push(lower_layer); - } - } - - // Now turn the layers into modules, if they exist. - for forbidden_layer in &layers_forbidden_to_import_higher_layer { - let forbidden_module_name = _module_from_layer(forbidden_layer, &container); - if let Some(_value) = graph.ids_by_name.get(&forbidden_module_name as &str) { - permutations.push(( - higher_layer_module_name.clone(), - forbidden_module_name.clone(), - container.clone(), - )); - }; - } - } - } - } - - permutations -} - -fn _module_from_layer<'a>(module: &'a str, container: &'a Option) -> String { - match container { - Some(true_container) => format!("{}.{}", true_container, module), - None => module.to_string(), - } -} - -fn _search_for_package_dependency<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - layers: &'a Vec<&'a str>, - container: &'a Option, - graph: &'a ImportGraph, -) -> Option { - let mut temp_graph = graph.clone(); - _remove_other_layers( - &mut temp_graph, - layers, - container, - (higher_layer_package, lower_layer_package), - ); - let mut routes: Vec = vec![]; - - // Direct routes. - let direct_links = - _pop_direct_imports(higher_layer_package, lower_layer_package, &mut temp_graph); - for (importer, imported) in direct_links { - routes.push(Route { - heads: vec![importer], - middle: vec![], - tails: vec![imported], - }); - } - - // Indirect routes. - for indirect_route in - _get_indirect_routes(higher_layer_package, lower_layer_package, &temp_graph) - { - routes.push(indirect_route); - } - if routes.is_empty() { - None - } else { - Some(PackageDependency { - imported: graph.ids_by_name[&higher_layer_package], - importer: graph.ids_by_name[&lower_layer_package], - routes, - }) - } -} - -fn _layers_from_levels<'a>(levels: &'a Vec) -> Vec<&'a str> { - let mut layers: Vec<&str> = vec![]; - for level in levels { - layers.extend(level.layers.iter().map(|s| s.as_str())); - } - layers -} - -fn _remove_other_layers<'a>( - graph: &'a mut ImportGraph, - layers: &'a Vec<&'a str>, - container: &'a Option, - layers_to_preserve: (&'a str, &'a str), -) { - for layer in layers { - let layer_module = _module_from_layer(layer, container); - if layers_to_preserve.0 == layer_module || layers_to_preserve.1 == layer_module { - continue; - } - if graph.contains_module(&layer_module) { - graph.remove_package(&layer_module); - } - } -} - -fn _pop_direct_imports<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - graph: &'a mut ImportGraph, -) -> HashSet<(u32, u32)> { - // Remove the direct imports, returning them as (importer, imported) tuples. - let mut imports = HashSet::new(); - - let higher_layer_namespace: String = format!("{}.", higher_layer_package); - let mut lower_layer_module_ids: Vec = vec![graph.ids_by_name[lower_layer_package]]; - lower_layer_module_ids.append(&mut graph.get_descendant_ids(lower_layer_package)); - - for lower_layer_module_id in lower_layer_module_ids { - let _lower = graph.names_by_id[&lower_layer_module_id]; - let imported_module_ids = graph.importeds_by_importer[&lower_layer_module_id].clone(); - for imported_module_id in imported_module_ids { - let imported_module = graph.names_by_id[&imported_module_id]; - - if imported_module.starts_with(&higher_layer_namespace) - || imported_module == higher_layer_package - { - imports.insert((lower_layer_module_id, imported_module_id)); - graph.remove_import_ids(lower_layer_module_id, imported_module_id) - } - } - } - imports -} - -fn _get_indirect_routes<'a>( - imported_package: &'a str, - importer_package: &'a str, - graph: &'a ImportGraph, -) -> Vec { - // Squashes the two packages. - // Gets a list of paths between them, called middles. - // Add the heads and tails to the middles. - let mut temp_graph = graph.clone(); - temp_graph.squash_module(imported_package); - temp_graph.squash_module(importer_package); - - let middles = _find_middles(&mut temp_graph, importer_package, imported_package); - _middles_to_routes(graph, middles, importer_package, imported_package) -} - -fn _find_middles<'a>( - graph: &'a mut ImportGraph, - importer: &'a str, - imported: &'a str, -) -> Vec> { - let mut middles = vec![]; - - for chain in graph.pop_shortest_chains(importer, imported) { - // Remove first and last element. - // TODO surely there's a better way? - let mut middle: Vec = vec![]; - let chain_length = chain.len(); - for (index, module) in chain.iter().enumerate() { - if index != 0 && index != chain_length - 1 { - middle.push(*module); - } - } - middles.push(middle); - } - - middles -} - -fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { - let route_count = match dependency_or_none { - Some(dependency) => dependency.routes.len(), - None => 0, - }; - let pluralized = if route_count == 1 { "" } else { "s" }; - info!( - "Found {} illegal route{} in {}s.", - route_count, pluralized, duration_in_s - ); -} - -fn _middles_to_routes<'a>( - graph: &'a ImportGraph, - middles: Vec>, - importer: &'a str, - imported: &'a str, -) -> Vec { - let mut routes = vec![]; - let importer_id = graph.ids_by_name[&importer]; - let imported_id = graph.ids_by_name[&imported]; - - for middle in middles { - // Construct heads. - let mut heads: Vec = vec![]; - let first_imported_id = middle[0]; - let candidate_modules = &graph.importers_by_imported[&first_imported_id]; - for candidate_module in candidate_modules { - if importer_id == *candidate_module - || graph - .get_descendant_ids(importer) - .contains(candidate_module) - { - heads.push(*candidate_module); - } - } - - // Construct tails. - let mut tails: Vec = vec![]; - let last_importer_id = middle[middle.len() - 1]; - let candidate_modules = &graph.importeds_by_importer[&last_importer_id]; - for candidate_module in candidate_modules { - if imported_id == *candidate_module - || graph - .get_descendant_ids(imported) - .contains(candidate_module) - { - tails.push(*candidate_module); - } - } - routes.push(Route { - heads, - middle, - tails, - }) - } - - routes -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn test_find_illegal_dependencies_no_container() { - let graph = ImportGraph::new(HashMap::from([ - ("low", HashSet::new()), - ("low.blue", HashSet::from(["utils"])), - ("low.green", HashSet::new()), - ("low.green.alpha", HashSet::from(["high.yellow"])), - ("mid_a", HashSet::from(["mid_b"])), - ("mid_a.orange", HashSet::new()), - ("mid_b", HashSet::from(["mid_c"])), - ("mid_b.brown", HashSet::new()), - ("mid_c", HashSet::new()), - ("mid_c.purple", HashSet::new()), - ("high", HashSet::from(["low.blue"])), - ("high.yellow", HashSet::new()), - ("high.red", HashSet::new()), - ("high.red.beta", HashSet::new()), - ("utils", HashSet::from(["high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::new(); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![ - PackageDependency { - importer: *graph.ids_by_name.get("low").unwrap(), - imported: *graph.ids_by_name.get("high").unwrap(), - routes: vec![ - Route { - heads: vec![*graph.ids_by_name.get("low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("high.yellow").unwrap()], - }, - Route { - heads: vec![*graph.ids_by_name.get("low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("utils").unwrap()], - tails: vec![*graph.ids_by_name.get("high.red").unwrap()], - }, - ], - }, - PackageDependency { - importer: *graph.ids_by_name.get("mid_a").unwrap(), - imported: *graph.ids_by_name.get("mid_b").unwrap(), - routes: vec![Route { - heads: vec![*graph.ids_by_name.get("mid_a").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_b").unwrap()], - },], - }, - PackageDependency { - importer: *graph.ids_by_name.get("mid_b").unwrap(), - imported: *graph.ids_by_name.get("mid_c").unwrap(), - routes: vec![Route { - heads: vec![*graph.ids_by_name.get("mid_b").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_c").unwrap()], - },], - }, - ] - ); - } - - #[test] - fn test_find_illegal_dependencies_with_container() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![PackageDependency { - importer: *graph.ids_by_name.get("mypackage.low").unwrap(), - imported: *graph.ids_by_name.get("mypackage.high").unwrap(), - routes: vec![ - Route { - heads: vec![*graph.ids_by_name.get("mypackage.low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mypackage.high.yellow").unwrap()], - }, - Route { - heads: vec![*graph.ids_by_name.get("mypackage.low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("mypackage.utils").unwrap()], - tails: vec![*graph.ids_by_name.get("mypackage.high.red").unwrap()], - }, - ], - }] - ); - } - - #[test] - fn test_generate_module_permutations() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_generate_module_permutations_sibling_layer_not_independent() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: false, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_layers_from_levels() { - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "medium_a".to_string(), - "medium_b".to_string(), - "medium_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - - let result = _layers_from_levels(&levels); - - assert_eq!( - HashSet::<&str>::from_iter(result), - HashSet::from_iter(["high", "medium_a", "medium_b", "medium_c", "low",]), - ) - } -} diff --git a/src/grimp/adaptors/_layers.py b/src/grimp/adaptors/_layers.py deleted file mode 100644 index 7ff3c7e4..00000000 --- a/src/grimp/adaptors/_layers.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterator, Sequence, TypedDict - -from grimp import Route -from grimp import _rustgrimp as rust # type: ignore[attr-defined] - -if TYPE_CHECKING: - from grimp.adaptors.graph import ImportGraph - -from grimp.domain.analysis import PackageDependency -from grimp.exceptions import NoSuchContainer -from grimp.domain.valueobjects import Layer - - -def parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]: - """ - Convert the passed raw `layers` into `Layer`s. - """ - out_layers = [] - for layer in layers: - if isinstance(layer, Layer): - out_layers.append(layer) - elif isinstance(layer, str): - out_layers.append(Layer(layer, independent=True)) - else: - out_layers.append(Layer(*tuple(layer), independent=True)) - return tuple(out_layers) - - -def find_illegal_dependencies( - graph: ImportGraph, - layers: Sequence[Layer], - containers: set[str], -) -> set[PackageDependency]: - """ - Find dependencies that don't conform to the supplied layered architecture. - - See ImportGraph.find_illegal_dependencies_for_layers. - - The only difference between this and the method is that the containers passed in - is already a (potentially empty) set. - """ - try: - rust_package_dependency_tuple = rust.find_illegal_dependencies( - levels=tuple( - {"layers": layer.module_tails, "independent": layer.independent} - for layer in layers - ), - containers=set(containers), - importeds_by_importer=graph._importeds_by_importer, - ) - except rust.NoSuchContainer as e: - raise NoSuchContainer(str(e)) - - rust_package_dependencies = _dependencies_from_tuple(rust_package_dependency_tuple) - return rust_package_dependencies - - -class _RustRoute(TypedDict): - heads: frozenset[str] - middle: tuple[str, ...] - tails: frozenset[str] - - -class _RustPackageDependency(TypedDict): - importer: str - imported: str - routes: tuple[_RustRoute, ...] - - -def _dependencies_from_tuple( - rust_package_dependency_tuple: tuple[_RustPackageDependency, ...] -) -> set[PackageDependency]: - return { - PackageDependency( - imported=dep_dict["imported"], - importer=dep_dict["importer"], - routes=frozenset( - { - Route( - heads=route_dict["heads"], - middle=route_dict["middle"], - tails=route_dict["tails"], - ) - for route_dict in dep_dict["routes"] - } - ), - ) - for dep_dict in rust_package_dependency_tuple - } - - -class _Module: - """ - A Python module. - """ - - def __init__(self, name: str) -> None: - """ - Args: - name: The fully qualified name of a Python module, e.g. 'package.foo.bar'. - """ - self.name = name - - def __str__(self) -> str: - return self.name - - def __eq__(self, other: Any) -> bool: - if isinstance(other, self.__class__): - return hash(self) == hash(other) - else: - return False - - def __hash__(self) -> int: - return hash(str(self)) - - def is_descendant_of(self, module: "_Module") -> bool: - return self.name.startswith(f"{module.name}.") - - -@dataclass(frozen=True) -class _Link: - importer: str - imported: str - - -# A chain of modules, each of which imports the next. -_Chain = tuple[str, ...] - - -def _generate_module_permutations( - graph: ImportGraph, - layers: Sequence[str], - containers: set[str], -) -> Iterator[tuple[_Module, _Module, str | None]]: - """ - Return all possible combinations of higher level and lower level modules, in pairs. - - Each pair of modules consists of immediate children of two different layers. The first - module is in a layer higher than the layer of the second module. This means the first - module is allowed to import the second, but not the other way around. - - Returns: - module_in_higher_layer, module_in_lower_layer, container - """ - # If there are no containers, we still want to run the loop once. - quasi_containers = containers or [None] - - for container in quasi_containers: - for index, higher_layer in enumerate(layers): - higher_layer_module = _module_from_layer(higher_layer, container) - - if higher_layer_module.name not in graph.modules: - continue - - for lower_layer in layers[index + 1 :]: - lower_layer_module = _module_from_layer(lower_layer, container) - - if lower_layer_module.name not in graph.modules: - continue - - yield higher_layer_module, lower_layer_module, container - - -def _module_from_layer(layer: str, container: str | None = None) -> _Module: - if container: - name = ".".join([container, layer]) - else: - name = layer - return _Module(name) From 43235d9e640c6e2f2b3cb686120484079f8d5ae8 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Wed, 15 Jan 2025 08:52:57 +0000 Subject: [PATCH 5/5] Use faster hash library This is faster, but not cryptographically safe. That shouldn't matter for our purposes. --- rust/Cargo.lock | 7 ++ rust/Cargo.toml | 1 + rust/src/graph.rs | 228 ++++++++++++++++++++++++++------------------ rust/src/lib.rs | 38 ++++---- rust/tests/large.rs | 4 +- 5 files changed, 161 insertions(+), 117 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 37709188..9773fef5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -12,6 +12,7 @@ dependencies = [ "pyo3", "pyo3-log", "rayon", + "rustc-hash", "serde_json", ] @@ -277,6 +278,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + [[package]] name = "ryu" version = "1.0.18" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 09e0e5ec..00f48bc5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,6 +14,7 @@ serde_json = "1.0.103" rayon = "1.10" petgraph = "0.6.5" bimap = "0.6.3" +rustc-hash = "2.1.0" [dependencies.pyo3] version = "0.23.4" diff --git a/rust/src/graph.rs b/rust/src/graph.rs index ae9cf237..248e84d6 100644 --- a/rust/src/graph.rs +++ b/rust/src/graph.rs @@ -6,7 +6,7 @@ use petgraph::stable_graph::{NodeIndex, StableGraph}; use petgraph::visit::{Bfs, Walker}; use petgraph::Direction; use rayon::prelude::*; -use std::collections::{HashMap, HashSet}; +use rustc_hash::{FxHashMap, FxHashSet}; use std::fmt; use std::time::Instant; @@ -93,10 +93,10 @@ pub struct Graph { hierarchy: StableGraph, imports_module_indices: BiMap, imports: StableGraph, - squashed_modules: HashSet, + squashed_modules: FxHashSet, // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. - invisible_modules: HashSet, - detailed_imports_map: HashMap<(Module, Module), HashSet>, + invisible_modules: FxHashSet, + detailed_imports_map: FxHashMap<(Module, Module), FxHashSet>, } #[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] @@ -245,7 +245,7 @@ impl Graph { }; } - pub fn get_modules(&self) -> HashSet<&Module> { + pub fn get_modules(&self) -> FxHashSet<&Module> { self.hierarchy_module_indices .left_values() .filter(|module| !self.invisible_modules.contains(module)) @@ -260,23 +260,23 @@ impl Graph { &self, importer: &Module, imported: &Module, - ) -> HashSet { + ) -> FxHashSet { let key = (importer.clone(), imported.clone()); match self.detailed_imports_map.get(&key) { Some(import_details) => import_details.clone(), - None => HashSet::new(), + None => FxHashSet::default(), } } - pub fn find_children(&self, module: &Module) -> HashSet<&Module> { + pub fn find_children(&self, module: &Module) -> FxHashSet<&Module> { if self.invisible_modules.contains(module) { - return HashSet::new(); + return FxHashSet::default(); } let module_index = match self.hierarchy_module_indices.get_by_left(module) { Some(index) => index, // Module does not exist. // TODO: should this return a result, to handle if module is not in graph? - None => return HashSet::new(), + None => return FxHashSet::default(), }; self.hierarchy .neighbors(*module_index) @@ -285,7 +285,10 @@ impl Graph { .collect() } - pub fn find_descendants(&self, module: &Module) -> Result, ModuleNotPresent> { + pub fn find_descendants( + &self, + module: &Module, + ) -> Result, ModuleNotPresent> { let module_index = match self.hierarchy_module_indices.get_by_left(module) { Some(index) => index, None => { @@ -343,7 +346,7 @@ impl Graph { let key = (import.importer.clone(), import.imported.clone()); self.detailed_imports_map .entry(key) - .or_insert_with(HashSet::new) + .or_insert_with(FxHashSet::default) .insert(import.clone()); self.add_import(&import.importer, &import.imported); } @@ -411,17 +414,17 @@ impl Graph { .contains_edge(importer_index, imported_index) } - pub fn find_modules_that_directly_import(&self, imported: &Module) -> HashSet<&Module> { + pub fn find_modules_that_directly_import(&self, imported: &Module) -> FxHashSet<&Module> { let imported_index = match self.imports_module_indices.get_by_left(imported) { Some(imported_index) => *imported_index, - None => return HashSet::new(), + None => return FxHashSet::default(), }; - let importer_indices: HashSet = self + let importer_indices: FxHashSet = self .imports .neighbors_directed(imported_index, Direction::Incoming) .collect(); - let importers: HashSet<&Module> = importer_indices + let importers: FxHashSet<&Module> = importer_indices .iter() .map(|importer_index| { self.imports_module_indices @@ -432,17 +435,17 @@ impl Graph { importers } - pub fn find_modules_directly_imported_by(&self, importer: &Module) -> HashSet<&Module> { + pub fn find_modules_directly_imported_by(&self, importer: &Module) -> FxHashSet<&Module> { let importer_index = match self.imports_module_indices.get_by_left(importer) { Some(importer_index) => *importer_index, - None => return HashSet::new(), + None => return FxHashSet::default(), }; - let imported_indices: HashSet = self + let imported_indices: FxHashSet = self .imports .neighbors_directed(importer_index, Direction::Outgoing) .collect(); - let importeds: HashSet<&Module> = imported_indices + let importeds: FxHashSet<&Module> = imported_indices .iter() .map(|imported_index| { self.imports_module_indices @@ -453,12 +456,14 @@ impl Graph { importeds } - pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { - let mut upstream_modules = HashSet::new(); + pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> FxHashSet<&Module> { + let mut upstream_modules = FxHashSet::default(); - let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + let mut modules_to_check: FxHashSet<&Module> = FxHashSet::from_iter([module]); if as_package { - let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + let descendants = self + .find_descendants(&module) + .unwrap_or(FxHashSet::default()); modules_to_check.extend(descendants.into_iter()); }; @@ -479,12 +484,14 @@ impl Graph { upstream_modules } - pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { - let mut downstream_modules = HashSet::new(); + pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> FxHashSet<&Module> { + let mut downstream_modules = FxHashSet::default(); - let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + let mut modules_to_check: FxHashSet<&Module> = FxHashSet::from_iter([module]); if as_package { - let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + let descendants = self + .find_descendants(&module) + .unwrap_or(FxHashSet::default()); modules_to_check.extend(descendants.into_iter()); }; @@ -551,12 +558,12 @@ impl Graph { importer: &Module, imported: &Module, as_packages: bool, - ) -> Result>, String> { - let mut chains = HashSet::new(); + ) -> Result>, String> { + let mut chains = FxHashSet::default(); let mut temp_graph = self.clone(); - let mut downstream_modules: HashSet = HashSet::from([importer.clone()]); - let mut upstream_modules: HashSet = HashSet::from([imported.clone()]); + let mut downstream_modules: FxHashSet = FxHashSet::from_iter([importer.clone()]); + let mut upstream_modules: FxHashSet = FxHashSet::from_iter([imported.clone()]); // TODO don't do this if module is squashed? if as_packages { @@ -597,9 +604,10 @@ impl Graph { } // Keep track of imports into/out of upstream/downstream packages, and remove them. - let mut map_of_imports: HashMap> = HashMap::new(); + let mut map_of_imports: FxHashMap> = + FxHashMap::default(); for module in upstream_modules.union(&downstream_modules) { - let mut imports_to_or_from_module = HashSet::new(); + let mut imports_to_or_from_module = FxHashSet::default(); for imported_module in temp_graph.find_modules_directly_imported_by(&module) { imports_to_or_from_module.insert((module.clone(), imported_module.clone())); } @@ -661,7 +669,7 @@ impl Graph { pub fn find_illegal_dependencies_for_layers( &self, levels: Vec, - containers: HashSet, + containers: FxHashSet, ) -> Result, NoSuchContainer> { // Check that containers exist. let modules = self.get_modules(); @@ -714,7 +722,7 @@ impl Graph { fn _generate_module_permutations( &self, levels: &Vec, - containers: &HashSet, + containers: &FxHashSet, ) -> Vec<(Module, Module, Option)> { let mut permutations: Vec<(Module, Module, Option)> = vec![]; @@ -869,9 +877,9 @@ impl Graph { } // Set up importer/imported package contents. - let mut importer_modules: HashSet<&Module> = HashSet::from([importer_package]); + let mut importer_modules: FxHashSet<&Module> = FxHashSet::from_iter([importer_package]); importer_modules.extend(self.find_descendants(&importer_package).unwrap()); - let mut imported_modules: HashSet<&Module> = HashSet::from([imported_package]); + let mut imported_modules: FxHashSet<&Module> = FxHashSet::from_iter([imported_package]); imported_modules.extend(self.find_descendants(&imported_package).unwrap()); // Build routes from middles. @@ -934,10 +942,10 @@ impl Graph { &mut self, lower_layer_module: &Module, higher_layer_module: &Module, - ) -> HashSet<(Module, Module)> { - let mut imports = HashSet::new(); + ) -> FxHashSet<(Module, Module)> { + let mut imports = FxHashSet::default(); - let mut lower_layer_modules = HashSet::from([lower_layer_module.clone()]); + let mut lower_layer_modules = FxHashSet::from_iter([lower_layer_module.clone()]); for descendant in self .find_descendants(lower_layer_module) .unwrap() @@ -947,7 +955,7 @@ impl Graph { lower_layer_modules.insert(descendant.clone()); } - let mut higher_layer_modules = HashSet::from([higher_layer_module.clone()]); + let mut higher_layer_modules = FxHashSet::from_iter([higher_layer_module.clone()]); for descendant in self .find_descendants(higher_layer_module) .unwrap() @@ -1050,7 +1058,7 @@ mod tests { fn modules_when_empty() { let graph = Graph::default(); - assert_eq!(graph.get_modules(), HashSet::new()); + assert_eq!(graph.get_modules(), FxHashSet::default()); } #[test] @@ -1069,7 +1077,7 @@ mod tests { let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage])); + assert_eq!(result, FxHashSet::from_iter([&mypackage])); } #[test] @@ -1080,7 +1088,7 @@ mod tests { let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage])); + assert_eq!(result, FxHashSet::from_iter([&mypackage])); } #[test] @@ -1093,7 +1101,7 @@ mod tests { let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage, &mypackage_foo])); + assert_eq!(result, FxHashSet::from_iter([&mypackage, &mypackage_foo])); assert_eq!( graph.pretty_str(), " @@ -1117,7 +1125,7 @@ imports: graph.remove_module(&mypackage_foo); let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage])); + assert_eq!(result, FxHashSet::from_iter([&mypackage])); } #[test] @@ -1136,7 +1144,7 @@ imports: let result = graph.get_modules(); assert_eq!( result, - HashSet::from([ + FxHashSet::from_iter([ &mypackage, &mypackage_foo_alpha, // To be consistent with previous versions of Grimp. ]) @@ -1162,7 +1170,7 @@ imports: let result = graph.get_modules(); assert_eq!( result, - HashSet::from([&mypackage, &mypackage_foo_alpha, &importer, &imported]) + FxHashSet::from_iter([&mypackage, &mypackage_foo_alpha, &importer, &imported]) ); assert_eq!( graph.direct_import_exists(&importer, &mypackage_foo, false), @@ -1190,7 +1198,7 @@ imports: assert_eq!( graph.get_import_details(&importer, &imported), - HashSet::new() + FxHashSet::default() ); } @@ -1210,7 +1218,7 @@ imports: assert_eq!( graph.get_import_details(&importer, &imported), - HashSet::new() + FxHashSet::default() ); } @@ -1229,7 +1237,10 @@ imports: false ); // ...but the modules are still there. - assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + assert_eq!( + graph.get_modules(), + FxHashSet::from_iter([&importer, &imported]) + ); } #[test] @@ -1243,7 +1254,10 @@ imports: graph.remove_import(&importer, &imported); // The modules are still there. - assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + assert_eq!( + graph.get_modules(), + FxHashSet::from_iter([&importer, &imported]) + ); } #[test] @@ -1336,7 +1350,7 @@ imports: graph.add_module(mypackage.clone()); graph.add_module(mypackage_foo.clone()); - assert_eq!(graph.find_children(&mypackage_foo), HashSet::new()); + assert_eq!(graph.find_children(&mypackage_foo), FxHashSet::default()); } #[test] @@ -1352,7 +1366,7 @@ imports: assert_eq!( graph.find_children(&mypackage), - HashSet::from([&mypackage_foo, &mypackage_bar]) + FxHashSet::from_iter([&mypackage_foo, &mypackage_bar]) ); } @@ -1369,7 +1383,7 @@ imports: assert_eq!( graph.find_children(&mypackage), - HashSet::from([&mypackage_foo, &mypackage_bar]) + FxHashSet::from_iter([&mypackage_foo, &mypackage_bar]) ); } @@ -1385,7 +1399,7 @@ imports: assert_eq!( graph.find_children(&Module::new("mypackage".to_string())), - HashSet::new() + FxHashSet::default() ); } @@ -1408,7 +1422,10 @@ imports: graph.add_module(mypackage_foo_alpha_green.clone()); graph.add_module(mypackage_foo_beta.clone()); - assert_eq!(graph.find_descendants(&mypackage_bar), Ok(HashSet::new())); + assert_eq!( + graph.find_descendants(&mypackage_bar), + Ok(FxHashSet::default()) + ); } #[test] @@ -1447,7 +1464,7 @@ imports: assert_eq!( graph.find_descendants(&mypackage_foo), - Ok(HashSet::from([ + Ok(FxHashSet::from_iter([ &mypackage_foo_alpha, &mypackage_foo_alpha_blue, &mypackage_foo_alpha_green, @@ -1479,7 +1496,7 @@ imports: assert_eq!( graph.find_descendants(&mypackage_foo), // mypackage.foo.blue is not included. - Ok(HashSet::from([ + Ok(FxHashSet::from_iter([ &mypackage_foo_blue_alpha, &mypackage_foo_blue_alpha_one, &mypackage_foo_blue_alpha_two, @@ -1512,7 +1529,7 @@ imports: assert_eq!( graph.find_descendants(&mypackage_foo), - Ok(HashSet::from([ + Ok(FxHashSet::from_iter([ &mypackage_foo_blue, // Should be included. &mypackage_foo_blue_alpha, &mypackage_foo_blue_alpha_one, @@ -1608,7 +1625,7 @@ imports: assert_eq!( graph.get_modules(), - HashSet::from([&mypackage_bar, &mypackage_foo]) + FxHashSet::from_iter([&mypackage_bar, &mypackage_foo]) ); assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); assert_eq!( @@ -1635,7 +1652,7 @@ imports: assert_eq!( graph.get_modules(), - HashSet::from([&mypackage_bar, &mypackage_foo]) + FxHashSet::from_iter([&mypackage_bar, &mypackage_foo]) ); assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); assert_eq!( @@ -1777,7 +1794,7 @@ imports: assert_eq!( result, - HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + FxHashSet::from_iter([&mypackage_foo_alpha, &anotherpackage]) ) } @@ -1793,7 +1810,7 @@ imports: graph.remove_import(&green, &blue); let result = graph.find_modules_that_directly_import(&blue); - assert_eq!(result, HashSet::from([&yellow])) + assert_eq!(result, FxHashSet::from_iter([&yellow])) } #[test] @@ -1822,7 +1839,7 @@ imports: assert_eq!( result, - HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + FxHashSet::from_iter([&mypackage_foo_alpha, &anotherpackage]) ) } @@ -1838,7 +1855,7 @@ imports: graph.remove_import(&blue, &green); let result = graph.find_modules_directly_imported_by(&blue); - assert_eq!(result, HashSet::from([&yellow])) + assert_eq!(result, FxHashSet::from_iter([&yellow])) } #[test] @@ -2059,7 +2076,10 @@ imports: let result = graph.find_upstream_modules(&blue, false); - assert_eq!(result, HashSet::from([&green, &red, &yellow, &purple])) + assert_eq!( + result, + FxHashSet::from_iter([&green, &red, &yellow, &purple]) + ) } #[test] @@ -2069,7 +2089,7 @@ imports: let result = graph.find_upstream_modules(&blue, false); - assert_eq!(result, HashSet::new()) + assert_eq!(result, FxHashSet::default()) } #[test] @@ -2106,7 +2126,10 @@ imports: let result = graph.find_upstream_modules(&blue, true); - assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + assert_eq!( + result, + FxHashSet::from_iter([&green, &yellow, &purple, &brown]) + ) } #[test] @@ -2138,7 +2161,7 @@ imports: let result = graph.find_downstream_modules(&purple, false); - assert_eq!(result, HashSet::from([&yellow, &green, &blue])) + assert_eq!(result, FxHashSet::from_iter([&yellow, &green, &blue])) } #[test] @@ -2148,7 +2171,7 @@ imports: let result = graph.find_downstream_modules(&blue, false); - assert_eq!(result, HashSet::new()) + assert_eq!(result, FxHashSet::default()) } #[test] @@ -2185,7 +2208,10 @@ imports: let result = graph.find_downstream_modules(&blue, true); - assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + assert_eq!( + result, + FxHashSet::from_iter([&green, &yellow, &purple, &brown]) + ) } // find_shortest_chain @@ -2336,7 +2362,7 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::new())); + assert_eq!(result, Ok(FxHashSet::default())); } #[test] @@ -2361,7 +2387,7 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::from([vec![blue, red, green],]))); + assert_eq!(result, Ok(FxHashSet::from_iter([vec![blue, red, green],]))); } #[test] @@ -2388,7 +2414,10 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::from([vec![blue, red, green_alpha]]))); + assert_eq!( + result, + Ok(FxHashSet::from_iter([vec![blue, red, green_alpha]])) + ); } #[test] @@ -2419,7 +2448,7 @@ imports: assert_eq!( result, - Ok(HashSet::from([vec![blue, red, green_alpha_one],])) + Ok(FxHashSet::from_iter([vec![blue, red, green_alpha_one],])) ) } @@ -2447,7 +2476,10 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::from([vec![blue_alpha, red, green],]))); + assert_eq!( + result, + Ok(FxHashSet::from_iter([vec![blue_alpha, red, green],])) + ); } #[test] @@ -2478,7 +2510,7 @@ imports: assert_eq!( result, - Ok(HashSet::from([vec![blue_alpha_one, red, green],])) + Ok(FxHashSet::from_iter([vec![blue_alpha_one, red, green],])) ) } @@ -2593,7 +2625,7 @@ imports: fn find_illegal_dependencies_for_layers_empty_everything() { let graph = Graph::default(); - let dependencies = graph.find_illegal_dependencies_for_layers(vec![], HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(vec![], FxHashSet::default()); assert_eq!(dependencies, Ok(vec![])); } @@ -2603,8 +2635,10 @@ imports: let graph = Graph::default(); let container = "nonexistent_container".to_string(); - let dependencies = - graph.find_illegal_dependencies_for_layers(vec![], HashSet::from([container.clone()])); + let dependencies = graph.find_illegal_dependencies_for_layers( + vec![], + FxHashSet::from_iter([container.clone()]), + ); assert_eq!( dependencies, @@ -2622,7 +2656,8 @@ imports: independent: true, }; - let dependencies = graph.find_illegal_dependencies_for_layers(vec![level], HashSet::new()); + let dependencies = + graph.find_illegal_dependencies_for_layers(vec![level], FxHashSet::default()); assert_eq!(dependencies, Ok(vec![])); } @@ -2637,8 +2672,8 @@ imports: }; let container = "mypackage".to_string(); - let dependencies = - graph.find_illegal_dependencies_for_layers(vec![level], HashSet::from([container])); + let dependencies = graph + .find_illegal_dependencies_for_layers(vec![level], FxHashSet::from_iter([container])); assert_eq!(dependencies, Ok(vec![])); } @@ -2660,7 +2695,7 @@ imports: }, ]; - let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); assert_eq!( dependencies, @@ -2695,7 +2730,7 @@ imports: }, ]; - let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); assert_eq!( dependencies, @@ -2741,7 +2776,7 @@ imports: independent: true, }, ]; - let containers = HashSet::from(["blue".to_string(), "green".to_string()]); + let containers = FxHashSet::from_iter(["blue".to_string(), "green".to_string()]); let dependencies = graph.find_illegal_dependencies_for_layers(levels, containers); @@ -2786,7 +2821,7 @@ imports: independent: true, }]; - let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); assert_eq!( dependencies, @@ -2810,7 +2845,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::new()); + assert_eq!(result, FxHashSet::default()); } #[test] @@ -2822,7 +2857,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::new()); + assert_eq!(result, FxHashSet::default()); } #[test] @@ -2847,7 +2882,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::from([import])); + assert_eq!(result, FxHashSet::from_iter([import])); } #[test] @@ -2872,7 +2907,10 @@ imports: let result = graph.get_import_details(&blue, &green); - assert_eq!(result, HashSet::from([blue_to_green_a, blue_to_green_b])); + assert_eq!( + result, + FxHashSet::from_iter([blue_to_green_a, blue_to_green_b]) + ); } #[test] @@ -2898,7 +2936,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::new()); + assert_eq!(result, FxHashSet::default()); } #[test] @@ -2924,6 +2962,6 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::from([import])); + assert_eq!(result, FxHashSet::from_iter([import])); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 112dbb0e..10d52631 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,7 +6,7 @@ use pyo3::create_exception; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; -use std::collections::HashSet; +use rustc_hash::FxHashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -34,7 +34,7 @@ impl GraphWrapper { } } - pub fn get_modules(&self) -> HashSet { + pub fn get_modules(&self) -> FxHashSet { self._graph .get_modules() .iter() @@ -125,7 +125,7 @@ impl GraphWrapper { self._graph.count_imports() } - pub fn find_children(&self, module: &str) -> HashSet { + pub fn find_children(&self, module: &str) -> FxHashSet { self._graph .find_children(&Module::new(module.to_string())) .iter() @@ -133,7 +133,7 @@ impl GraphWrapper { .collect() } - pub fn find_descendants(&self, module: &str) -> HashSet { + pub fn find_descendants(&self, module: &str) -> FxHashSet { self._graph .find_descendants(&Module::new(module.to_string())) .unwrap() @@ -169,7 +169,7 @@ impl GraphWrapper { )) } - pub fn find_modules_directly_imported_by(&self, module: &str) -> HashSet { + pub fn find_modules_directly_imported_by(&self, module: &str) -> FxHashSet { self._graph .find_modules_directly_imported_by(&Module::new(module.to_string())) .iter() @@ -177,7 +177,7 @@ impl GraphWrapper { .collect() } - pub fn find_modules_that_directly_import(&self, module: &str) -> HashSet { + pub fn find_modules_that_directly_import(&self, module: &str) -> FxHashSet { self._graph .find_modules_that_directly_import(&Module::new(module.to_string())) .iter() @@ -226,7 +226,7 @@ impl GraphWrapper { #[allow(unused_variables)] #[pyo3(signature = (module, as_package=false))] - pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> HashSet { + pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> FxHashSet { // Turn the Modules to Strings. self._graph .find_downstream_modules(&Module::new(module.to_string()), as_package) @@ -237,7 +237,7 @@ impl GraphWrapper { #[allow(unused_variables)] #[pyo3(signature = (module, as_package=false))] - pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> HashSet { + pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> FxHashSet { self._graph .find_upstream_modules(&Module::new(module.to_string()), as_package) .iter() @@ -262,7 +262,7 @@ impl GraphWrapper { imported: &str, as_packages: bool, ) -> PyResult> { - let rust_chains: HashSet> = self + let rust_chains: FxHashSet> = self ._graph .find_shortest_chains( &Module::new(importer.to_string()), @@ -308,18 +308,16 @@ impl GraphWrapper { )) } - #[allow(unused_variables)] #[pyo3(signature = (layers, containers))] pub fn find_illegal_dependencies_for_layers<'py>( &self, py: Python<'py>, layers: &Bound<'py, PyTuple>, - containers: HashSet, + containers: FxHashSet, ) -> PyResult> { info!("Using Rust to find illegal dependencies."); let levels = rustify_levels(layers); - println!("\nIncoming {:?}, {:?}", levels, containers); let dependencies = py.allow_threads(|| { self._graph .find_illegal_dependencies_for_layers(levels, containers) @@ -343,7 +341,7 @@ fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { let mut rust_levels: Vec = vec![]; for level_python in levels_python.into_iter() { let level_dict = level_python.downcast::().unwrap(); - let layers: HashSet = level_dict + let layers: FxHashSet = level_dict .get_item("layers") .unwrap() .unwrap() @@ -431,15 +429,15 @@ mod tests { let elements = vec![ pydict! (py, { "independent" => true, - "layers" => HashSet::from(["high"]), + "layers" => FxHashSet::from_iter(["high"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["medium"]), + "layers" => FxHashSet::from_iter(["medium"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["low"]), + "layers" => FxHashSet::from_iter(["low"]), }), ]; let python_levels = PyTuple::new(py, elements)?; @@ -476,19 +474,19 @@ mod tests { let elements = vec![ pydict! (py, { "independent" => true, - "layers" => HashSet::from(["high"]), + "layers" => FxHashSet::from_iter(["high"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["blue", "green", "orange"]), + "layers" => FxHashSet::from_iter(["blue", "green", "orange"]), }), pydict! (py, { "independent" => false, - "layers" => HashSet::from(["red", "yellow"]), + "layers" => FxHashSet::from_iter(["red", "yellow"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["low"]), + "layers" => FxHashSet::from_iter(["low"]), }), ]; let python_levels = PyTuple::new(py, elements)?; diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 1cdaf9ab..5eaca91b 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,6 +1,6 @@ use _rustgrimp::graph::{Graph, Level, Module}; +use rustc_hash::FxHashSet; use serde_json::{Map, Value}; -use std::collections::HashSet; use std::fs; #[test] @@ -40,7 +40,7 @@ fn test_large_graph_deep_layers() { layers: vec![layer.to_string()], }) .collect(); - let containers = HashSet::new(); + let containers = FxHashSet::default(); let deps = graph .find_illegal_dependencies_for_layers(levels, containers)