diff --git a/rust/src/dependencies.rs b/rust/src/dependencies.rs deleted file mode 100644 index 26a84e7c..00000000 --- a/rust/src/dependencies.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct OldRoute { - pub heads: Vec, - pub middle: Vec, - pub tails: Vec, -} - -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct OldPackageDependency { - pub importer: u32, - pub imported: u32, - pub routes: Vec, -} diff --git a/rust/src/graph.rs b/rust/src/graph.rs index 44004144..679e7c5f 100644 --- a/rust/src/graph.rs +++ b/rust/src/graph.rs @@ -1,25 +1,4 @@ /* -modules (get_modules) DONE -find_children - DONE -find_descendants - DONE -direct_import_exists - DONE -find_modules_directly_imported_by - DONE -find_modules_that_directly_import - DONE -get_import_details - DONE -count_imports - DONE -find_upstream_modules - DONE -find_downstream_modules - DONE -find_shortest_chain - DONE - find_shortest_chains - TODO -chain_exists - DONE -find_illegal_dependencies_for_layers - DONE -add_module - DONE -remove_module - DONE -add_import - DONE -remove_import - DONE -squash_module - DONE -is_module_squashed - DONE - Also, sensible behaviour when passing modules that don't exist in the graph. */ #![allow(dead_code)] @@ -35,7 +14,12 @@ use std::collections::{HashMap, HashSet}; use std::fmt; use std::time::Instant; -use crate::layers::Level; +/// A group of layers at the same level in the layering. +#[derive(PartialEq, Eq, Hash, Debug)] +pub struct Level { + pub layers: Vec, + pub independent: bool, +} // Delimiter for Python modules. const DELIMITER: char = '.'; @@ -1059,7 +1043,6 @@ impl Graph { #[cfg(test)] mod tests { use super::*; - use crate::layers::Level; #[test] fn modules_when_empty() { diff --git a/rust/src/importgraph.rs b/rust/src/importgraph.rs deleted file mode 100644 index cf58aaf9..00000000 --- a/rust/src/importgraph.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::collections::hash_map::Entry::Vacant; -use std::collections::{HashMap, HashSet}; -use std::fmt; - -#[derive(Clone)] -pub struct ImportGraph<'a> { - pub names_by_id: HashMap, - pub ids_by_name: HashMap<&'a str, u32>, - pub importers_by_imported: HashMap>, - pub importeds_by_importer: HashMap>, -} - -impl<'a> ImportGraph<'a> { - pub fn new(importeds_by_importer: HashMap<&'a str, HashSet<&'a str>>) -> ImportGraph<'a> { - // Build the name/id lookup maps. - let mut names_by_id: HashMap = HashMap::new(); - let mut ids_by_name: HashMap<&'a str, u32> = HashMap::new(); - let mut current_id: u32 = 1; - for name in importeds_by_importer.keys() { - names_by_id.insert(current_id, name); - ids_by_name.insert(name, current_id); - current_id += 1; - } - - // Convert importeds_by_importer to id-based. - let mut importeds_by_importer_u32: HashMap> = HashMap::new(); - for (importer_str, importeds_strs) in importeds_by_importer.iter() { - let mut importeds_u32 = HashSet::new(); - for imported_str in importeds_strs { - importeds_u32.insert(*ids_by_name.get(imported_str).unwrap()); - } - - importeds_by_importer_u32 - .insert(*ids_by_name.get(importer_str).unwrap(), importeds_u32); - } - - let importers_by_imported_u32 = - ImportGraph::_build_importers_by_imported_u32(&importeds_by_importer_u32); - - ImportGraph { - names_by_id, - ids_by_name, - importers_by_imported: importers_by_imported_u32, - importeds_by_importer: importeds_by_importer_u32, - } - } - - fn _build_importers_by_imported_u32( - importeds_by_importer_u32: &HashMap>, - ) -> HashMap> { - // Build importers_by_imported from importeds_by_importer. - let mut importers_by_imported_u32: HashMap> = HashMap::new(); - for (importer, importeds) in importeds_by_importer_u32.iter() { - for imported in importeds { - let entry = importers_by_imported_u32.entry(*imported).or_default(); - entry.insert(*importer); - } - } - - // Check that there is an empty set for any remaining. - for importer in importeds_by_importer_u32.keys() { - importers_by_imported_u32.entry(*importer).or_default(); - } - importers_by_imported_u32 - } - - pub fn get_module_ids(&self) -> HashSet { - self.names_by_id.keys().copied().collect() - } - - pub fn contains_module(&self, module_name: &str) -> bool { - self.ids_by_name.contains_key(module_name) - } - - pub fn remove_import(&mut self, importer: &str, imported: &str) { - self.remove_import_ids(self.ids_by_name[importer], self.ids_by_name[imported]); - } - - pub fn add_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.insert(imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.insert(importer); - } - - pub fn remove_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.remove(&imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.remove(&importer); - } - - pub fn remove_module_by_id(&mut self, module_id: u32) { - let _module = self.names_by_id[&module_id]; - - let mut imports_to_remove = Vec::with_capacity(self.names_by_id.len()); - { - for imported_id in &self.importeds_by_importer[&module_id] { - imports_to_remove.push((module_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&module_id] { - imports_to_remove.push((*importer_id, module_id)); - } - } - - for (importer, imported) in imports_to_remove { - self.remove_import_ids(importer, imported); - } - - self.importeds_by_importer.remove(&module_id); - self.importers_by_imported.remove(&module_id); - } - - pub fn get_descendant_ids(&self, module_name: &str) -> Vec { - let mut descendant_ids = vec![]; - let namespace: String = format!("{}.", module_name); - for (candidate_name, candidate_id) in &self.ids_by_name { - if candidate_name.starts_with(&namespace) { - descendant_ids.push(*candidate_id); - } - } - descendant_ids - } - - pub fn remove_package(&mut self, module_name: &str) { - for descendant_id in self.get_descendant_ids(module_name) { - self.remove_module_by_id(descendant_id); - } - self.remove_module_by_id(self.ids_by_name[&module_name]); - } - - pub fn squash_module(&mut self, module_name: &str) { - let squashed_root_id = self.ids_by_name[module_name]; - let descendant_ids = &self.get_descendant_ids(module_name); - - // Assemble imports to add first, then add them in a second loop, - // to avoid needing to clone importeds_by_importer. - let mut imports_to_add = Vec::with_capacity(self.names_by_id.len()); - // Imports from the root. - { - for descendant_id in descendant_ids { - for imported_id in &self.importeds_by_importer[&descendant_id] { - imports_to_add.push((squashed_root_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&descendant_id] { - imports_to_add.push((*importer_id, squashed_root_id)); - } - } - } - - for (importer, imported) in imports_to_add { - self.add_import_ids(importer, imported); - } - - // Now we've added imports to/from the root, we can delete the root's descendants. - for descendant_id in descendant_ids { - self.remove_module_by_id(*descendant_id); - } - } - - pub fn pop_shortest_chains(&mut self, importer: &str, imported: &str) -> Vec> { - let mut chains = vec![]; - let importer_id = self.ids_by_name[&importer]; - let imported_id = self.ids_by_name[&imported]; - - while let Some(chain) = self.find_shortest_chain(importer_id, imported_id) { - // Remove chain - let _mods: Vec<&str> = chain.iter().map(|i| self.names_by_id[&i]).collect(); - for i in 0..chain.len() - 1 { - self.remove_import_ids(chain[i], chain[i + 1]); - } - chains.push(chain); - } - - chains - } - - pub fn find_shortest_chain(&self, importer_id: u32, imported_id: u32) -> Option> { - let results_or_none = self._search_for_path(importer_id, imported_id); - match results_or_none { - Some(results) => { - let (pred, succ, initial_w) = results; - - let mut w_or_none: Option = Some(initial_w); - // Transform results into vector. - let mut path: Vec = Vec::new(); - // From importer to w: - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = pred[&w]; - } - path.reverse(); - - // From w to imported: - w_or_none = succ[path.last().unwrap()]; - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = succ[&w]; - } - - Some(path) - } - None => None, - } - } - /// Performs a breadth first search from both source and target, meeting in the middle. - // - // Returns: - // (pred, succ, w) where - // - pred is a dictionary of predecessors from w to the source, and - // - succ is a dictionary of successors from w to the target. - // - fn _search_for_path( - &self, - importer: u32, - imported: u32, - ) -> Option<(HashMap>, HashMap>, u32)> { - if importer == imported { - Some(( - HashMap::from([(imported, None)]), - HashMap::from([(importer, None)]), - importer, - )) - } else { - let mut pred: HashMap> = HashMap::from([(importer, None)]); - let mut succ: HashMap> = HashMap::from([(imported, None)]); - - // Initialize fringes, start with forward. - let mut forward_fringe: Vec = Vec::from([importer]); - let mut reverse_fringe: Vec = Vec::from([imported]); - let mut this_level: Vec; - - while !forward_fringe.is_empty() && !reverse_fringe.is_empty() { - if forward_fringe.len() <= reverse_fringe.len() { - this_level = forward_fringe.to_vec(); - forward_fringe = Vec::new(); - for v in this_level { - for w in self.importeds_by_importer[&v].clone() { - pred.entry(w).or_insert_with(|| { - forward_fringe.push(w); - Some(v) - }); - if succ.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } else { - this_level = reverse_fringe.to_vec(); - reverse_fringe = Vec::new(); - for v in this_level { - for w in self.importers_by_imported[&v].clone() { - if let Vacant(e) = succ.entry(w) { - e.insert(Some(v)); - reverse_fringe.push(w); - } - if pred.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } - } - None - } - } -} - -impl fmt::Display for ImportGraph<'_> { - fn fmt(&self, dest: &mut fmt::Formatter) -> fmt::Result { - let mut strings = vec![]; - for (importer, importeds) in self.importeds_by_importer.iter() { - let mut string = format!("IMPORTER {}: ", self.names_by_id[&importer]); - for imported in importeds { - string.push_str(format!("{}, ", self.names_by_id[&imported]).as_str()); - } - strings.push(string); - } - strings.push(" ".to_string()); - for (imported, importers) in self.importers_by_imported.iter() { - let mut string = format!("IMPORTED {}: ", self.names_by_id[&imported]); - for importer in importers { - string.push_str(format!("{}, ", self.names_by_id[&importer]).as_str()); - } - strings.push(string); - } - write!(dest, "{}", strings.join("\n")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn _make_graph() -> ImportGraph<'static> { - ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["blue.alpha", "blue.beta", "green"])), - ("blue.alpha", HashSet::new()), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ])) - } - - #[test] - fn get_module_ids() { - let graph = _make_graph(); - - assert_eq!( - graph.get_module_ids(), - HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]) - ); - } - - #[test] - fn new_stores_importeds_by_importer_using_id() { - let graph = _make_graph(); - - let expected_importeds: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importeds_by_importer - .get(graph.ids_by_name.get("blue").unwrap()) - .unwrap(), - expected_importeds - ); - } - - #[test] - fn new_stores_importers_by_imported_using_id() { - let graph = _make_graph(); - - let expected_importers: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importers_by_imported - .get(graph.ids_by_name.get("blue.alpha").unwrap()) - .unwrap(), - expected_importers - ); - } - - #[test] - fn test_squash_module() { - let mut graph = ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["orange", "green"])), - ("blue.alpha", HashSet::from(["green.delta"])), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ("green.gamma", HashSet::new()), - ("green.delta", HashSet::new()), - ("orange", HashSet::new()), - ])); - - graph.squash_module("blue"); - - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["blue"]], - HashSet::from([ - graph.ids_by_name["orange"], - graph.ids_by_name["green"], - graph.ids_by_name["green.delta"], - ]) - ); - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["orange"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green.delta"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["blue"]], - HashSet::from([graph.ids_by_name["green"],]) - ); - } - - #[test] - fn test_find_shortest_chain() { - let blue = "blue"; - let green = "green"; - let yellow = "yellow"; - let blue_alpha = "blue.alpha"; - let blue_beta = "blue.beta"; - - let graph = ImportGraph::new(HashMap::from([ - (green, HashSet::from([blue])), - (blue_alpha, HashSet::from([blue])), - (yellow, HashSet::from([green])), - (blue_beta, HashSet::from([green])), - (blue, HashSet::new()), - ])); - - let path_or_none: Option> = - graph.find_shortest_chain(graph.ids_by_name[&yellow], graph.ids_by_name[&blue]); - - assert_eq!( - path_or_none, - Some(Vec::from([ - graph.ids_by_name[&yellow], - graph.ids_by_name[&green], - graph.ids_by_name[&blue] - ])) - ); - } -} diff --git a/rust/src/layers.rs b/rust/src/layers.rs deleted file mode 100644 index ba2019e7..00000000 --- a/rust/src/layers.rs +++ /dev/null @@ -1,699 +0,0 @@ -use crate::dependencies::{OldPackageDependency, OldRoute}; -use crate::importgraph::ImportGraph; - -use log::info; -use rayon::prelude::*; -use std::collections::HashSet; -use std::time::Instant; - -/// A group of layers at the same level in the layering. -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Level { - pub layers: Vec, - pub independent: bool, -} - -pub fn find_illegal_dependencies<'a>( - graph: &'a ImportGraph, - levels: &'a Vec, - containers: &'a HashSet, -) -> Vec { - let layers = _layers_from_levels(levels); - - _generate_module_permutations(graph, levels, containers) - .into_par_iter() - .filter_map(|(higher_layer_package, lower_layer_package, container)| { - // TODO: it's inefficient to do this for sibling layers, as we don't need - // to clone and trim the graph for identical pairs. - info!( - "Searching for import chains from {} to {}...", - lower_layer_package, higher_layer_package - ); - let now = Instant::now(); - let dependency_or_none = _search_for_package_dependency( - &higher_layer_package, - &lower_layer_package, - &layers, - &container, - graph, - ); - _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); - dependency_or_none - }) - .collect() -} - -/// Return every permutation of modules that exist in the graph -/// in which the second should not import the first. -fn _generate_module_permutations<'a>( - graph: &'a ImportGraph, - levels: &'a [Level], - containers: &'a HashSet, -) -> Vec<(String, String, Option)> { - let mut permutations: Vec<(String, String, Option)> = vec![]; - - let quasi_containers: Vec> = if containers.is_empty() { - vec![None] - } else { - containers.iter().map(|i| Some(i.to_string())).collect() - }; - for container in quasi_containers { - for (index, higher_level) in levels.iter().enumerate() { - for higher_layer in &higher_level.layers { - let higher_layer_module_name = _module_from_layer(higher_layer, &container); - if graph - .ids_by_name - .get(&higher_layer_module_name as &str) - .is_none() - { - continue; - } - - // Build the layers that mustn't import this higher layer. - // That includes: - // * lower layers. - // * sibling layers, if the layer is independent. - let mut layers_forbidden_to_import_higher_layer: Vec<&str> = vec![]; - - if higher_level.independent { - for potential_sibling_layer in &higher_level.layers { - if potential_sibling_layer != higher_layer { - // It's a sibling layer. - layers_forbidden_to_import_higher_layer.push(potential_sibling_layer); - } - } - } - - for lower_level in &levels[index + 1..] { - for lower_layer in &lower_level.layers { - layers_forbidden_to_import_higher_layer.push(lower_layer); - } - } - - // Now turn the layers into modules, if they exist. - for forbidden_layer in &layers_forbidden_to_import_higher_layer { - let forbidden_module_name = _module_from_layer(forbidden_layer, &container); - if let Some(_value) = graph.ids_by_name.get(&forbidden_module_name as &str) { - permutations.push(( - higher_layer_module_name.clone(), - forbidden_module_name.clone(), - container.clone(), - )); - }; - } - } - } - } - - permutations -} - -fn _module_from_layer<'a>(module: &'a str, container: &'a Option) -> String { - match container { - Some(true_container) => format!("{}.{}", true_container, module), - None => module.to_string(), - } -} - -fn _search_for_package_dependency<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - layers: &'a Vec<&'a str>, - container: &'a Option, - graph: &'a ImportGraph, -) -> Option { - let mut temp_graph = graph.clone(); - _remove_other_layers( - &mut temp_graph, - layers, - container, - (higher_layer_package, lower_layer_package), - ); - let mut routes: Vec = vec![]; - - // Direct routes. - let direct_links = - _pop_direct_imports(higher_layer_package, lower_layer_package, &mut temp_graph); - for (importer, imported) in direct_links { - routes.push(OldRoute { - heads: vec![importer], - middle: vec![], - tails: vec![imported], - }); - } - - // Indirect routes. - for indirect_route in - _get_indirect_routes(higher_layer_package, lower_layer_package, &temp_graph) - { - routes.push(indirect_route); - } - if routes.is_empty() { - None - } else { - Some(OldPackageDependency { - imported: graph.ids_by_name[&higher_layer_package], - importer: graph.ids_by_name[&lower_layer_package], - routes, - }) - } -} - -fn _layers_from_levels<'a>(levels: &'a Vec) -> Vec<&'a str> { - let mut layers: Vec<&str> = vec![]; - for level in levels { - layers.extend(level.layers.iter().map(|s| s.as_str())); - } - layers -} - -fn _remove_other_layers<'a>( - graph: &'a mut ImportGraph, - layers: &'a Vec<&'a str>, - container: &'a Option, - layers_to_preserve: (&'a str, &'a str), -) { - for layer in layers { - let layer_module = _module_from_layer(layer, container); - if layers_to_preserve.0 == layer_module || layers_to_preserve.1 == layer_module { - continue; - } - if graph.contains_module(&layer_module) { - graph.remove_package(&layer_module); - } - } -} - -fn _pop_direct_imports<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - graph: &'a mut ImportGraph, -) -> HashSet<(u32, u32)> { - // Remove the direct imports, returning them as (importer, imported) tuples. - let mut imports = HashSet::new(); - - let higher_layer_namespace: String = format!("{}.", higher_layer_package); - let mut lower_layer_module_ids: Vec = vec![graph.ids_by_name[lower_layer_package]]; - lower_layer_module_ids.append(&mut graph.get_descendant_ids(lower_layer_package)); - - for lower_layer_module_id in lower_layer_module_ids { - let _lower = graph.names_by_id[&lower_layer_module_id]; - let imported_module_ids = graph.importeds_by_importer[&lower_layer_module_id].clone(); - for imported_module_id in imported_module_ids { - let imported_module = graph.names_by_id[&imported_module_id]; - - if imported_module.starts_with(&higher_layer_namespace) - || imported_module == higher_layer_package - { - imports.insert((lower_layer_module_id, imported_module_id)); - graph.remove_import_ids(lower_layer_module_id, imported_module_id) - } - } - } - imports -} - -fn _get_indirect_routes<'a>( - imported_package: &'a str, - importer_package: &'a str, - graph: &'a ImportGraph, -) -> Vec { - // Squashes the two packages. - // Gets a list of paths between them, called middles. - // Add the heads and tails to the middles. - let mut temp_graph = graph.clone(); - temp_graph.squash_module(imported_package); - temp_graph.squash_module(importer_package); - - let middles = _find_middles(&mut temp_graph, importer_package, imported_package); - _middles_to_routes(graph, middles, importer_package, imported_package) -} - -fn _find_middles<'a>( - graph: &'a mut ImportGraph, - importer: &'a str, - imported: &'a str, -) -> Vec> { - let mut middles = vec![]; - - for chain in graph.pop_shortest_chains(importer, imported) { - // Remove first and last element. - // TODO surely there's a better way? - let mut middle: Vec = vec![]; - let chain_length = chain.len(); - for (index, module) in chain.iter().enumerate() { - if index != 0 && index != chain_length - 1 { - middle.push(*module); - } - } - middles.push(middle); - } - - middles -} - -fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { - let route_count = match dependency_or_none { - Some(dependency) => dependency.routes.len(), - None => 0, - }; - let pluralized = if route_count == 1 { "" } else { "s" }; - info!( - "Found {} illegal route{} in {}s.", - route_count, pluralized, duration_in_s - ); -} - -fn _middles_to_routes<'a>( - graph: &'a ImportGraph, - middles: Vec>, - importer: &'a str, - imported: &'a str, -) -> Vec { - let mut routes = vec![]; - let importer_id = graph.ids_by_name[&importer]; - let imported_id = graph.ids_by_name[&imported]; - - for middle in middles { - // Construct heads. - let mut heads: Vec = vec![]; - let first_imported_id = middle[0]; - let candidate_modules = &graph.importers_by_imported[&first_imported_id]; - for candidate_module in candidate_modules { - if importer_id == *candidate_module - || graph - .get_descendant_ids(importer) - .contains(candidate_module) - { - heads.push(*candidate_module); - } - } - - // Construct tails. - let mut tails: Vec = vec![]; - let last_importer_id = middle[middle.len() - 1]; - let candidate_modules = &graph.importeds_by_importer[&last_importer_id]; - for candidate_module in candidate_modules { - if imported_id == *candidate_module - || graph - .get_descendant_ids(imported) - .contains(candidate_module) - { - tails.push(*candidate_module); - } - } - routes.push(OldRoute { - heads, - middle, - tails, - }) - } - - routes -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn test_find_illegal_dependencies_no_container() { - let graph = ImportGraph::new(HashMap::from([ - ("low", HashSet::new()), - ("low.blue", HashSet::from(["utils"])), - ("low.green", HashSet::new()), - ("low.green.alpha", HashSet::from(["high.yellow"])), - ("mid_a", HashSet::from(["mid_b"])), - ("mid_a.orange", HashSet::new()), - ("mid_b", HashSet::from(["mid_c"])), - ("mid_b.brown", HashSet::new()), - ("mid_c", HashSet::new()), - ("mid_c.purple", HashSet::new()), - ("high", HashSet::from(["low.blue"])), - ("high.yellow", HashSet::new()), - ("high.red", HashSet::new()), - ("high.red.beta", HashSet::new()), - ("utils", HashSet::from(["high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::new(); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![ - OldPackageDependency { - importer: *graph.ids_by_name.get("low").unwrap(), - imported: *graph.ids_by_name.get("high").unwrap(), - routes: vec![ - OldRoute { - heads: vec![*graph.ids_by_name.get("low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("high.yellow").unwrap()], - }, - OldRoute { - heads: vec![*graph.ids_by_name.get("low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("utils").unwrap()], - tails: vec![*graph.ids_by_name.get("high.red").unwrap()], - }, - ], - }, - OldPackageDependency { - importer: *graph.ids_by_name.get("mid_a").unwrap(), - imported: *graph.ids_by_name.get("mid_b").unwrap(), - routes: vec![OldRoute { - heads: vec![*graph.ids_by_name.get("mid_a").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_b").unwrap()], - },], - }, - OldPackageDependency { - importer: *graph.ids_by_name.get("mid_b").unwrap(), - imported: *graph.ids_by_name.get("mid_c").unwrap(), - routes: vec![OldRoute { - heads: vec![*graph.ids_by_name.get("mid_b").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_c").unwrap()], - },], - }, - ] - ); - } - - #[test] - fn test_find_illegal_dependencies_with_container() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![OldPackageDependency { - importer: *graph.ids_by_name.get("mypackage.low").unwrap(), - imported: *graph.ids_by_name.get("mypackage.high").unwrap(), - routes: vec![ - OldRoute { - heads: vec![*graph.ids_by_name.get("mypackage.low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mypackage.high.yellow").unwrap()], - }, - OldRoute { - heads: vec![*graph.ids_by_name.get("mypackage.low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("mypackage.utils").unwrap()], - tails: vec![*graph.ids_by_name.get("mypackage.high.red").unwrap()], - }, - ], - }] - ); - } - - #[test] - fn test_generate_module_permutations() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_generate_module_permutations_sibling_layer_not_independent() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: false, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_layers_from_levels() { - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "medium_a".to_string(), - "medium_b".to_string(), - "medium_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - - let result = _layers_from_levels(&levels); - - assert_eq!( - HashSet::<&str>::from_iter(result), - HashSet::from_iter(["high", "medium_a", "medium_b", "medium_c", "low",]), - ) - } -} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 26832350..5278a7a8 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,29 +1,18 @@ -mod containers; -// TODO make these private. -pub mod dependencies; -mod graph; -pub mod importgraph; -pub mod layers; - -use crate::dependencies::OldPackageDependency; -use crate::graph::{DetailedImport, Graph, Module, PackageDependency}; -use containers::check_containers_exist; -use importgraph::ImportGraph; -use layers::Level; +pub mod graph; + +use crate::graph::{DetailedImport, Graph, Module, PackageDependency, Level}; use log::info; use pyo3::create_exception; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; -use std::collections::{HashMap, HashSet}; -//use petgraph::Graph; +use std::collections::HashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { pyo3_log::init(); m.add_class::()?; - m.add_function(wrap_pyfunction!(find_illegal_dependencies, m)?)?; m.add("NoSuchContainer", py.get_type_bound::())?; Ok(()) } @@ -316,7 +305,7 @@ impl GraphWrapper { ._graph .find_illegal_dependencies_for_layers(levels, containers) { - Ok(dependencies) => _convert_dependencies_to_python_new(py, &dependencies), + Ok(dependencies) => _convert_dependencies_to_python(py, &dependencies), Err(error) => Err(NoSuchContainer::new_err(format!( "Container {} does not exist.", error.container @@ -330,48 +319,7 @@ impl GraphWrapper { } } -#[pyfunction] -pub fn find_illegal_dependencies<'py>( - py: Python<'py>, - levels: &Bound<'py, PyTuple>, - containers: &Bound<'py, PySet>, - importeds_by_importer: &Bound<'py, PyDict>, -) -> PyResult> { - info!("Using Rust to find illegal dependencies."); - - let importeds_by_importer_strings: HashMap> = - importeds_by_importer.extract()?; - let importeds_by_importer_strs = strings_to_strs_hashmap(&importeds_by_importer_strings); - - let graph = ImportGraph::new(importeds_by_importer_strs); - let levels_rust = rustify_levels(levels); - let containers_rust: HashSet = containers.extract()?; - - if let Err(err) = check_containers_exist(&graph, &containers_rust) { - return Err(NoSuchContainer::new_err(err)); - } - - let dependencies = py.allow_threads(|| { - layers::find_illegal_dependencies(&graph, &levels_rust, &containers_rust) - }); - - convert_dependencies_to_python(py, dependencies, &graph) -} - -fn strings_to_strs_hashmap<'a>( - string_map: &'a HashMap>, -) -> HashMap<&'a str, HashSet<&'a str>> { - let mut str_map: HashMap<&str, HashSet<&str>> = HashMap::new(); - for (key, set) in string_map { - let mut str_set: HashSet<&str> = HashSet::new(); - for item in set.iter() { - str_set.insert(item); - } - str_map.insert(key.as_str(), str_set); - } - str_map -} fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { let mut rust_levels: Vec = vec![]; @@ -398,50 +346,8 @@ fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { rust_levels } -fn convert_dependencies_to_python<'py>( - py: Python<'py>, - dependencies: Vec, - graph: &ImportGraph, -) -> PyResult> { - let mut python_dependencies: Vec> = vec![]; - - for rust_dependency in dependencies { - let python_dependency = PyDict::new_bound(py); - python_dependency.set_item("imported", graph.names_by_id[&rust_dependency.imported])?; - python_dependency.set_item("importer", graph.names_by_id[&rust_dependency.importer])?; - let mut python_routes: Vec> = vec![]; - for rust_route in rust_dependency.routes { - let route = PyDict::new_bound(py); - let heads: Vec> = rust_route - .heads - .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) - .collect(); - route.set_item("heads", PyFrozenSet::new_bound(py, &heads)?)?; - let middle: Vec> = rust_route - .middle - .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) - .collect(); - route.set_item("middle", PyTuple::new_bound(py, &middle))?; - let tails: Vec> = rust_route - .tails - .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) - .collect(); - route.set_item("tails", PyFrozenSet::new_bound(py, &tails)?)?; - - python_routes.push(route); - } - - python_dependency.set_item("routes", PyTuple::new_bound(py, python_routes))?; - python_dependencies.push(python_dependency) - } - - Ok(PyTuple::new_bound(py, python_dependencies)) -} -fn _convert_dependencies_to_python_new<'py>( +fn _convert_dependencies_to_python<'py>( py: Python<'py>, dependencies: &Vec, ) -> PyResult> { diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 64df80e3..b06047be 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,7 +1,6 @@ -use _rustgrimp::importgraph::ImportGraph; -use _rustgrimp::layers::{find_illegal_dependencies, Level}; +use _rustgrimp::graph::{Graph, Level, Module}; use serde_json::{Map, Value}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::fs; #[test] @@ -9,15 +8,20 @@ fn test_large_graph() { let data = fs::read_to_string("tests/large_graph.json").expect("Unable to read file"); let value: Value = serde_json::from_str(&data).unwrap(); let items: &Map = value.as_object().unwrap(); - let mut importeds_by_importer: HashMap<&str, HashSet<&str>> = HashMap::new(); + let mut graph = Graph::default(); for (importer, importeds_value) in items.iter() { - let mut importeds = HashSet::new(); for imported in importeds_value.as_array().unwrap() { - importeds.insert(imported.as_str().unwrap()); + graph.add_import( + &Module { + name: importer.to_string(), + }, + &Module { + name: imported.to_string(), + }, + ); } - importeds_by_importer.insert(importer, importeds); } - let graph = ImportGraph::new(importeds_by_importer); + assert_eq!(graph.get_modules().len(), 28346); let levels = vec![ Level { @@ -43,5 +47,5 @@ fn test_large_graph() { ]; let containers = HashSet::from(["mypackage".to_string()]); - find_illegal_dependencies(&graph, &levels, &containers); + assert_eq!(graph.find_illegal_dependencies_for_layers(levels, containers).unwrap().len(), 9); }