diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 6072d5680..e9f25c6ab 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -4,7 +4,7 @@ FILE( GLOB ALLCC RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" ) STRING( REPLACE ".cc" ";" APLLPROG ${ALLCC} ) # make sure not to statically linked installed tools -SET( LINKALLSYM CalculateReusableBlocks DownloadFiles ) +SET( LINKALLSYM CalculateReusableBlocks DownloadFiles zypp-runpurge) FOREACH( loop_var ${APLLPROG} ) ADD_EXECUTABLE( ${loop_var} diff --git a/tools/zypp-runpurge.cc b/tools/zypp-runpurge.cc index ca5f3f800..4ebea7e02 100644 --- a/tools/zypp-runpurge.cc +++ b/tools/zypp-runpurge.cc @@ -24,6 +24,7 @@ int usage( const argparse::Options & options_r, int return_r = 0 ) int main ( int argc, char *argv[] ) { + auto z = getZYpp(); appname = Pathname::basename( argv[0] ); argparse::Options options; @@ -34,7 +35,7 @@ int main ( int argc, char *argv[] ) auto result = options.parse( argc, argv ); - if ( result.count( "help" ) || !result.count("uname") || !result.positionals().size() ) + if ( result.count( "help" ) || !result.positionals().size() ) return usage( options, 1 ); const std::string &testcaseDir = result.positionals().front(); @@ -51,31 +52,65 @@ int main ( int argc, char *argv[] ) TestSetup t; try { - t.LoadSystemAt( result.positionals().front() ); + t.LoadSystemAt( "." ); } catch ( const zypp::Exception &e ) { std::cerr << "Failed to load the testcase at " << result.positionals().front() << std::endl; std::cerr << "Got exception: " << e << std::endl; return 1; } + std::string unameR; + if ( result.count("uname") ) { + unameR = result["uname"].arg(); + } else { + std::cout << "No --uname provided. Guessing it from the latest kernel-default installed...." << std::endl; + const PoolItem & running( ui::Selectable::get("kernel-default")->theObj() ); + if ( not running ) { + std::cerr << "Oops: No installed kernel-default." << std::endl; + return usage( options, 1 ); + } + std::cout << "Guess running: " << running.asString() << endl; + const Capability unameRProvides { "kernel-uname-r" }; // indicator provides + for ( const auto & cap : running.provides() ) { + if ( cap.matches( unameRProvides ) == CapMatch::yes ) { + unameR = cap.detail().ed().asString(); + break; + } + } + if ( unameR.empty() ) { + std::cerr << "Oops: Guessed kernel does not provide " << unameRProvides << std::endl; + return usage( options, 1 ); + } + std::cout << "Guess --uname: " << unameR << endl; + } + std::string keepSpec = "oldest,running,latest"; if ( result.count("keepSpec") ) { keepSpec = result["keepSpec"].arg(); } + PurgeKernels krnls; - krnls.setUnameR( result["uname"].arg() ); + krnls.setUnameR( unameR ); krnls.setKeepSpec( keepSpec ); krnls.markObsoleteKernels(); - const auto &makeNVRA = []( const PoolItem &pck ) -> std::string { - return pck.name() + "-" + pck.edition().asString() + "." + pck.arch().asString(); - }; - + unsigned count = 0; auto pool = ResPool::instance(); const filter::ByStatus toBeUninstalledFilter( &ResStatus::isToBeUninstalled ); for ( auto it = pool.byStatusBegin( toBeUninstalledFilter ); it != pool.byStatusEnd( toBeUninstalledFilter ); it++ ) { - std::cout << "Removing " << makeNVRA(*it) + (it->status().isByUser() ? " (by user)" : " (autoremoved)") << std::endl; + std::cout << "Removing " << it->asString() + (it->status().isByUser() ? " (by user)" : " (autoremoved)") << std::endl; + ++count; + } + std::cout << "Purged kernels: " << count << std::endl; + + const auto & clusterResult { krnls.clusterKmps() }; + std::cout << clusterResult.first << endl; + + const auto & purgableKmps { clusterResult.second }; + std::cout << "Purgable Kmps: " << purgableKmps.size() << std::endl; + for ( const auto & kmp : purgableKmps ) { + std::cout << " purge: " << kmp << std::endl; } return 0; diff --git a/zypp/PurgeKernels.cc b/zypp/PurgeKernels.cc index bb7d16a84..20dd3efca 100644 --- a/zypp/PurgeKernels.cc +++ b/zypp/PurgeKernels.cc @@ -56,6 +56,18 @@ namespace zypp { }; using GroupMap = std::unordered_map; + std::ostream & operator<<( std::ostream & str, const GroupInfo::GroupType & obj ) { + switch ( obj ) { +#define OUTS(t) case GroupInfo::t: return str << #t + OUTS( None ); + OUTS( Kernels ); + OUTS( RelatedBinaries ); + OUTS( Sources ); +#undef OUTS + }; + return str << "[UNKNOWN_GroupType]"; + } + struct PurgeKernels::Impl { Impl() { @@ -505,7 +517,8 @@ namespace zypp { const filter::ByStatus toBeUninstalledFilter( &ResStatus::isToBeUninstalled ); // kernel flavour regex - const str::regex kernelFlavourRegex("^kernel-(.*)$"); + // XXX: No dashes in flavor names + const str::regex kernelFlavourRegex("^kernel-([^-]+)(-.*)?$"); // the map of all installed kernel packages, grouped by Flavour -> Arch -> Version -> (List of all packages in that category) // devel and source packages are grouped together @@ -594,12 +607,6 @@ namespace zypp { std::string flavour = what[1]; - // XXX: No dashes in flavor names - const auto dash = flavour.find_first_of('-'); - if ( dash != std::string::npos ) { - flavour = flavour.substr( 0, dash ); - } - // the ident for kernels is the flavour, to also handle cases like kernel-base and kernel which should be in the same group handled together addPackageToMap( GroupInfo::Kernels, flavour, flavour, installedKrnlPck ); @@ -694,4 +701,185 @@ namespace zypp { return _pimpl->_keepSpec; } -} + std::pair PurgeKernels::clusterKmps( const bool checkSystem ) + { + using SolvableSet =std::unordered_set; + + // Whether slv is in container + auto contains = []( const SolvableSet & container_r, const sat::Solvable & slv_r ) -> bool { + return container_r.find( slv_r ) != container_r.end(); + }; + + // Remove items from fitting which do not occur in matching + auto intersect = [&contains]( SolvableSet & fitting_r, const SolvableSet & matching_r ) { + for ( auto it = fitting_r.begin(); it != fitting_r.end(); ) { + if ( contains( matching_r, *it ) ) + ++it; + else + it = fitting_r.erase( it ); + } + }; + + SolvableSet kernels; // multiversion kernels, if checkSystem is true only the ones to keep. + SolvableSet purged; // If checkSystem is true, kernels to be purged are kept here. + SolvableSet kmps; // multiversion kmps + { + const Capability idxcap { "kernel-uname-r" }; + sat::WhatProvides idx { idxcap }; + for ( const auto & i : idx ) { + if ( i.isSystem() != checkSystem ) + continue; + if ( not checkSystem || PoolItem(i).status().staysInstalled() ) + kernels.insert( i ); + else + purged.insert( i ); + } + } + { + const Capability idxcap { "multiversion(kernel)" }; + sat::WhatProvides idx { idxcap }; + + StrMatcher matchMod { "kmod(*)", Match::GLOB }; + auto isKmp = [&matchMod]( const sat::Solvable & slv_r ) -> bool { + for ( const auto & prov : slv_r.provides() ) { + if ( matchMod.doMatch( prov.detail().name().c_str() ) ) + return true; + } + return false; + }; + + for ( const auto & i : idx ) { + if ( i.isSystem() != checkSystem ) + continue; + if ( contains( kernels, i ) || contains( purged, i ) ) + continue; + if ( isKmp( i ) ) kmps.insert( i ); + // else: multiversion others + } + } + MIL << "!Kernel " << kernels.size() << ", purged " << purged.size() << ", KMPs " << kmps.size() << std::endl; + + // Caching which KMP requirement is provided by what kernel (kept ones only). + std::unordered_map whatKernelProvidesCache; + auto whatKernelProvides = [&]( const Capability & req ) -> const SolvableSet & { + auto iter = whatKernelProvidesCache.find( req ); + if ( iter != whatKernelProvidesCache.end() ) + return iter->second; // hit + // miss: + SolvableSet & providingKernels = whatKernelProvidesCache[req]; + sat::WhatProvides idx { req }; + for ( const auto & i : idx ) { + if ( contains( kernels, i ) ) + providingKernels.insert( i ); + } + return providingKernels; + }; + + // using Bucket = std::set; ///< fitting KMP versions + // using KmpBuckets = std::map; ///< KMP name : fitting KMP versions + // using KernelKmps = std::map; ///< kernel : KMP name : fitting KMP versions + // using PurgableKmps = std::unordered_set; ///< KMPs found to be superfluous + + // Cluster the KMPs.... + KernelKmps kernelKmps; + SolvableSet protectedKmps; + + for ( const auto & kmp : kmps ) { + // Kernels satisfying all kernel related requirements + std::optional fittingKernels; + for ( const auto & req : kmp.requires() ) { + const SolvableSet & providingKernels { whatKernelProvides( req ) }; + if ( providingKernels.size() == 0 ) + continue; // not a kernel related requirement + if ( not fittingKernels ) { + fittingKernels = providingKernels; // initial set + } else { + intersect( *fittingKernels, providingKernels ); + } + } + if ( fittingKernels ) { + for ( const auto & kernel : *fittingKernels ) { + kernelKmps[kernel][kmp.ident()].insert( kmp ); + } + } + else { + // The Solvable::noSolvable entry for KMPs with no kernel requirements + kernelKmps[sat::Solvable::noSolvable][kmp.ident()].insert( kmp ); + DBG << "PROTECT (no kernel requirements)" << kmp << std::endl; + protectedKmps.insert( kmp ); + } + } + + // Now protect the highest KMP versions per kernel. + // ==== kernel-default 5.14.21-150400.24.136.1 (i) + // drbd-kmp-default fit 3 versions(s) + // - drbd-kmp-default 9.0.30~1+git.10bee2d5_k5.14.21_150400.22-150400.1.75 (i) + // - drbd-kmp-default 9.0.30~1+git.10bee2d5_k5.14.21_150400.24.11-150400.3.2.9 (i) + // * drbd-kmp-default 9.0.30~1+git.10bee2d5_k5.14.21_150400.24.46-150400.3.4.1 (i) + for ( const auto & [kernel,bucket] : kernelKmps ) { + if ( not kernel ) // The Solvable::noSolvable entry for KMPs with no kernel requirements + continue; // (already in protectedKmps) + + for ( const auto & el : bucket ) { + const auto & versions { el.second }; + sat::Solvable highest { *versions.rbegin() }; + DBG << "PROTECT (best)" << highest << " for " << kernel << std::endl; + protectedKmps.insert( highest ); + } + } + + // Build result + SolvableSet purgableKmps { kmps }; + for ( const auto & kmp : protectedKmps ) { + purgableKmps.erase( kmp ); + } + MIL << "PROTECTED: " << protectedKmps.size() << std::endl; + MIL << "PURGABLE: " << purgableKmps.size() << std::endl; + for ( const auto & kmp : purgableKmps ) { + DBG << "purge: " << kmp << std::endl; + } + return { kernelKmps, purgableKmps }; + } + + std::ostream & operator<<( std::ostream & str, const PurgeKernels::KernelKmps & obj ) + { + using SolvableSet = PurgeKernels::PurgableKmps; + SolvableSet kmps; + SolvableSet protectedKmps; + + for ( const auto & [kernel,bucket] : obj ) { + if ( not kernel ) + str << std::endl << "==== Does not require a kept kernel" << std::endl; + else + str << std::endl << "==== " << kernel << std::endl; + + for ( const auto & [kmp,versions] : bucket ) { + sat::Solvable highest { *versions.rbegin() }; + if ( kernel ) + protectedKmps.insert( highest ); + str << " " << kmp << " fit " << versions.size() << " versions(s)" << std::endl; + for ( const auto & version : versions ) { + str << " "<<(not kernel||version==highest?"*":"-")<<" " << version << std::endl; + kmps.insert( version ); + if ( not kernel ) + protectedKmps.insert( version ); + } + } + } + + SolvableSet purgableKmps { kmps }; + for ( const auto & kmp : protectedKmps ) { + purgableKmps.erase( kmp ); + } + + str << "KMPS: " << kmps.size() << std::endl; + str << "PROTECTED: " << protectedKmps.size() << std::endl; + str << "PURGABLE: " << purgableKmps.size() << std::endl; + for ( const auto & kmp : purgableKmps ) { + str << " purge: " << kmp << std::endl; + } + + return str; + } + +} // namespace zypp diff --git a/zypp/PurgeKernels.h b/zypp/PurgeKernels.h index 1a32fce41..3004e68e5 100644 --- a/zypp/PurgeKernels.h +++ b/zypp/PurgeKernels.h @@ -10,6 +10,12 @@ * */ +#include +#include +#include +#include +#include + #include #include #include @@ -64,10 +70,69 @@ namespace zypp { void setKeepSpec( const std::string &val ); std::string keepSpec () const; + public: + /// Helper for version sorted solvable container + struct byNVRA { + bool operator()( const sat::Solvable & lhs, const sat::Solvable & rhs ) const + { return compareByNVRA( lhs, rhs ) < 0; } + }; + + using Bucket = std::set; ///< fitting KMP versions + using KmpBuckets = std::map; ///< KMP name : fitting KMP versions + using KernelKmps = std::map; ///< kernel : KMP name : fitting KMP versions + + using PurgableKmps = std::unordered_set; ///< KMPs found to be superfluous + + /** + * Suggest purgable KMP versions. + * + * Per kernel whose \ref ResStatus::staysInstalled return the installed KMPs whose requirements + * are satisfied by this kernel. Intended to be run after \ref markObsoleteKernels. It returns + * the KMP stats for the kernels kept and suggests the KMPs which could be purged because they may + * be superfluous. It does not change the ResPool status. + * + * \note The computation here is solely based on kernel<>KMP requirements. Requirements between + * a KMP and other KMPs or packages are not taken into accout here. So you should \b never file + * hard remove requests for PurgableKmps. Use weak requests, so they stay in case other non-kernel + * packages need them. + * + * For each kernel just the highest KMP version in it's Buckets is needed. Lower versions + * may be needed by different kernels or may be superfluous. + * + * \note A special entry for \ref noSolvable may exist in \ref KernelKmps. It collects all + * KMPs which do not have any requirements regarding a (kept) kernel at all. Whether they are + * needed or are superfluous can not be decided here. We leave leave them untouched. Maybe they + * are removed along with a purged kernel. + * + * The optional \a checkSystem_r argument is for testing and debugging only. If set to \c false + * the stats are built for all kernel and KMP versions available in the repositories. + * + * \code + * ==== (39558)kernel-default-5.14.21-150400.24.136.1.x86_64(@System) + * cluster-md-kmp-default fit 1 versions(s) + * * (39429)cluster-md-kmp-default-5.14.21-150400.24.136.1.x86_64(@System) + * crash-kmp-default fit 1 versions(s) + * * (39441)crash-kmp-default-7.3.0_k5.14.21_150400.24.49-150400.3.5.8.x86_64(@System) + * dlm-kmp-default fit 1 versions(s) + * * (39467)dlm-kmp-default-5.14.21-150400.24.136.1.x86_64(@System) + * drbd-kmp-default fit 3 versions(s) + * - (39477)drbd-kmp-default-9.0.30~1+git.10bee2d5_k5.14.21_150400.22-150400.1.75.x86_64(@System) + * - (39478)drbd-kmp-default-9.0.30~1+git.10bee2d5_k5.14.21_150400.24.11-150400.3.2.9.x86_64(@System) + * * (39476)drbd-kmp-default-9.0.30~1+git.10bee2d5_k5.14.21_150400.24.46-150400.3.4.1.x86_64(@System) + * gfs2-kmp-default fit 1 versions(s) + * * (39501)gfs2-kmp-default-5.14.21-150400.24.136.1.x86_64(@System) + * ocfs2-kmp-default fit 1 versions(s) + * * (39948)ocfs2-kmp-default-5.14.21-150400.24.136.1.x86_64(@System) + * \endcode + */ + static std::pair clusterKmps( const bool checkSystem_r = true ); + struct Impl; private: RW_pointer _pimpl; }; + /** \relates PurgeKernels::KernelKmps stream output */ + std::ostream & operator<<( std::ostream & str, const PurgeKernels::KernelKmps & obj ); } diff --git a/zypp/misc/LoadTestcase.cc b/zypp/misc/LoadTestcase.cc index a0eda0b10..25925205b 100644 --- a/zypp/misc/LoadTestcase.cc +++ b/zypp/misc/LoadTestcase.cc @@ -177,7 +177,7 @@ namespace zypp::misc::testcase { if ( !trial["trial"] ) return makeError("Every element in the trials sequence needs to have the 'trial' key."); - if ( !yamltest::detail::parseTrial( trial["trial"], t, err) ) + if ( !yamltest::detail::parseTrial( trial["trial"], t, _setup.data().globalPath, err) ) return false; _trials.push_back( t ); } diff --git a/zypp/misc/YamlTestcaseHelpers.h b/zypp/misc/YamlTestcaseHelpers.h index eb8d9b576..31ead764c 100644 --- a/zypp/misc/YamlTestcaseHelpers.h +++ b/zypp/misc/YamlTestcaseHelpers.h @@ -46,7 +46,7 @@ namespace yamltest::detail { const std::string &fName = data.as(); MIL << "Trying to load list from file " << fName << std::endl; try { - auto doc = YAML::LoadFile( fName ); + auto doc = YAML::LoadFile( (target.globalPath / fName).asString() ); if ( doc.Type() != YAML::NodeType::Sequence ) { if ( err ) *err = "Expected the top node to be a sequence in external file for key: "; return false; @@ -218,7 +218,7 @@ namespace yamltest::detail { if ( !success ) return false; } else if ( key == ("autoinst") ) { - bool success = readListInlineOrFromFile( [&]( const YAML::Node &dataNode, auto ){ + bool success = readListInlineOrFromFile( [&]( const YAML::Node &dataNode, std::string * err ){ target.autoinstalled.push( zypp::IdString( dataNode.as() ).id() ); return true; }, err ); @@ -238,10 +238,10 @@ namespace yamltest::detail { } template - bool parseJobs ( const YAML::Node &trial, std::vector &target, std::string *err ); + bool parseJobs ( const YAML::Node &trial, std::vector &target, const zypp::Pathname &testcaseDir, std::string *err ); template - bool parseSingleJob ( const YAML::Node &jobNode, std::vector &target, std::string *err ) { + bool parseSingleJob ( const YAML::Node &jobNode, std::vector &target, const zypp::Pathname &testcaseDir, std::string *err ) { constexpr bool isSubNode = std::is_same_v>; if ( jobNode["include"] ) { @@ -249,8 +249,8 @@ namespace yamltest::detail { const auto &fName = jobNode["include"].as(); MIL << "Including file " << fName << std::endl; try { - auto doc = YAML::LoadFile( fName ); - if ( !parseJobs( doc, target, err ) ) + auto doc = YAML::LoadFile( (testcaseDir / fName).asString() ); + if ( !parseJobs( doc, target, testcaseDir, err ) ) return false; MIL << "Including file " << fName << "was successful" << std::endl; } catch ( YAML::Exception &e ) { @@ -291,13 +291,13 @@ namespace yamltest::detail { // if the type of a data field is a sequence, we treat all the elements in there // as sub elements. Just like in XML you can have sub nodes its the same here // the key name is ignored in those cases and can be chosen freely - if ( !parseJobs( data, n.children(), err ) ) + if ( !parseJobs( data, n.children(), testcaseDir, err ) ) return false; } else if ( data.IsMap() ) { // if the type of a data field is a map, we build a child node from it. // Just like with sequence but a single field. // The key name is ignored in those cases and can be chosen freely - if ( !parseSingleJob( data, n.children(), err) ) + if ( !parseSingleJob( data, n.children(), testcaseDir, err) ) return false; } else { ERR << "Ignoring field " << key << " with unsupported type:" << data.Type() << std::endl; @@ -313,17 +313,17 @@ namespace yamltest::detail { } template - bool parseJobs ( const YAML::Node &trial, std::vector &target, std::string *err ) { + bool parseJobs ( const YAML::Node &trial, std::vector &target, const zypp::Pathname &testcaseDir, std::string *err ) { for ( const auto &jobNode : trial ) { - if ( !parseSingleJob( jobNode, target, err ) ) + if ( !parseSingleJob( jobNode, target, testcaseDir, err ) ) return false; } return true; } - bool parseTrial ( const YAML::Node &trial, zypp::misc::testcase::TestcaseTrial &target, std::string *err ) { + bool parseTrial ( const YAML::Node &trial, zypp::misc::testcase::TestcaseTrial &target, const zypp::Pathname &testcaseDir, std::string *err ) { MIL << "Parsing trials." << std::endl; - return parseJobs( trial, target.nodes(), err ); + return parseJobs( trial, target.nodes(), testcaseDir, err ); } }