PurgeKernels: Add clusterKmps to suggest purgable KMP versions (for b…

…sc#1232399)
openSUSE · Dec 13, 2024 · 9d25a89 · 9d25a89
1 parent c760fed
commit 9d25a89
Show file tree

Hide file tree

Showing 4 changed files with 261 additions and 4 deletions.
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
@@ -4,7 +4,7 @@ FILE( GLOB ALLCC RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" )
 STRING( REPLACE ".cc" ";" APLLPROG ${ALLCC} )
 
 # make sure not to statically linked installed tools
-SET( LINKALLSYM CalculateReusableBlocks DownloadFiles )
+SET( LINKALLSYM CalculateReusableBlocks DownloadFiles zypp-runpurge)
 
 FOREACH( loop_var ${APLLPROG} )
   ADD_EXECUTABLE( ${loop_var}

diff --git a/tools/zypp-runpurge.cc b/tools/zypp-runpurge.cc
@@ -24,6 +24,7 @@ int usage( const argparse::Options & options_r, int return_r = 0 )
 
 int main ( int argc, char *argv[] )
 {
+  auto z = getZYpp();
 
   appname = Pathname::basename( argv[0] );
   argparse::Options options;
@@ -94,12 +95,22 @@ int main ( int argc, char *argv[] )
   krnls.setKeepSpec( keepSpec );
   krnls.markObsoleteKernels();
 
-
-  std::cout << "Purged kernels: " << std::endl;
+  unsigned count = 0;
   auto pool = ResPool::instance();
   const filter::ByStatus toBeUninstalledFilter( &ResStatus::isToBeUninstalled );
   for ( auto it = pool.byStatusBegin( toBeUninstalledFilter ); it != pool.byStatusEnd( toBeUninstalledFilter );  it++  ) {
     std::cout << "Removing " << it->asString() + (it->status().isByUser() ? " (by user)" : " (autoremoved)") << std::endl;
+    ++count;
+  }
+  std::cout << "Purged kernels: " << count << std::endl;
+
+  const auto & clusterResult { krnls.clusterKmps() };
+  std::cout << clusterResult.first << endl;
+
+  const auto & purgableKmps { clusterResult.second };
+  std::cout << "Purgable Kmps: " << purgableKmps.size() << std::endl;
+  for ( const auto & kmp : purgableKmps ) {
+    std::cout << "  purge: " << kmp << std::endl;
   }
 
   return 0;

diff --git a/zypp/PurgeKernels.cc b/zypp/PurgeKernels.cc
@@ -701,4 +701,185 @@ namespace zypp {
     return _pimpl->_keepSpec;
   }
 
-}
+  std::pair<PurgeKernels::KernelKmps,PurgeKernels::PurgableKmps> PurgeKernels::clusterKmps( const bool checkSystem )
+  {
+    using SolvableSet =std::unordered_set<sat::Solvable>;
+
+    // Whether slv is in container
+    auto contains = []( const SolvableSet & container_r, const sat::Solvable & slv_r ) -> bool {
+      return container_r.find( slv_r ) != container_r.end();
+    };
+
+    // Remove items from fitting which do not occur in matching
+    auto intersect = [&contains]( SolvableSet & fitting_r, const SolvableSet & matching_r ) {
+      for ( auto it = fitting_r.begin(); it != fitting_r.end(); ) {
+        if ( contains( matching_r, *it ) )
+          ++it;
+        else
+          it = fitting_r.erase( it );
+      }
+    };
+
+    SolvableSet kernels;  // multiversion kernels, if checkSystem is true only the ones to keep.
+    SolvableSet purged;   // If checkSystem is true, kernels to be purged are kept here.
+    SolvableSet kmps;     // multiversion kmps
+    {
+      const Capability idxcap { "kernel-uname-r" };
+      sat::WhatProvides idx { idxcap };
+      for ( const auto & i : idx ) {
+        if ( i.isSystem() != checkSystem )
+          continue;
+        if ( not checkSystem || PoolItem(i).status().staysInstalled() )
+          kernels.insert( i );
+        else
+          purged.insert( i );
+      }
+    }
+    {
+      const Capability idxcap { "multiversion(kernel)" };
+      sat::WhatProvides idx { idxcap };
+
+      StrMatcher matchMod { "kmod(*)", Match::GLOB };
+      auto isKmp = [&matchMod]( const sat::Solvable & slv_r ) -> bool {
+        for ( const auto & prov : slv_r.provides() ) {
+          if ( matchMod.doMatch( prov.detail().name().c_str() ) )
+            return true;
+        }
+        return false;
+      };
+
+      for ( const auto & i : idx ) {
+        if ( i.isSystem() != checkSystem )
+          continue;
+        if ( contains( kernels, i ) || contains( purged, i ) )
+          continue;
+        if ( isKmp( i ) ) kmps.insert( i );
+          // else: multiversion others
+      }
+    }
+    MIL << "!Kernel " << kernels.size() << ", purged " << purged.size() << ", KMPs " << kmps.size() << std::endl;
+
+    // Caching which KMP requirement is provided by what kernel (kept ones only).
+    std::unordered_map<Capability,SolvableSet> whatKernelProvidesCache;
+    auto whatKernelProvides = [&]( const Capability & req ) -> const SolvableSet & {
+      auto iter = whatKernelProvidesCache.find( req );
+      if ( iter != whatKernelProvidesCache.end() )
+        return iter->second;  // hit
+      // miss:
+      SolvableSet & providingKernels = whatKernelProvidesCache[req];
+      sat::WhatProvides idx { req };
+      for ( const auto & i : idx ) {
+        if ( contains( kernels, i ) )
+          providingKernels.insert( i );
+      }
+      return providingKernels;
+    };
+
+    // using Bucket     = std::set<sat::Solvable,byNVRA>;              ///< fitting KMP versions
+    // using KmpBuckets = std::map<IdString,Bucket>;                   ///< KMP name : fitting KMP versions
+    // using KernelKmps = std::map<sat::Solvable,KmpBuckets,byNVRA>;   ///< kernel : KMP name : fitting KMP versions
+    // using PurgableKmps = std::unordered_set<sat::Solvable>;         ///< KMPs found to be superfluous
+
+    // Cluster the KMPs....
+    KernelKmps kernelKmps;
+    SolvableSet protectedKmps;
+
+    for ( const auto & kmp : kmps ) {
+      // Kernels satisfying all kernel related requirements
+      std::optional<SolvableSet> fittingKernels;
+      for ( const auto & req : kmp.requires() ) {
+        const SolvableSet & providingKernels { whatKernelProvides( req ) };
+        if ( providingKernels.size() == 0 )
+          continue; // not a kernel related requirement
+        if ( not fittingKernels ) {
+          fittingKernels = providingKernels; // initial set
+        } else {
+          intersect( *fittingKernels, providingKernels );
+        }
+      }
+      if ( fittingKernels ) {
+        for ( const auto & kernel : *fittingKernels ) {
+          kernelKmps[kernel][kmp.ident()].insert( kmp );
+        }
+      }
+      else {
+        // The Solvable::noSolvable entry for KMPs with no kernel requirements
+        kernelKmps[sat::Solvable::noSolvable][kmp.ident()].insert( kmp );
+        DBG << "PROTECT (no kernel requirements)" << kmp << std::endl;
+        protectedKmps.insert( kmp );
+      }
+    }
+
+    // Now protect the highest KMP versions per kernel.
+    // ==== kernel-default 5.14.21-150400.24.136.1 (i)
+    //      drbd-kmp-default fit 3 versions(s)
+    //          - drbd-kmp-default 9.0.30~1+git.10bee2d5_k5.14.21_150400.22-150400.1.75 (i)
+    //          - drbd-kmp-default 9.0.30~1+git.10bee2d5_k5.14.21_150400.24.11-150400.3.2.9 (i)
+    //          * drbd-kmp-default 9.0.30~1+git.10bee2d5_k5.14.21_150400.24.46-150400.3.4.1 (i)
+    for ( const auto & [kernel,bucket] : kernelKmps ) {
+      if ( not kernel ) // The Solvable::noSolvable entry for KMPs with no kernel requirements
+        continue;       // (already in protectedKmps)
+
+      for ( const auto & el : bucket ) {
+        const auto & versions { el.second };
+        sat::Solvable highest { *versions.rbegin() };
+        DBG << "PROTECT (best)" << highest << " for " << kernel << std::endl;
+        protectedKmps.insert( highest );
+      }
+    }
+
+    // Build result
+    SolvableSet purgableKmps { kmps };
+    for ( const auto & kmp : protectedKmps ) {
+      purgableKmps.erase( kmp );
+    }
+    MIL << "PROTECTED: " << protectedKmps.size() << std::endl;
+    MIL << "PURGABLE:  " << purgableKmps.size() << std::endl;
+    for ( const auto & kmp : purgableKmps ) {
+      DBG << "purge:     " << kmp << std::endl;
+    }
+    return { kernelKmps, purgableKmps };
+  }
+
+  std::ostream & operator<<( std::ostream & str, const PurgeKernels::KernelKmps & obj )
+  {
+    using SolvableSet = PurgeKernels::PurgableKmps;
+    SolvableSet kmps;
+    SolvableSet protectedKmps;
+
+    for ( const auto & [kernel,bucket] : obj ) {
+      if ( not kernel )
+        str << std::endl << "==== Not kernel specific" << std::endl;
+      else
+        str << std::endl << "==== " << kernel << std::endl;
+
+      for ( const auto & [kmp,versions] : bucket ) {
+        sat::Solvable highest { *versions.rbegin() };
+        if ( kernel )
+          protectedKmps.insert( highest );
+        str << "     " << kmp << " fit " << versions.size() << " versions(s)" << std::endl;
+        for ( const auto & version : versions ) {
+          str << "         "<<(not kernel||version==highest?"*":"-")<<" " << version << std::endl;
+          kmps.insert( version );
+          if ( not kernel )
+            protectedKmps.insert( version );
+        }
+      }
+    }
+
+    SolvableSet purgableKmps { kmps };
+    for ( const auto & kmp : protectedKmps ) {
+      purgableKmps.erase( kmp );
+    }
+
+    str << "KMPS:      " << kmps.size() << std::endl;
+    str << "PROTECTED: " << protectedKmps.size() << std::endl;
+    str << "PURGABLE:  " << purgableKmps.size() << std::endl;
+    for ( const auto & kmp : purgableKmps ) {
+      str << "  purge:   " << kmp << std::endl;
+    }
+
+    return str;
+  }
+
+} // namespace zypp
diff --git a/zypp/PurgeKernels.h b/zypp/PurgeKernels.h
@@ -10,6 +10,12 @@
  *
 */
 
+#include <iosfwd>
+#include <utility>
+#include <map>
+#include <set>
+#include <unordered_set>
+
 #include <zypp/Globals.h>
 #include <zypp/PoolItem.h>
 #include <zypp/base/PtrTypes.h>
@@ -64,10 +70,69 @@ namespace zypp {
     void setKeepSpec( const std::string &val );
     std::string keepSpec () const;
 
+  public:
+    /// Helper for version sorted solvable container
+    struct byNVRA {
+      bool operator()( const sat::Solvable & lhs, const sat::Solvable & rhs ) const
+      { return compareByNVRA( lhs, rhs ) < 0; }
+    };
+
+    using Bucket     = std::set<sat::Solvable,byNVRA>;              ///< fitting KMP versions
+    using KmpBuckets = std::map<IdString,Bucket>;                   ///< KMP name : fitting KMP versions
+    using KernelKmps = std::map<sat::Solvable,KmpBuckets,byNVRA>;   ///< kernel : KMP name : fitting KMP versions
+
+    using PurgableKmps = std::unordered_set<sat::Solvable>;         ///< KMPs found to be superfluous
+
+    /**
+     * Suggest purgable KMP versions.
+     *
+     * Per kernel whose \ref ResStatus::staysInstalled return the installed KMPs whose requirements
+     * are satisfied by this kernel. Intended to be run after \ref markObsoleteKernels. It returns
+     * the KMP stats for the kernels kept and suggests the KMPs which could be purged because they may
+     * be superfluous. It does not change the ResPool status.
+     *
+     * \note The computation here is solely based on kernel<>KMP requirements. Requirements between
+     * a KMP and other KMPs or packages are not taken into accout here. So you should \b never file
+     * hard remove requests for PurgableKmps. Use weak requests, so they stay in case other non-kernel
+     * packages need them.
+     *
+     * For each kernel just the highest KMP version in it's Buckets is needed. Lower versions
+     * may be needed by different kernels or may be superfluous.
+     *
+     * \note A special entry for \ref noSolvable may exist in \ref KernelKmps. It collects all
+     * KMPs which do not have any requirements regarding a (kept) kernel at all. Whether they are
+     * needed or are superfluous can not be decided here. We leave leave them untouched. Maybe they
+     * are removed along with a purged kernel.
+     *
+     * The optional \a checkSystem_r argument is for testing and debugging only. If set to \c false
+     * the stats are built for all kernel and KMP versions available in the repositories.
+     *
+     * \code
+     * ==== (39558)kernel-default-5.14.21-150400.24.136.1.x86_64(@System)
+     *      cluster-md-kmp-default fit 1 versions(s)
+     *          * (39429)cluster-md-kmp-default-5.14.21-150400.24.136.1.x86_64(@System)
+     *      crash-kmp-default fit 1 versions(s)
+     *          * (39441)crash-kmp-default-7.3.0_k5.14.21_150400.24.49-150400.3.5.8.x86_64(@System)
+     *      dlm-kmp-default fit 1 versions(s)
+     *          * (39467)dlm-kmp-default-5.14.21-150400.24.136.1.x86_64(@System)
+     *      drbd-kmp-default fit 3 versions(s)
+     *          - (39477)drbd-kmp-default-9.0.30~1+git.10bee2d5_k5.14.21_150400.22-150400.1.75.x86_64(@System)
+     *          - (39478)drbd-kmp-default-9.0.30~1+git.10bee2d5_k5.14.21_150400.24.11-150400.3.2.9.x86_64(@System)
+     *          * (39476)drbd-kmp-default-9.0.30~1+git.10bee2d5_k5.14.21_150400.24.46-150400.3.4.1.x86_64(@System)
+     *      gfs2-kmp-default fit 1 versions(s)
+     *          * (39501)gfs2-kmp-default-5.14.21-150400.24.136.1.x86_64(@System)
+     *      ocfs2-kmp-default fit 1 versions(s)
+     *          * (39948)ocfs2-kmp-default-5.14.21-150400.24.136.1.x86_64(@System)
+     * \endcode
+     */
+    static std::pair<KernelKmps,PurgableKmps> clusterKmps( const bool checkSystem_r = true );
+
     struct Impl;
   private:
     RW_pointer<Impl> _pimpl;
   };
 
+  /** \relates PurgeKernels::KernelKmps stream output */
+  std::ostream & operator<<( std::ostream & str, const PurgeKernels::KernelKmps & obj );
 }