compose_kiwi_description: check for changed files

kiwi_keg/tools/compose_kiwi_description.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 SUSE Software Solutions Germany GmbH. All rights reserved.
+# Copyright (c) 2023 SUSE Software Solutions Germany GmbH. All rights reserved.
 #
 # This file is part of keg.
 #
@@ -28,6 +28,8 @@
         [--generate-multibuild=<true|false>]
         [--new-image-change=<changelog_entry>]
         [--changelog-format=<format>]
+        [--purge-stale-files=<true|false>]
+        [--purge-ignore=<regex>]
     compose_kiwi_description -h | --help
     compose_kiwi_description --version
 
@@ -86,21 +88,32 @@
         converted if necessary. Conversion from 'osc' is not supported.
         [default: json]
 
+    --purge-stale-files=<true|false>
+        Purge files from existing image description if the generated image
+        description does not contain them. [default: true]
+
+    --purge-ignore=<regex>
+        When checking for old files to purge, ignore files matching <regex>
+        (optional). [default: '']
 
 """
 import docopt
+import filecmp
 import glob
+import hashlib
 import itertools
 import json
 import logging
 import os
 import pathlib
+import re
 import subprocess
 import sys
 import tempfile
 import yaml
 import xml.etree.ElementTree as ET
 
+from difflib import Differ
 from datetime import datetime, timezone
 
 from kiwi_keg.version import __version__
@@ -317,6 +330,90 @@ def write_changelog(log_file, log_format, changes, append=False):
             json.dump(changes, outf, indent=2, default=str)
 
 
+def kiwi_files_equivalent(old_kiwi, new_kiwi, ignore_version_change):
+
+    with open(old_kiwi, 'r') as fh:
+        old_kiwi_content = fh.readlines()
+    with open(new_kiwi, 'r') as fh:
+        new_kiwi_content = fh.readlines()
+
+    differ = Differ()
+    diff = differ.compare(old_kiwi_content, new_kiwi_content)
+    ignore_str = 'generated by keg on'
+    if ignore_version_change:
+        ignore_str += r'|<version>[0123456789\.]+</version>'
+    ignore_regex = re.compile(ignore_str)
+
+    for d in diff:
+        if d[0] == '-' and not ignore_regex.search(d):
+            return False
+
+    return True
+
+
+def tar_files_equivalent(file1, file2):
+    result = subprocess.run(
+        ['tar', 'xf', file1, '-O'],
+        stdout=subprocess.PIPE
+    )
+    sum1 = hashlib.sha256(result.stdout).digest()
+    result = subprocess.run(
+        ['tar', 'xf', file2, '-O'],
+        stdout=subprocess.PIPE
+    )
+    sum2 = hashlib.sha256(result.stdout).digest()
+    return sum1 == sum2
+
+
+def files_equivalent(filename, dir1, dir2, ignore_kiwi_version):
+    path1 = os.path.join(dir1, filename)
+    path2 = os.path.join(dir2, filename)
+
+    if not os.path.exists(path1) or not os.path.exists(path2):
+        return False
+
+    if filename.endswith('.kiwi'):
+        return kiwi_files_equivalent(path1, path2, ignore_kiwi_version)
+
+    if 'tar' in filename.split('.'):
+        return tar_files_equivalent(path1, path2)
+
+    return filecmp.cmp(path1, path2, shallow=False)
+
+
+def delete_unchanged_files(old_dir, new_dir, ignore_kiwi_version):
+    new_files = list(os.scandir(new_dir))
+
+    have_changes = False
+    ignore_regex = re.compile('log_sources.*')
+    for f in new_files:
+        if not ignore_regex.match(f.name):
+            if files_equivalent(f.name, old_dir, new_dir, ignore_kiwi_version):
+                if f.name != 'config.kiwi':
+                    log.info('Deleting unchanged file {}'.format(f.name))
+                    os.remove(f.path)
+            else:
+                have_changes = True
+
+    return have_changes
+
+
+def get_stale_files(old_dir, new_dir, ignore_exp):
+    old_files = list(os.scandir(old_dir))
+    stale_files = []
+
+    purge_ignore = '_service|_keg_revisions'
+    if ignore_exp:
+        purge_ignore += '|' + ignore_exp
+    ignore_regex = re.compile(purge_ignore)
+    for f in old_files:
+        if f.is_file() and not ignore_regex.match(f.name):
+            if not os.path.exists(os.path.join(new_dir, f.name)):
+                stale_files.append(f.name)
+
+    return stale_files
+
+
 def main() -> None:
     args = docopt.docopt(__doc__, version=__version__)
 
@@ -390,9 +487,24 @@ def main() -> None:
     image_generator.create_custom_files(
         overwrite=True
     )
-    if args['--generate-multibuild']:
+    if args['--generate-multibuild'] == 'true':
         image_generator.create_multibuild_file(overwrite=True)
 
+    stale_files = []
+    if args['--purge-stale-files'] == 'true':
+        stale_files = get_stale_files('.', args['--outdir'], args['--purge-ignore'])
+
+    files_changed = delete_unchanged_files('.', args['--outdir'], args['--version-bump'] == 'true')
+    if not files_changed:
+        if stale_files:
+            log.info('Generated files are identical to existing ones, '
+                     'but old image description has stale files.')
+        else:
+            log.warning('Generated image description is identical to existing one.')
+            if args['--force'] != 'true':
+                log.info('Aborting.')
+                sys.exit()
+
     if handle_changelog:
         sig = SourceInfoGenerator(image_definition, dest_dir=args['--outdir'])
         sig.write_source_info()
@@ -427,12 +539,7 @@ def main() -> None:
                 have_changes |= generate_changelog(source_log, changes_path, args['--changelog-format'], image_version, rev_args)
 
             if not have_changes:
-                log.warning('Image has no changes.')
-                if args['--force'] != 'true':
-                    log.info('Deleting generated files.')
-                    for f in next(os.walk(args['--outdir']))[2]:
-                        os.remove(os.path.join(args['--outdir'], f))
-                    sys.exit()
+                log.warning('Image description has changed but no new change log entries were generated.')
 
             for source_log, flavor in get_log_sources(os.path.join(args['--outdir'])):
                 changes_filename = f'{flavor}{"." if flavor else ""}changes.{log_ext}'
@@ -444,3 +551,7 @@ def main() -> None:
     if args['--update-revisions'] == 'true':
         # capture current commits
         update_revisions(repos, args['--outdir'])
+
+    for f in stale_files:
+        log.info('Deleting stale file {}'.format(f))
+        os.remove(f)

obs/compose_kiwi_description.service

@@ -40,4 +40,10 @@
   <parameter name="changelog-format">
     <description>Set output format for generated change log. Supported values are 'yaml', 'json', and 'osc'. Existing change logs will be converted if necessary. Conversion from 'osc' is not supported.  [default: json]</description>
   </parameter>
+  <parameter name="purge-stale-files">
+    <description>Purge files from existing image description if the generated image description does not contain them. [default: true]</description>
+  </parameter>
+  <parameter name="purge-ignore">
+    <description>Regular expression. When checking for old files to purge, ignore matching files. (optional)</description>
+  </parameter>
 </service>

test/data/compose_tests/new/config.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image</specification>
+    </description>
+    <preferences>
+        <version>1.2.3</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>

test/data/compose_tests/new/config2.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image</specification>
+    </description>
+    <preferences>
+        <version>1.2.4</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>

test/data/compose_tests/new/config3.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image2</specification>
+    </description>
+    <preferences>
+        <version>1.2.3</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>

test/data/compose_tests/new/differs.txt

@@ -0,0 +1 @@
+bar

test/data/compose_tests/new/same.txt

@@ -0,0 +1 @@
+foo

test/data/compose_tests/old/config.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image</specification>
+    </description>
+    <preferences>
+        <version>1.2.3</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>