compose_kiwi_description: check for changed files

Check generated files for changes compared to existing ones and delete unchanged
files.

Add option to check existing image description for stale files (i.e. files not
generated by the new image definition), and remove those (enabled by default).

Do not abort anymore if the generated change log has no new entries, but base
decision to abort or proceed on result of the above checks.

kiwi_keg/tools/compose_kiwi_description.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 SUSE Software Solutions Germany GmbH. All rights reserved.
+# Copyright (c) 2023 SUSE Software Solutions Germany GmbH. All rights reserved.
 #
 # This file is part of keg.
 #
@@ -28,6 +28,8 @@
         [--generate-multibuild=<true|false>]
         [--new-image-change=<changelog_entry>]
         [--changelog-format=<format>]
+        [--purge-stale-files=<true|false>]
+        [--purge-ignore=<regex>]
     compose_kiwi_description -h | --help
     compose_kiwi_description --version
 
@@ -86,21 +88,32 @@
         converted if necessary. Conversion from 'osc' is not supported.
         [default: json]
 
+    --purge-stale-files=<true|false>
+        Purge files from existing image description if the generated image
+        description does not contain them. [default: true]
+
+    --purge-ignore=<regex>
+        When checking for old files to purge, ignore files matching <regex>
+        (optional). [default: '']
 
 """
 import docopt
+import filecmp
 import glob
+import hashlib
 import itertools
 import json
 import logging
 import os
 import pathlib
+import re
 import subprocess
 import sys
 import tempfile
 import yaml
 import xml.etree.ElementTree as ET
 
+from difflib import Differ
 from datetime import datetime, timezone
 
 from kiwi_keg.version import __version__
@@ -317,6 +330,90 @@ def write_changelog(log_file, log_format, changes, append=False):
             json.dump(changes, outf, indent=2, default=str)
 
 
+def kiwi_files_equivalent(old_kiwi, new_kiwi, ignore_version_change):
+
+    with open(old_kiwi, 'r') as fh:
+        old_kiwi_content = fh.readlines()
+    with open(new_kiwi, 'r') as fh:
+        new_kiwi_content = fh.readlines()
+
+    differ = Differ()
+    diff = differ.compare(old_kiwi_content, new_kiwi_content)
+    ignore_str = 'generated by keg on'
+    if ignore_version_change:
+        ignore_str += r'|<version>[0123456789\.]+</version>'
+    ignore_regex = re.compile(ignore_str)
+
+    for d in diff:
+        if d[0] == '-' and not ignore_regex.search(d):
+            return False
+
+    return True
+
+
+def tar_files_equivalent(file1, file2):
+    result = subprocess.run(
+        ['tar', 'xf', file1, '-O'],
+        stdout=subprocess.PIPE
+    )
+    sum1 = hashlib.sha256(result.stdout).digest()
+    result = subprocess.run(
+        ['tar', 'xf', file2, '-O'],
+        stdout=subprocess.PIPE
+    )
+    sum2 = hashlib.sha256(result.stdout).digest()
+    return sum1 == sum2
+
+
+def files_equivalent(filename, dir1, dir2, ignore_kiwi_version):
+    path1 = os.path.join(dir1, filename)
+    path2 = os.path.join(dir2, filename)
+
+    if not os.path.exists(path1) or not os.path.exists(path2):
+        return False
+
+    if filename.endswith('.kiwi'):
+        return kiwi_files_equivalent(path1, path2, ignore_kiwi_version)
+
+    if 'tar' in filename.split('.'):
+        return tar_files_equivalent(path1, path2)
+
+    return filecmp.cmp(path1, path2, shallow=False)
+
+
+def delete_unchanged_files(old_dir, new_dir, ignore_kiwi_version):
+    new_files = list(os.scandir(new_dir))
+
+    have_changes = False
+    ignore_regex = re.compile('log_sources.*')
+    for f in new_files:
+        if not ignore_regex.match(f.name):
+            if files_equivalent(f.name, old_dir, new_dir, ignore_kiwi_version):
+                if f.name != 'config.kiwi':
+                    log.info('Deleting unchanged file {}'.format(f.name))
+                    os.remove(f.path)
+            else:
+                have_changes = True
+
+    return have_changes
+
+
+def get_stale_files(old_dir, new_dir, ignore_exp):
+    old_files = list(os.scandir(old_dir))
+    stale_files = []
+
+    purge_ignore = '_service|_keg_revisions'
+    if ignore_exp:
+        purge_ignore += '|' + ignore_exp
+    ignore_regex = re.compile(purge_ignore)
+    for f in old_files:
+        if f.is_file() and not ignore_regex.match(f.name):
+            if not os.path.exists(os.path.join(new_dir, f.name)):
+                stale_files.append(f.name)
+
+    return stale_files
+
+
 def main() -> None:
     args = docopt.docopt(__doc__, version=__version__)
 
@@ -390,9 +487,24 @@ def main() -> None:
     image_generator.create_custom_files(
         overwrite=True
     )
-    if args['--generate-multibuild']:
+    if args['--generate-multibuild'] == 'true':
         image_generator.create_multibuild_file(overwrite=True)
 
+    stale_files = []
+    if args['--purge-stale-files'] == 'true':
+        stale_files = get_stale_files('.', args['--outdir'], args['--purge-ignore'])
+
+    files_changed = delete_unchanged_files('.', args['--outdir'], args['--version-bump'] == 'true')
+    if not files_changed:
+        if stale_files:
+            log.info('Generated files are identical to existing ones, '
+                     'but old image description has stale files.')
+        else:
+            log.warning('Generated image description is identical to existing one.')
+            if args['--force'] != 'true':
+                log.info('Aborting.')
+                sys.exit()
+
     if handle_changelog:
         sig = SourceInfoGenerator(image_definition, dest_dir=args['--outdir'])
         sig.write_source_info()
@@ -427,12 +539,7 @@ def main() -> None:
                 have_changes |= generate_changelog(source_log, changes_path, args['--changelog-format'], image_version, rev_args)
 
             if not have_changes:
-                log.warning('Image has no changes.')
-                if args['--force'] != 'true':
-                    log.info('Deleting generated files.')
-                    for f in next(os.walk(args['--outdir']))[2]:
-                        os.remove(os.path.join(args['--outdir'], f))
-                    sys.exit()
+                log.warning('Image description has changed but no new change log entries were generated.')
 
             for source_log, flavor in get_log_sources(os.path.join(args['--outdir'])):
                 changes_filename = f'{flavor}{"." if flavor else ""}changes.{log_ext}'
@@ -444,3 +551,7 @@ def main() -> None:
     if args['--update-revisions'] == 'true':
         # capture current commits
         update_revisions(repos, args['--outdir'])
+
+    for f in stale_files:
+        log.info('Deleting stale file {}'.format(f))
+        os.remove(f)

obs/compose_kiwi_description.service

@@ -40,4 +40,10 @@
   <parameter name="changelog-format">
     <description>Set output format for generated change log. Supported values are 'yaml', 'json', and 'osc'. Existing change logs will be converted if necessary. Conversion from 'osc' is not supported.  [default: json]</description>
   </parameter>
+  <parameter name="purge-stale-files">
+    <description>Purge files from existing image description if the generated image description does not contain them. [default: true]</description>
+  </parameter>
+  <parameter name="purge-ignore">
+    <description>Regular expression. When checking for old files to purge, ignore matching files. (optional)</description>
+  </parameter>
 </service>

test/data/compose_tests/new/config.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image</specification>
+    </description>
+    <preferences>
+        <version>1.2.3</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>

test/data/compose_tests/new/config2.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image</specification>
+    </description>
+    <preferences>
+        <version>1.2.4</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>

test/data/compose_tests/new/config3.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image2</specification>
+    </description>
+    <preferences>
+        <version>1.2.3</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>

test/data/compose_tests/new/differs.txt

@@ -0,0 +1 @@
+bar

test/data/compose_tests/new/same.txt

@@ -0,0 +1 @@
+foo

test/data/compose_tests/old/config.kiwi

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+some comment
+-->
+
+<!-- some other comment -->
+
+<!--
+Some long text
+and more text-->
+
+<image schemaversion="7.3" name="KEG">
+    <description type="system">
+        <author>Public Cloud Team</author>
+        <contact>[email protected]</contact>
+        <specification>KEG generated image</specification>
+    </description>
+    <preferences>
+        <version>1.2.3</version>
+        <packagemanager>zypper</packagemanager>
+        <locale>en_US</locale>
+        <keytable>us.map.gz</keytable>
+        <timezone>UTC</timezone>
+        <type image="oem" filesystem="btrfs" firmware="efi">
+            <oemconfig>
+                <oem-multipath-scan>false</oem-multipath-scan>
+                <oem-swap>true</oem-swap>
+            </oemconfig>
+        </type>
+    </preferences>
+    <repository>
+        <source path="obs://path/to/project"/>
+    </repository>
+    <packages type="image">
+        <!-- some inline comment -->
+        <package name="joe"/>
+    </packages>
+    <packages type="bootstrap">
+        <package name="udev"/>
+        <package name="filesystem"/>
+        <package name="glibc-locale"/>
+    </packages>
+</image>