From 04dcae465b9cfd81a84fcaa51833171eae20194b Mon Sep 17 00:00:00 2001 From: Joachim Gleissner Date: Thu, 26 Oct 2023 14:42:55 +0100 Subject: [PATCH] compose_kiwi_description: check for changed files Check generated files for changes compared to existing ones and delete unchanged files. Add option to check existing image description for stale files (i.e. files not generated by the new image definition), and remove those (enabled by default). Do not abort anymore if the generated change log has no new entries, but base decision to abort or proceed on result of the above checks. --- kiwi_keg/tools/compose_kiwi_description.py | 127 +++++++- obs/compose_kiwi_description.service | 6 + test/data/compose_tests/new/config.kiwi | 44 +++ test/data/compose_tests/new/config2.kiwi | 44 +++ test/data/compose_tests/new/config3.kiwi | 44 +++ test/data/compose_tests/new/differs.tar.gz | Bin 0 -> 740 bytes test/data/compose_tests/new/differs.txt | 1 + test/data/compose_tests/new/same.tar.gz | Bin 0 -> 737 bytes test/data/compose_tests/new/same.txt | 1 + test/data/compose_tests/old/config.kiwi | 44 +++ test/data/compose_tests/old/config2.kiwi | 44 +++ test/data/compose_tests/old/config3.kiwi | 44 +++ test/data/compose_tests/old/differs.tar.gz | Bin 0 -> 728 bytes test/data/compose_tests/old/differs.txt | 1 + test/data/compose_tests/old/same.tar.gz | Bin 0 -> 728 bytes test/data/compose_tests/old/same.txt | 1 + .../tools/compose_kiwi_description_test.py | 289 +++++++++++++++++- 17 files changed, 670 insertions(+), 20 deletions(-) create mode 100644 test/data/compose_tests/new/config.kiwi create mode 100644 test/data/compose_tests/new/config2.kiwi create mode 100644 test/data/compose_tests/new/config3.kiwi create mode 100644 test/data/compose_tests/new/differs.tar.gz create mode 100644 test/data/compose_tests/new/differs.txt create mode 100644 test/data/compose_tests/new/same.tar.gz create mode 100644 test/data/compose_tests/new/same.txt create mode 100644 test/data/compose_tests/old/config.kiwi create mode 100644 test/data/compose_tests/old/config2.kiwi create mode 100644 test/data/compose_tests/old/config3.kiwi create mode 100644 test/data/compose_tests/old/differs.tar.gz create mode 100644 test/data/compose_tests/old/differs.txt create mode 100644 test/data/compose_tests/old/same.tar.gz create mode 100644 test/data/compose_tests/old/same.txt diff --git a/kiwi_keg/tools/compose_kiwi_description.py b/kiwi_keg/tools/compose_kiwi_description.py index 7f20b47..41af217 100644 --- a/kiwi_keg/tools/compose_kiwi_description.py +++ b/kiwi_keg/tools/compose_kiwi_description.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 SUSE Software Solutions Germany GmbH. All rights reserved. +# Copyright (c) 2023 SUSE Software Solutions Germany GmbH. All rights reserved. # # This file is part of keg. # @@ -28,6 +28,8 @@ [--generate-multibuild=] [--new-image-change=] [--changelog-format=] + [--purge-stale-files=] + [--purge-ignore=] compose_kiwi_description -h | --help compose_kiwi_description --version @@ -86,21 +88,32 @@ converted if necessary. Conversion from 'osc' is not supported. [default: json] + --purge-stale-files= + Purge files from existing image description if the generated image + description does not contain them. [default: true] + + --purge-ignore= + When checking for old files to purge, ignore files matching + (optional). [default: ''] """ import docopt +import filecmp import glob +import hashlib import itertools import json import logging import os import pathlib +import re import subprocess import sys import tempfile import yaml import xml.etree.ElementTree as ET +from difflib import Differ from datetime import datetime, timezone from kiwi_keg.version import __version__ @@ -317,6 +330,90 @@ def write_changelog(log_file, log_format, changes, append=False): json.dump(changes, outf, indent=2, default=str) +def kiwi_files_equivalent(old_kiwi, new_kiwi, ignore_version_change): + + with open(old_kiwi, 'r') as fh: + old_kiwi_content = fh.readlines() + with open(new_kiwi, 'r') as fh: + new_kiwi_content = fh.readlines() + + differ = Differ() + diff = differ.compare(old_kiwi_content, new_kiwi_content) + ignore_str = 'generated by keg on' + if ignore_version_change: + ignore_str += r'|[0123456789\.]+' + ignore_regex = re.compile(ignore_str) + + for d in diff: + if d[0] == '-' and not ignore_regex.search(d): + return False + + return True + + +def tar_files_equivalent(file1, file2): + result = subprocess.run( + ['tar', 'xf', file1, '-O'], + stdout=subprocess.PIPE + ) + sum1 = hashlib.sha256(result.stdout).digest() + result = subprocess.run( + ['tar', 'xf', file2, '-O'], + stdout=subprocess.PIPE + ) + sum2 = hashlib.sha256(result.stdout).digest() + return sum1 == sum2 + + +def files_equivalent(filename, dir1, dir2, ignore_kiwi_version): + path1 = os.path.join(dir1, filename) + path2 = os.path.join(dir2, filename) + + if not os.path.exists(path1) or not os.path.exists(path2): + return False + + if filename.endswith('.kiwi'): + return kiwi_files_equivalent(path1, path2, ignore_kiwi_version) + + if 'tar' in filename.split('.'): + return tar_files_equivalent(path1, path2) + + return filecmp.cmp(path1, path2, shallow=False) + + +def delete_unchanged_files(old_dir, new_dir, ignore_kiwi_version): + new_files = list(os.scandir(new_dir)) + + have_changes = False + ignore_regex = re.compile('log_sources.*') + for f in new_files: + if not ignore_regex.match(f.name): + if files_equivalent(f.name, old_dir, new_dir, ignore_kiwi_version): + if f.name != 'config.kiwi': + log.info('Deleting unchanged file {}'.format(f.name)) + os.remove(f.path) + else: + have_changes = True + + return have_changes + + +def get_stale_files(old_dir, new_dir, ignore_exp): + old_files = list(os.scandir(old_dir)) + stale_files = [] + + purge_ignore = '_service|_keg_revisions' + if ignore_exp: + purge_ignore += '|' + ignore_exp + ignore_regex = re.compile(purge_ignore) + for f in old_files: + if f.is_file() and not ignore_regex.match(f.name): + if not os.path.exists(os.path.join(new_dir, f.name)): + stale_files.append(f.name) + + return stale_files + + def main() -> None: args = docopt.docopt(__doc__, version=__version__) @@ -390,9 +487,24 @@ def main() -> None: image_generator.create_custom_files( overwrite=True ) - if args['--generate-multibuild']: + if args['--generate-multibuild'] == 'true': image_generator.create_multibuild_file(overwrite=True) + stale_files = [] + if args['--purge-stale-files'] == 'true': + stale_files = get_stale_files('.', args['--outdir'], args['--purge-ignore']) + + files_changed = delete_unchanged_files('.', args['--outdir'], args['--version-bump'] == 'true') + if not files_changed: + if stale_files: + log.info('Generated files are identical to existing ones, ' + 'but old image description has stale files.') + else: + log.warning('Generated image description is identical to existing one.') + if args['--force'] != 'true': + log.info('Aborting.') + sys.exit() + if handle_changelog: sig = SourceInfoGenerator(image_definition, dest_dir=args['--outdir']) sig.write_source_info() @@ -427,12 +539,7 @@ def main() -> None: have_changes |= generate_changelog(source_log, changes_path, args['--changelog-format'], image_version, rev_args) if not have_changes: - log.warning('Image has no changes.') - if args['--force'] != 'true': - log.info('Deleting generated files.') - for f in next(os.walk(args['--outdir']))[2]: - os.remove(os.path.join(args['--outdir'], f)) - sys.exit() + log.warning('Image description has changed but no new change log entries were generated.') for source_log, flavor in get_log_sources(os.path.join(args['--outdir'])): changes_filename = f'{flavor}{"." if flavor else ""}changes.{log_ext}' @@ -444,3 +551,7 @@ def main() -> None: if args['--update-revisions'] == 'true': # capture current commits update_revisions(repos, args['--outdir']) + + for f in stale_files: + log.info('Deleting stale file {}'.format(f)) + os.remove(f) diff --git a/obs/compose_kiwi_description.service b/obs/compose_kiwi_description.service index 918a56a..52a4a46 100644 --- a/obs/compose_kiwi_description.service +++ b/obs/compose_kiwi_description.service @@ -40,4 +40,10 @@ Set output format for generated change log. Supported values are 'yaml', 'json', and 'osc'. Existing change logs will be converted if necessary. Conversion from 'osc' is not supported. [default: json] + + Purge files from existing image description if the generated image description does not contain them. [default: true] + + + Regular expression. When checking for old files to purge, ignore matching files. (optional) + diff --git a/test/data/compose_tests/new/config.kiwi b/test/data/compose_tests/new/config.kiwi new file mode 100644 index 0000000..0b14e34 --- /dev/null +++ b/test/data/compose_tests/new/config.kiwi @@ -0,0 +1,44 @@ + + + + + + + + + + + Public Cloud Team + pubcloud-dev@suse.com + KEG generated image + + + 1.2.3 + zypper + en_US + us.map.gz + UTC + + + false + true + + + + + + + + + + + + + + + + diff --git a/test/data/compose_tests/new/config2.kiwi b/test/data/compose_tests/new/config2.kiwi new file mode 100644 index 0000000..558e63a --- /dev/null +++ b/test/data/compose_tests/new/config2.kiwi @@ -0,0 +1,44 @@ + + + + + + + + + + + Public Cloud Team + pubcloud-dev@suse.com + KEG generated image + + + 1.2.4 + zypper + en_US + us.map.gz + UTC + + + false + true + + + + + + + + + + + + + + + + diff --git a/test/data/compose_tests/new/config3.kiwi b/test/data/compose_tests/new/config3.kiwi new file mode 100644 index 0000000..933cdb2 --- /dev/null +++ b/test/data/compose_tests/new/config3.kiwi @@ -0,0 +1,44 @@ + + + + + + + + + + + Public Cloud Team + pubcloud-dev@suse.com + KEG generated image2 + + + 1.2.3 + zypper + en_US + us.map.gz + UTC + + + false + true + + + + + + + + + + + + + + + + diff --git a/test/data/compose_tests/new/differs.tar.gz b/test/data/compose_tests/new/differs.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a1df8b73fe7bbba2bbed5f8d78363d6b14b2198c GIT binary patch literal 740 zcmV$NS8u@OTFY0%axC=MoJh@<#y6ofI#Wb=kGyhx=DQy9v7eCr=%JR-RfIn08|#Wak;L;z214rs)Jt}FL%!nY_;UMCNG>;dMjkOYA&Z$C_#>A< z!R0~}tC?Gwr8jjiQ^!d@c%GwW4us1*hr&3X*G`jWTADSeP9%HLxv9lW7As<4V;oj2 zNG=sLqn(w=*$Rl}YslF?kaK_Rl7ivo;(q)5#Z4W8Bn!}7iPE46G20SFz10TtR&Q)r zW!6%q*VQ}|oLptHT98{{c|u#IH^swm7&mEI&3SF-E#UE!#!CB`M?yQfU!Y6iVktPQ zBS`F6p=SD4cJ*Dy|rqBYGVo<*|dVyt+Ny&5jPw_P*Ohk%_E(HnnV55LcNR~Dme3~e)c%98SS%Kc#bU8oEEbE!VzF2( W7K_DVu~;mY-{1!$yUC&eC;$LVq-$yb literal 0 HcmV?d00001 diff --git a/test/data/compose_tests/new/differs.txt b/test/data/compose_tests/new/differs.txt new file mode 100644 index 0000000..5716ca5 --- /dev/null +++ b/test/data/compose_tests/new/differs.txt @@ -0,0 +1 @@ +bar diff --git a/test/data/compose_tests/new/same.tar.gz b/test/data/compose_tests/new/same.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..83ead5475b6a5033d9f2d25802530f6caea202ba GIT binary patch literal 737 zcmV<70v`PziwFP!000001MQYeZ`&{ofOGb*(0sS$SDbWk+yTX~0lO4fdma>pcGTLI zA<1phUq8iN_6t&-) z({IlYgNYwa{Wu81IEW{nA4FP&Hg6fkOEgjup+k9(v;ILs50#3Qu<+xHc@RZYFL1qh z8qXu|BCxcU9BKX74d)lZbRKqE`fTqk2&dkB7Fd5fGsCa&herO0gZcMo({Aql@jcK} zADzs96noJGy~|7v^M98Yz@)4|ykw_e$XEQFK5qXB$>pXj(1Xs6@oHiFu4f~tC||KE zU)YVxoVk6OT2}JGaV*J8K$MpyRLXLkUW{5XS8EW1677}N_norfRgM&Fltro)D!G7m z)mgEUQnwXL;MZl!{<*S9vLQEM*J z>t0JZ zx(1Qp6Y*$Ya7qeDq55(AJ;D8IRNuo^k);+j(k)5t=1&RU%SN-^BmcxJNN;bh61-m> z80cDb7qGCoE(gk30bOYp_EL#V?nI(|Btlz2#tsVBI?$WCm*`kN=$x`C6sw6^JCc$r z%}62jU7yjM-!30Xohs4vd)no30QgJ + + + + + + + + + + Public Cloud Team + pubcloud-dev@suse.com + KEG generated image + + + 1.2.3 + zypper + en_US + us.map.gz + UTC + + + false + true + + + + + + + + + + + + + + + + diff --git a/test/data/compose_tests/old/config2.kiwi b/test/data/compose_tests/old/config2.kiwi new file mode 100644 index 0000000..0b14e34 --- /dev/null +++ b/test/data/compose_tests/old/config2.kiwi @@ -0,0 +1,44 @@ + + + + + + + + + + + Public Cloud Team + pubcloud-dev@suse.com + KEG generated image + + + 1.2.3 + zypper + en_US + us.map.gz + UTC + + + false + true + + + + + + + + + + + + + + + + diff --git a/test/data/compose_tests/old/config3.kiwi b/test/data/compose_tests/old/config3.kiwi new file mode 100644 index 0000000..0b14e34 --- /dev/null +++ b/test/data/compose_tests/old/config3.kiwi @@ -0,0 +1,44 @@ + + + + + + + + + + + Public Cloud Team + pubcloud-dev@suse.com + KEG generated image + + + 1.2.3 + zypper + en_US + us.map.gz + UTC + + + false + true + + + + + + + + + + + + + + + + diff --git a/test/data/compose_tests/old/differs.tar.gz b/test/data/compose_tests/old/differs.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..0ab8e232ae116105ee0ab971adaf5f9599a437db GIT binary patch literal 728 zcmV;}0w?_+iwFP!000001MQYkZ<{a>fcwm^@OTFYLeq32bW$~K(q1aHeV!cWAZ`I8 z+q6l4{SE>l$&zZ5x^9#FJtXnn`F!WQv*Obmw)zTe0ZLOY%T#3kqj(l)Z^#IlL{a;V z9Da|2ad;L)=fNZl$E{ruMkpeyw+!MGwKhy)DDQFBKgf7Q3L^@b1(VBZ7&TS!$J5L3 zd^)}i9n`WtZ5a7c7+!>v>15hy`RToXJ2T6#@P|fzkDd7sF3vY|9}MpSPJMJR|IuU= zosoB$$$tJHvK)lgC8$$&_(Hzo=kRg+Pl&HpMNXbDHzCWJ8~7uaK*{Aol-bO!P3leE ztHg2Q51!{}Spea(D4;Zs=XGMzilteC>Xl@xbZ#4EF3XGV;35^AH4!8j7|8%Th8b9p zHjS9>n{2|SJ8^fSNb7$^G4ROt-5Y1FXg70%oqn za*$Nyuqn;Vog0VpQGjX_xx}&?6CGs+d<_!=EY^E2yLvMymC0i^y7WAWcMg7jylQ(&jfc5i!d3D#^?TzmC_IBDtnTrzk#_h02y6Jh4FdW$XKYrX?N~4Wpm3z`#jT6$l zJE9)S{)$nSi#hi;y?89Nn6`N*_8)}BVzF2(7K_DVu~;k?i^XEGSS%Kc#bUAi20s8v KVHWiOC;$KmsbCBM literal 0 HcmV?d00001 diff --git a/test/data/compose_tests/old/differs.txt b/test/data/compose_tests/old/differs.txt new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/test/data/compose_tests/old/differs.txt @@ -0,0 +1 @@ +foo diff --git a/test/data/compose_tests/old/same.tar.gz b/test/data/compose_tests/old/same.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..0ab8e232ae116105ee0ab971adaf5f9599a437db GIT binary patch literal 728 zcmV;}0w?_+iwFP!000001MQYkZ<{a>fcwm^@OTFYLeq32bW$~K(q1aHeV!cWAZ`I8 z+q6l4{SE>l$&zZ5x^9#FJtXnn`F!WQv*Obmw)zTe0ZLOY%T#3kqj(l)Z^#IlL{a;V z9Da|2ad;L)=fNZl$E{ruMkpeyw+!MGwKhy)DDQFBKgf7Q3L^@b1(VBZ7&TS!$J5L3 zd^)}i9n`WtZ5a7c7+!>v>15hy`RToXJ2T6#@P|fzkDd7sF3vY|9}MpSPJMJR|IuU= zosoB$$$tJHvK)lgC8$$&_(Hzo=kRg+Pl&HpMNXbDHzCWJ8~7uaK*{Aol-bO!P3leE ztHg2Q51!{}Spea(D4;Zs=XGMzilteC>Xl@xbZ#4EF3XGV;35^AH4!8j7|8%Th8b9p zHjS9>n{2|SJ8^fSNb7$^G4ROt-5Y1FXg70%oqn za*$Nyuqn;Vog0VpQGjX_xx}&?6CGs+d<_!=EY^E2yLvMymC0i^y7WAWcMg7jylQ(&jfc5i!d3D#^?TzmC_IBDtnTrzk#_h02y6Jh4FdW$XKYrX?N~4Wpm3z`#jT6$l zJE9)S{)$nSi#hi;y?89Nn6`N*_8)}BVzF2(7K_DVu~;k?i^XEGSS%Kc#bUAi20s8v KVHWiOC;$KmsbCBM literal 0 HcmV?d00001 diff --git a/test/data/compose_tests/old/same.txt b/test/data/compose_tests/old/same.txt new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/test/data/compose_tests/old/same.txt @@ -0,0 +1 @@ +foo diff --git a/test/unit/tools/compose_kiwi_description_test.py b/test/unit/tools/compose_kiwi_description_test.py index f2e2c34..07b7d1c 100644 --- a/test/unit/tools/compose_kiwi_description_test.py +++ b/test/unit/tools/compose_kiwi_description_test.py @@ -17,9 +17,11 @@ parse_revisions, update_revisions, update_changelog, - RepoInfo + RepoInfo, + files_equivalent ) +test_data = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../data') fake_changes_txt_new = '''------------------------------------------------------------------- Fri May 12 16:37:47 2023 UTC @@ -42,6 +44,15 @@ ''' +class MockDirEntry: + def __init__(self, name, path): + self.name = name + self.path = path + + def is_file(self): + return True + + class TestFetchFromKeg: @fixture(autouse=True) def inject_fixtures(self, caplog): @@ -62,9 +73,11 @@ def setup(self): 'obs_out' ] + @patch('kiwi_keg.tools.compose_kiwi_description.files_equivalent') + @patch('kiwi_keg.tools.compose_kiwi_description.os.scandir') @patch('kiwi_keg.tools.compose_kiwi_description.update_changelog') @patch('kiwi_keg.tools.compose_kiwi_description.update_revisions') - @patch('os.remove') + @patch('kiwi_keg.tools.compose_kiwi_description.os.remove') @patch('kiwi_keg.tools.compose_kiwi_description.get_revision_args') @patch('glob.glob') @patch('kiwi_keg.tools.compose_kiwi_description.SourceInfoGenerator') @@ -80,7 +93,7 @@ def test_compose_kiwi_description( mock_KegGenerator, mock_KegImageDefinition, mock_TemporaryDirectory, mock_ET, mock_SourceInfoGenerator, mock_glob, mock_get_revision_args, mock_remove, mock_update_revisions, - mock_update_changelog + mock_update_changelog, mock_scandir, mock_files_equivalent ): mock_tree = Mock() mock_root = Mock() @@ -91,7 +104,7 @@ def test_compose_kiwi_description( mock_root.findall.return_value = [mock_prefs] mock_prefs.find.return_value = mock_ver mock_ver.text = '1.1.1' - mock_path_exists.side_effect = [False, False, True, True, True, False] + mock_path_exists.side_effect = [False, False, True, False] image_definition = Mock() mock_KegImageDefinition.return_value = image_definition image_generator = Mock() @@ -105,6 +118,8 @@ def test_compose_kiwi_description( mock_result = Mock() mock_result.returncode = 0 mock_subprocess_run.return_value = mock_result + mock_scandir.return_value = [MockDirEntry('stale_file', './stale_file')] + mock_files_equivalent.return_value = False with patch('builtins.open', create=True): main() @@ -183,7 +198,8 @@ def test_compose_kiwi_description( ) assert mock_remove.call_args_list == [ call('obs_out/log_sources_flavor1'), - call('obs_out/log_sources_flavor2') + call('obs_out/log_sources_flavor2'), + call('stale_file'), ] source_info_generator.write_source_info.assert_called_once() mock_update_revisions.assert_called_once() @@ -192,6 +208,8 @@ def test_compose_kiwi_description( call('obs_out/flavor2.changes.json', 'json') ] + @patch('kiwi_keg.tools.compose_kiwi_description.files_equivalent') + @patch('kiwi_keg.tools.compose_kiwi_description.os.scandir') @patch('kiwi_keg.tools.compose_kiwi_description.update_revisions') @patch('os.walk') @patch('os.remove') @@ -205,11 +223,12 @@ def test_compose_kiwi_description( @patch('kiwi_keg.tools.compose_kiwi_description.subprocess.run') @patch('os.mkdir') @patch('os.path.exists') - def test_compose_kiwi_description_no_version_bump( + def test_compose_kiwi_description_no_changes( self, mock_path_exists, mock_mkdir, mock_subprocess_run, mock_KegGenerator, mock_KegImageDefinition, mock_TemporaryDirectory, mock_ET, mock_SourceInfoGenerator, mock_glob, - mock_get_revision_args, mock_remove, mock_walk, mock_update_revisions + mock_get_revision_args, mock_remove, mock_walk, mock_update_revisions, + mock_scandir, mock_files_equivalent ): sys.argv = [ sys.argv[0], @@ -222,6 +241,8 @@ def test_compose_kiwi_description_no_version_bump( '--outdir', 'obs_out', '--version-bump=false', + '--purge-stale-files', + 'false', '--changelog-format=yaml' ] mock_tree = Mock() @@ -233,7 +254,7 @@ def test_compose_kiwi_description_no_version_bump( mock_root.findall.return_value = [mock_prefs] mock_prefs.find.return_value = mock_ver mock_ver.text = '1.1.1' - mock_path_exists.side_effect = [False, True, True, True] + mock_path_exists.side_effect = [False, False, True] image_definition = Mock() mock_KegImageDefinition.return_value = image_definition image_generator = Mock() @@ -248,11 +269,234 @@ def test_compose_kiwi_description_no_version_bump( mock_result.returncode = 2 mock_subprocess_run.return_value = mock_result mock_walk.return_value = iter([('obs_out', [], ['config.kiwi'])]) + mock_scandir.return_value = [MockDirEntry('unchanged_file', 'obs_out/unchanged_file')] + mock_files_equivalent.return_value = True with patch('builtins.open', create=True), raises(SystemExit), self._caplog.at_level(logging.WARNING): main() - assert 'Image has no changes.' in self._caplog.text + assert 'Generated image description is identical to existing one' in self._caplog.text + + mock_mkdir.assert_called_once_with('obs_out') + assert mock_subprocess_run.call_args_list == [ + call( + [ + 'git', 'clone', '-b', 'develop', + 'https://github.com/SUSE-Enceladus/keg-recipes.git', + temp_dir.name + ] + ), + call( + [ + 'git', '-C', temp_dir.name, + 'show', '--no-patch', '--format=%H', 'HEAD' + ], + stdout=subprocess.PIPE, encoding='UTF-8' + ) + ] + mock_KegImageDefinition.assert_called_once_with( + image_name='leap/jeos/15.2', + recipes_roots=[temp_dir.name], + track_sources=True, + image_version=None + ) + mock_KegGenerator.assert_called_once_with( + image_definition=image_definition, dest_dir='obs_out', archs=[] + ) + image_generator.create_kiwi_description.assert_called_once_with( + overwrite=True + ) + image_generator.create_custom_scripts.assert_called_once_with( + overwrite=True + ) + image_generator.create_overlays.assert_called_once_with( + disable_root_tar=False, overwrite=True + ) + mock_remove.assert_called_with('obs_out/unchanged_file') + + @patch('kiwi_keg.tools.compose_kiwi_description.files_equivalent') + @patch('kiwi_keg.tools.compose_kiwi_description.os.scandir') + @patch('kiwi_keg.tools.compose_kiwi_description.update_revisions') + @patch('os.walk') + @patch('os.remove') + @patch('kiwi_keg.tools.compose_kiwi_description.get_revision_args') + @patch('glob.glob') + @patch('kiwi_keg.tools.compose_kiwi_description.SourceInfoGenerator') + @patch('kiwi_keg.tools.compose_kiwi_description.ET') + @patch('tempfile.TemporaryDirectory') + @patch('kiwi_keg.tools.compose_kiwi_description.KegImageDefinition') + @patch('kiwi_keg.tools.compose_kiwi_description.KegGenerator') + @patch('kiwi_keg.tools.compose_kiwi_description.subprocess.run') + @patch('os.mkdir') + @patch('os.path.exists') + def test_compose_kiwi_description_no_changes_stale_files( + self, mock_path_exists, mock_mkdir, mock_subprocess_run, + mock_KegGenerator, mock_KegImageDefinition, mock_TemporaryDirectory, + mock_ET, mock_SourceInfoGenerator, mock_glob, + mock_get_revision_args, mock_remove, mock_walk, mock_update_revisions, + mock_scandir, mock_files_equivalent + ): + sys.argv = [ + sys.argv[0], + '--git-recipes', + 'https://github.com/SUSE-Enceladus/keg-recipes.git', + '--image-source', + 'leap/jeos/15.2', + '--git-branch', + 'develop', + '--outdir', + 'obs_out', + '--version-bump=false', + '--purge-stale-files', + 'true', + '--changelog-format=yaml' + ] + mock_tree = Mock() + mock_root = Mock() + mock_prefs = Mock() + mock_ver = Mock() + mock_ET.parse.return_value = mock_tree + mock_tree.getroot.return_value = mock_root + mock_root.findall.return_value = [mock_prefs] + mock_prefs.find.return_value = mock_ver + mock_ver.text = '1.1.1' + mock_path_exists.side_effect = [False, False, True, False] + image_definition = Mock() + mock_KegImageDefinition.return_value = image_definition + image_generator = Mock() + mock_KegGenerator.return_value = image_generator + temp_dir = Mock() + mock_TemporaryDirectory.return_value = temp_dir + source_info_generator = Mock() + mock_SourceInfoGenerator.return_value = source_info_generator + mock_glob.return_value = ['obs_out/log_sources'] + mock_get_revision_args.return_value = ['-r', 'fake_repo:fake_rev..'] + mock_result = Mock() + mock_result.returncode = 2 + mock_subprocess_run.return_value = mock_result + mock_walk.return_value = iter([('obs_out', [], ['config.kiwi'])]) + mock_scandir.return_value = [MockDirEntry('stale_file', 'stale_file')] + mock_files_equivalent.return_value = True + + with patch('builtins.open', create=True), self._caplog.at_level(logging.INFO): + main() + + assert 'Generated files are identical to existing ones, but old image description has stale files' in self._caplog.text + + mock_mkdir.assert_called_once_with('obs_out') + assert mock_subprocess_run.call_args_list == [ + call( + [ + 'git', 'clone', '-b', 'develop', + 'https://github.com/SUSE-Enceladus/keg-recipes.git', + temp_dir.name + ] + ), + call( + [ + 'git', '-C', temp_dir.name, + 'show', '--no-patch', '--format=%H', 'HEAD' + ], + stdout=subprocess.PIPE, encoding='UTF-8' + ), + call( + [ + 'generate_recipes_changelog', + '-o', 'obs_out/changes.yaml', + '-f', 'yaml', + '-t', '1.1.1', + '-r', 'fake_repo:fake_rev..', + 'obs_out/log_sources' + ] + ) + ] + mock_KegImageDefinition.assert_called_once_with( + image_name='leap/jeos/15.2', + recipes_roots=[temp_dir.name], + track_sources=True, + image_version=None + ) + mock_KegGenerator.assert_called_once_with( + image_definition=image_definition, dest_dir='obs_out', archs=[] + ) + image_generator.create_kiwi_description.assert_called_once_with( + overwrite=True + ) + image_generator.create_custom_scripts.assert_called_once_with( + overwrite=True + ) + image_generator.create_overlays.assert_called_once_with( + disable_root_tar=False, overwrite=True + ) + mock_remove.assert_called_with('stale_file') + + @patch('kiwi_keg.tools.compose_kiwi_description.delete_unchanged_files') + @patch('kiwi_keg.tools.compose_kiwi_description.os.scandir') + @patch('kiwi_keg.tools.compose_kiwi_description.update_revisions') + @patch('os.walk') + @patch('os.remove') + @patch('kiwi_keg.tools.compose_kiwi_description.get_revision_args') + @patch('glob.glob') + @patch('kiwi_keg.tools.compose_kiwi_description.SourceInfoGenerator') + @patch('kiwi_keg.tools.compose_kiwi_description.ET') + @patch('tempfile.TemporaryDirectory') + @patch('kiwi_keg.tools.compose_kiwi_description.KegImageDefinition') + @patch('kiwi_keg.tools.compose_kiwi_description.KegGenerator') + @patch('kiwi_keg.tools.compose_kiwi_description.subprocess.run') + @patch('os.mkdir') + @patch('os.path.exists') + def test_compose_kiwi_description_no_change_entries( + self, mock_path_exists, mock_mkdir, mock_subprocess_run, + mock_KegGenerator, mock_KegImageDefinition, mock_TemporaryDirectory, + mock_ET, mock_SourceInfoGenerator, mock_glob, + mock_get_revision_args, mock_remove, mock_walk, mock_update_revisions, + mock_scandir, mock_delete_unchanged_files + ): + sys.argv = [ + sys.argv[0], + '--git-recipes', + 'https://github.com/SUSE-Enceladus/keg-recipes.git', + '--image-source', + 'leap/jeos/15.2', + '--git-branch', + 'develop', + '--outdir', + 'obs_out', + '--version-bump=false', + '--purge-stale-files', + 'false', + '--changelog-format=yaml' + ] + mock_tree = Mock() + mock_root = Mock() + mock_prefs = Mock() + mock_ver = Mock() + mock_ET.parse.return_value = mock_tree + mock_tree.getroot.return_value = mock_root + mock_root.findall.return_value = [mock_prefs] + mock_prefs.find.return_value = mock_ver + mock_ver.text = '1.1.1' + mock_path_exists.side_effect = [False, False, True, True, True, True] + image_definition = Mock() + mock_KegImageDefinition.return_value = image_definition + image_generator = Mock() + mock_KegGenerator.return_value = image_generator + temp_dir = Mock() + mock_TemporaryDirectory.return_value = temp_dir + source_info_generator = Mock() + mock_SourceInfoGenerator.return_value = source_info_generator + mock_glob.return_value = ['obs_out/log_sources'] + mock_get_revision_args.return_value = ['-r', 'fake_repo:fake_rev..'] + mock_result = Mock() + mock_result.returncode = 2 + mock_subprocess_run.return_value = mock_result + mock_walk.return_value = iter([('obs_out', [], ['config.kiwi'])]) + mock_delete_unchanged_files.return_value = True + + with patch('builtins.open', create=True), self._caplog.at_level(logging.WARNING): + main() + + assert 'Image description has changed but no new change log entries were generated' in self._caplog.text mock_mkdir.assert_called_once_with('obs_out') assert mock_subprocess_run.call_args_list == [ @@ -303,8 +547,9 @@ def test_compose_kiwi_description_no_version_bump( 'obs_out/config.kiwi' ) source_info_generator.write_source_info.assert_called_once() - mock_remove.assert_called_once_with('obs_out/config.kiwi') + mock_remove.assert_called_with('obs_out/log_sources') + @patch('kiwi_keg.tools.compose_kiwi_description.delete_unchanged_files') @patch('kiwi_keg.tools.compose_kiwi_description.datetime') @patch('kiwi_keg.tools.compose_kiwi_description.write_changelog') @patch('kiwi_keg.tools.compose_kiwi_description.update_revisions') @@ -324,7 +569,7 @@ def test_compose_kiwi_description_new_image( mock_KegGenerator, mock_KegImageDefinition, mock_TemporaryDirectory, mock_ET, mock_SourceInfoGenerator, mock_glob, mock_get_revision_args, mock_remove, mock_update_revisions, - mock_write_changelog, mock_datetime + mock_write_changelog, mock_datetime, mock_delete_unchanged_files ): sys.argv = [ sys.argv[0], @@ -337,6 +582,8 @@ def test_compose_kiwi_description_new_image( '--outdir', 'obs_out', '--version-bump=false', + '--purge-stale-files', + 'false', '--changelog-format=json', '--new-image-change=new image' ] @@ -366,6 +613,7 @@ def test_compose_kiwi_description_new_image( mock_timestamp = Mock() mock_timestamp.isoformat.return_value = 'TIMESTAMP' mock_datetime.now.return_value = mock_timestamp + mock_delete_unchanged_files.return_value = True with patch('builtins.open', create=True): main() @@ -417,6 +665,7 @@ def test_compose_kiwi_description_new_image( expected_changelog ) + @patch('kiwi_keg.tools.compose_kiwi_description.delete_unchanged_files') @patch('kiwi_keg.tools.compose_kiwi_description.update_changelog') @patch('kiwi_keg.tools.compose_kiwi_description.update_revisions') @patch('os.remove') @@ -435,7 +684,7 @@ def test_compose_kiwi_description_osc_log( mock_KegGenerator, mock_KegImageDefinition, mock_TemporaryDirectory, mock_ET, mock_SourceInfoGenerator, mock_glob, mock_get_revision_args, mock_remove, mock_update_revisions, - mock_update_changelog + mock_update_changelog, mock_delete_unchanged_files ): sys.argv = [ sys.argv[0], @@ -447,6 +696,8 @@ def test_compose_kiwi_description_osc_log( 'develop', '--outdir', 'obs_out', + '--purge-stale-files', + 'false', '--changelog-format=osc' ] mock_tree = Mock() @@ -472,6 +723,7 @@ def test_compose_kiwi_description_osc_log( mock_result = Mock() mock_result.returncode = 0 mock_subprocess_run.return_value = mock_result + mock_delete_unchanged_files.return_value = True with patch('builtins.open', create=True): main() @@ -731,3 +983,16 @@ def test_generate_changelog_error(self, mock_subprocess_run): with raises(SystemExit) as sysex: generate_changelog('log_sources', 'changes.yaml', 'yaml', '1.1.1', ['-r', 'fakerev']) assert sysex.value.code == 'Error generating change log.' + + def test_files_equivalent(self): + old_dir = os.path.join(test_data, 'compose_tests/old') + new_dir = os.path.join(test_data, 'compose_tests/new') + assert files_equivalent('config.kiwi', old_dir, new_dir, True) + assert files_equivalent('config2.kiwi', old_dir, new_dir, True) + assert not files_equivalent('config2.kiwi', old_dir, new_dir, False) + assert not files_equivalent('config3.kiwi', old_dir, new_dir, True) + assert files_equivalent('same.tar.gz', old_dir, new_dir, True) + assert not files_equivalent('differs.tar.gz', old_dir, new_dir, True) + assert files_equivalent('same.txt', old_dir, new_dir, True) + assert not files_equivalent('differs.txt', old_dir, new_dir, True) + assert not files_equivalent('no_such_file', old_dir, new_dir, True)