From 88636d8c207bb349f55c8edb7b4fddb16dea0de0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 20 Dec 2024 15:48:35 +0100
Subject: [PATCH] Add skip-files-fetch option

It's useful to fetch just git repos, but not distfiles. It's useful when
fetching all repositories, but fetching actual files only when building.
Especially saves a lot of space from kernel tarballs.

And also, useful when using builder just to verify sources.

QubesOS/qubes-issues#9662
---
 README.md                              |  2 ++
 qubesbuilder/plugins/fetch/__init__.py | 19 ++++++-----
 tests/test_cli.py                      | 44 ++++++++++++++++++++++++--
 3 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 0794776e..95356d12 100644
--- a/README.md
+++ b/README.md
@@ -687,6 +687,8 @@ Options available in `builder.yml`:
 
 - `skip-git-fetch: bool` --- When set, do not update already downloaded git repositories (those in `sources` artifacts dir). New components are still fetched (once). Useful when doing development builds from non-default branches, local modifications etc.
 
+- `skip-files-fetch: bool` --- When set, do not fetch component files like source tarballs (those in the `distfiles` artifacts dir). Component builds *will fail* without those files. Useful to save time and space when fetching git repositories.
+
 - `increment-devel-versions: bool`  --- When set, each package built will have local build number appended to package release number. This way, it's easy to update test environment without manually forcing reinstall of packages with unchanged versions. Example versions with devel build number:
 
   - `qubes-core-dom0-4.2.12-1.13.fc37.x86_64` (`.13` is the additional version here)
diff --git a/qubesbuilder/plugins/fetch/__init__.py b/qubesbuilder/plugins/fetch/__init__.py
index 3d70343e..00b8247f 100644
--- a/qubesbuilder/plugins/fetch/__init__.py
+++ b/qubesbuilder/plugins/fetch/__init__.py
@@ -194,14 +194,17 @@ def run(self, stage: str):
         distfiles_dir.mkdir(parents=True, exist_ok=True)
 
         # Download and verify files given in .qubesbuilder
-        for file in parameters.get("files", []):
-            if "url" in file:
-                self.download_file(file, executor, distfiles_dir)
-            elif "git-url" in file:
-                self.download_git_archive(file, executor, distfiles_dir)
-            else:
-                msg = "'files' entries must have either url or git-url entry"
-                raise FetchError(msg)
+        if not self.config.get("skip-files-fetch", False):
+            for file in parameters.get("files", []):
+                if "url" in file:
+                    self.download_file(file, executor, distfiles_dir)
+                elif "git-url" in file:
+                    self.download_git_archive(file, executor, distfiles_dir)
+                else:
+                    msg = (
+                        "'files' entries must have either url or git-url entry"
+                    )
+                    raise FetchError(msg)
 
         #
         # source hash and version tags determination
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d950965d..3fbf4531 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -19,8 +19,7 @@
 HASH_RE = re.compile(r"[a-f0-9]{40}")
 
 
-@pytest.fixture(scope="session")
-def artifacts_dir():
+def _artifacts_dir():
     if os.environ.get("BASE_ARTIFACTS_DIR"):
         tmpdir = tempfile.mktemp(
             prefix="github-", dir=os.environ.get("BASE_ARTIFACTS_DIR")
@@ -33,6 +32,16 @@ def artifacts_dir():
     yield artifacts_dir
 
 
+@pytest.fixture
+def artifacts_dir_single():
+    yield from _artifacts_dir()
+
+
+@pytest.fixture(scope="session")
+def artifacts_dir():
+    yield from _artifacts_dir()
+
+
 def qb_call(builder_conf, artifacts_dir, *args, **kwargs):
     cmd = [
         str(PROJECT_PATH / "qb"),
@@ -316,6 +325,37 @@ def test_common_component_fetch_inplace_updating(artifacts_dir):
         )
 
 
+def test_common_component_fetch_skip_files(artifacts_dir_single):
+    artifacts_dir = artifacts_dir_single
+
+    result = qb_call_output(
+        DEFAULT_BUILDER_CONF,
+        artifacts_dir,
+        "--option",
+        "skip-files-fetch=true",
+        "package",
+        "fetch",
+    ).decode()
+
+    infos = _get_infos(artifacts_dir)
+
+    for component in [
+        "core-qrexec",
+        "core-vchan-xen",
+        "desktop-linux-xfce4-xfwm4",
+        "python-qasync",
+        "app-linux-split-gpg",
+    ]:
+        assert (
+            artifacts_dir / "sources" / component / ".qubesbuilder"
+        ).exists()
+        assert not list((artifacts_dir / "distfiles" / component).iterdir())
+    assert (
+        "Enough distinct tag signatures. Found 3, mandatory minimum is 3."
+        in result
+    )
+
+
 #
 # Pipeline for core-qrexec and host-fc37
 #