Add skip-files-fetch option

It's useful to fetch just git repos, but not distfiles. It's useful when
fetching all repositories, but fetching actual files only when building.
Especially saves a lot of space from kernel tarballs.

And also, useful when using builder just to verify sources.

QubesOS/qubes-issues#9662

README.md

@@ -687,6 +687,8 @@ Options available in `builder.yml`:
 
 - `skip-git-fetch: bool` --- When set, do not update already downloaded git repositories (those in `sources` artifacts dir). New components are still fetched (once). Useful when doing development builds from non-default branches, local modifications etc.
 
+- `skip-files-fetch: bool` --- When set, do not fetch component files like source tarballs (those in the `distfiles` artifacts dir). Component builds *will fail* without those files. Useful to save time and space when fetching git repositories.
+
 - `increment-devel-versions: bool`  --- When set, each package built will have local build number appended to package release number. This way, it's easy to update test environment without manually forcing reinstall of packages with unchanged versions. Example versions with devel build number:
 
   - `qubes-core-dom0-4.2.12-1.13.fc37.x86_64` (`.13` is the additional version here)

qubesbuilder/plugins/fetch/__init__.py

@@ -194,14 +194,17 @@ def run(self, stage: str):
         distfiles_dir.mkdir(parents=True, exist_ok=True)
 
         # Download and verify files given in .qubesbuilder
-        for file in parameters.get("files", []):
-            if "url" in file:
-                self.download_file(file, executor, distfiles_dir)
-            elif "git-url" in file:
-                self.download_git_archive(file, executor, distfiles_dir)
-            else:
-                msg = "'files' entries must have either url or git-url entry"
-                raise FetchError(msg)
+        if not self.config.get("skip-files-fetch", False):
+            for file in parameters.get("files", []):
+                if "url" in file:
+                    self.download_file(file, executor, distfiles_dir)
+                elif "git-url" in file:
+                    self.download_git_archive(file, executor, distfiles_dir)
+                else:
+                    msg = (
+                        "'files' entries must have either url or git-url entry"
+                    )
+                    raise FetchError(msg)
 
         #
         # source hash and version tags determination

tests/test_cli.py

@@ -19,8 +19,7 @@
 HASH_RE = re.compile(r"[a-f0-9]{40}")
 
 
-@pytest.fixture(scope="session")
-def artifacts_dir():
+def _artifacts_dir():
     if os.environ.get("BASE_ARTIFACTS_DIR"):
         tmpdir = tempfile.mktemp(
             prefix="github-", dir=os.environ.get("BASE_ARTIFACTS_DIR")
@@ -33,6 +32,16 @@ def artifacts_dir():
     yield artifacts_dir
 
 
+@pytest.fixture
+def artifacts_dir_single():
+    yield from _artifacts_dir()
+
+
+@pytest.fixture(scope="session")
+def artifacts_dir():
+    yield from _artifacts_dir()
+
+
 def qb_call(builder_conf, artifacts_dir, *args, **kwargs):
     cmd = [
         str(PROJECT_PATH / "qb"),
@@ -316,6 +325,37 @@ def test_common_component_fetch_inplace_updating(artifacts_dir):
         )
 
 
+def test_common_component_fetch_skip_files(artifacts_dir_single):
+    artifacts_dir = artifacts_dir_single
+
+    result = qb_call_output(
+        DEFAULT_BUILDER_CONF,
+        artifacts_dir,
+        "--option",
+        "skip-files-fetch=true",
+        "package",
+        "fetch",
+    ).decode()
+
+    infos = _get_infos(artifacts_dir)
+
+    for component in [
+        "core-qrexec",
+        "core-vchan-xen",
+        "desktop-linux-xfce4-xfwm4",
+        "python-qasync",
+        "app-linux-split-gpg",
+    ]:
+        assert (
+            artifacts_dir / "sources" / component / ".qubesbuilder"
+        ).exists()
+        assert not list((artifacts_dir / "distfiles" / component).iterdir())
+    assert (
+        "Enough distinct tag signatures. Found 3, mandatory minimum is 3."
+        in result
+    )
+
+
 #
 # Pipeline for core-qrexec and host-fc37
 #