tuftool: Add clone subcommand

This adds a `clone` subcommand to `tuftool`, allowing a user to download
a fully functioning TUF repository.  A user has the option to download a
full repository, a subset of the targets, or just metadata.

tuftool/src/clone.rs

@@ -0,0 +1,144 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+use crate::common::UNUSED_URL;
+use crate::download_root::download_root;
+use crate::error::{self, Result};
+use snafu::ResultExt;
+use std::fs::File;
+use std::num::NonZeroU64;
+use std::path::{Path, PathBuf};
+use structopt::StructOpt;
+use tough::{ExpirationEnforcement, RepositoryLoader};
+use url::Url;
+
+#[derive(Debug, StructOpt)]
+pub(crate) struct CloneArgs {
+    /// Path to root.json file for the repository
+    #[structopt(short = "r", long = "root")]
+    root: Option<PathBuf>,
+
+    /// Remote root.json version number
+    #[structopt(short = "v", long = "root-version", default_value = "1")]
+    root_version: NonZeroU64,
+
+    /// TUF repository metadata base URL
+    #[structopt(short = "m", long = "metadata-url")]
+    metadata_base_url: Url,
+
+    /// TUF repository targets base URL
+    #[structopt(short = "t", long = "targets-url", required_unless = "metadata-only")]
+    targets_base_url: Option<Url>,
+
+    /// Allow downloading the root.json file (unsafe)
+    #[structopt(long)]
+    allow_root_download: bool,
+
+    /// Allow repo download for expired metadata
+    #[structopt(long)]
+    allow_expired_repo: bool,
+
+    /// Download only these targets, if specified
+    #[structopt(short = "n", long = "target-names", conflicts_with = "metadata-only")]
+    target_names: Vec<String>,
+
+    /// Output directory of targets
+    #[structopt(long, required_unless = "metadata-only")]
+    targets_dir: Option<PathBuf>,
+
+    /// Output directory of metadata
+    #[structopt(long)]
+    metadata_dir: PathBuf,
+
+    /// Only download the repository metadata, not the targets
+    #[structopt(long)]
+    metadata_only: bool,
+}
+
+fn expired_repo_warning<P: AsRef<Path>>(path: P) {
+    #[rustfmt::skip]
+    eprintln!("\
+=================================================================
+Downloading repo to {}
+WARNING: `--allow-expired-repo` was passed; this is unsafe and will not establish trust, use only for testing!
+=================================================================",
+              path.as_ref().display());
+}
+
+impl CloneArgs {
+    pub(crate) fn run(&self) -> Result<()> {
+        // Use local root.json or download from repository
+        let root_path = if let Some(path) = &self.root {
+            PathBuf::from(path)
+        } else if self.allow_root_download {
+            let outdir = std::env::current_dir().context(error::CurrentDir)?;
+            download_root(&self.metadata_base_url, self.root_version, outdir)?
+        } else {
+            eprintln!("No root.json available");
+            std::process::exit(1);
+        };
+
+        // Structopt won't allow `targets_base_url` to be None when it is required.  We require the
+        // user to supply `targets_base_url` in the case they actually plan to download targets.
+        // When downloading metadata, we don't ever need to access the targets URL, so we use a
+        // fake URL to satisfy the library.
+        let targets_base_url = self
+            .targets_base_url
+            .as_ref()
+            .unwrap_or(&Url::parse(UNUSED_URL).context(error::UrlParse {
+                url: UNUSED_URL.to_owned(),
+            })?)
+            .clone();
+
+        // Load repository
+        let expiration_enforcement = if self.allow_expired_repo {
+            expired_repo_warning(&self.metadata_dir);
+            ExpirationEnforcement::Unsafe
+        } else {
+            ExpirationEnforcement::Safe
+        };
+        let repository = RepositoryLoader::new(
+            File::open(&root_path).context(error::OpenRoot { path: &root_path })?,
+            self.metadata_base_url.clone(),
+            targets_base_url,
+        )
+        .expiration_enforcement(expiration_enforcement)
+        .load()
+        .context(error::RepoLoad)?;
+
+        // Clone the repository, downloading none, all, or a subset of targets
+        let clone_result = if self.metadata_only {
+            println!("Cloning repository metadata to {:?}", self.metadata_dir);
+            repository.cache_metadata(&self.metadata_dir, true)
+        } else {
+            // Similar to `targets_base_url, structopt's guard rails won't let us have a
+            // `targets_dir` that is None when the argument is required.  We only require the user
+            // to supply a targets directory if they actually plan on downloading targets.
+            let targets_dir = self.targets_dir.as_ref().expect(
+                "Developer error: `targets_dir` is required unless downloading metadata only",
+            );
+
+            if self.target_names.is_empty() {
+                println!(
+                    "Cloning repository:\n\tmetadata location: {:?}\n\ttargets location: {:?}",
+                    self.metadata_dir, targets_dir
+                );
+                // The None turbofish is required to satisfy the compiler, sorry.
+                repository.cache(&self.metadata_dir, &targets_dir, None::<&[&str]>, true)
+            } else {
+                println!(
+                    "Cloning repository with a subset of targets:\n\tmetadata location:{:?}\n\ttargets location: {:?}",
+                    self.metadata_dir, targets_dir
+                );
+                repository.cache(
+                    &self.metadata_dir,
+                    &targets_dir,
+                    Some(self.target_names.as_slice()),
+                    true,
+                )
+            }
+        };
+
+        clone_result.context(error::CloneRepository)
+    }
+}

tuftool/src/error.rs

@@ -22,6 +22,12 @@ fn get_status_code(source: &reqwest::Error) -> String {
 #[derive(Debug, Snafu)]
 #[snafu(visibility = "pub(crate)")]
 pub(crate) enum Error {
+    #[snafu(display("Failed to clone repository: {}", source))]
+    CloneRepository {
+        source: tough::error::Error,
+        backtrace: Backtrace,
+    },
+
     #[snafu(display("Failed to run {}: {}", command_str, source))]
     CommandExec {
         command_str: String,

tuftool/src/main.rs

@@ -13,6 +13,7 @@
 
 mod add_key_role;
 mod add_role;
+mod clone;
 mod common;
 mod create;
 mod create_role;
@@ -77,14 +78,16 @@ impl Program {
 enum Command {
     /// Create a TUF repository
     Create(create::CreateArgs),
-    /// Download a TUF repository's resources
+    /// Download a TUF repository's targets
     Download(download::DownloadArgs),
     /// Update a TUF repository's metadata and optionally add targets
     Update(Box<update::UpdateArgs>),
     /// Manipulate a root.json metadata file
     Root(root::Command),
     /// Delegation Commands
     Delegation(Delegation),
+    /// Clone a fully functional TUF repository
+    Clone(clone::CloneArgs),
 }
 
 impl Command {
@@ -95,6 +98,7 @@ impl Command {
             Command::Download(args) => args.run(),
             Command::Update(args) => args.run(),
             Command::Delegation(cmd) => cmd.run(),
+            Command::Clone(cmd) => cmd.run(),
         }
     }
 }

tuftool/tests/clone_command.rs

@@ -0,0 +1,190 @@
+mod test_utils;
+
+use assert_cmd::Command;
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use tempfile::TempDir;
+use test_utils::{dir_url, test_data};
+use url::Url;
+
+struct RepoPaths {
+    root_path: PathBuf,
+    metadata_base_url: Url,
+    targets_base_url: Url,
+    metadata_outdir: TempDir,
+    targets_outdir: TempDir,
+}
+
+impl RepoPaths {
+    fn new() -> Self {
+        let base = test_data().join("tuf-reference-impl");
+        RepoPaths {
+            root_path: base.join("metadata").join("1.root.json"),
+            metadata_base_url: dir_url(base.join("metadata")),
+            targets_base_url: dir_url(base.join("targets")),
+            metadata_outdir: TempDir::new().unwrap(),
+            targets_outdir: TempDir::new().unwrap(),
+        }
+    }
+}
+
+enum FileType {
+    Metadata,
+    Target,
+}
+
+/// Asserts that a target file is identical to the TUF reference example
+fn assert_target_match(indir: &TempDir, filename: &str) {
+    assert_reference_file_match(indir, filename, FileType::Target)
+}
+
+/// Asserts that a metadata file is identical to the TUF reference example
+fn assert_metadata_match(indir: &TempDir, filename: &str) {
+    assert_reference_file_match(indir, filename, FileType::Metadata)
+}
+
+/// Asserts that the named file in `indir` exactly matches the file in `tuf-reference-impl/`
+fn assert_reference_file_match(indir: &TempDir, filename: &str, filetype: FileType) {
+    let got = read_to_string(indir.path().join(filename)).unwrap();
+
+    let ref_dir = match filetype {
+        FileType::Metadata => "metadata",
+        FileType::Target => "targets",
+    };
+    let reference = read_to_string(
+        test_utils::test_data()
+            .join("tuf-reference-impl")
+            .join(ref_dir)
+            .join(filename),
+    )
+    .unwrap();
+
+    assert_eq!(got, reference, "{} contents do not match.", filename);
+}
+
+/// Asserts that all metadata files that should exist do and are identical to the reference example
+fn assert_all_metadata(metadata_dir: &TempDir) {
+    for f in &[
+        "snapshot.json",
+        "targets.json",
+        "timestamp.json",
+        "1.root.json",
+        "role1.json",
+        "role2.json",
+    ] {
+        assert_metadata_match(&metadata_dir, f)
+    }
+}
+
+/// Given a `Command`, attach all the base args necessary for the `clone` subcommand
+fn clone_base_command<'a>(cmd: &'a mut Command, repo_paths: &RepoPaths) -> &'a mut Command {
+    cmd.args(&[
+        "clone",
+        "--root",
+        repo_paths.root_path.to_str().unwrap(),
+        "--metadata-url",
+        repo_paths.metadata_base_url.as_str(),
+        "--metadata-dir",
+        repo_paths.metadata_outdir.path().to_str().unwrap(),
+    ])
+}
+
+#[test]
+// Ensure that we successfully clone all metadata
+fn clone_metadata() {
+    let repo_paths = RepoPaths::new();
+    let mut cmd = Command::cargo_bin("tuftool").unwrap();
+    clone_base_command(&mut cmd, &repo_paths)
+        .args(&["--metadata-only"])
+        .assert()
+        .success();
+
+    assert_all_metadata(&repo_paths.metadata_outdir)
+}
+
+#[test]
+// Ensure that the `--target-names` argument collides with the `--metadata-only` argument
+fn clone_metadata_target_names_failure() {
+    let repo_paths = RepoPaths::new();
+    let mut cmd = Command::cargo_bin("tuftool").unwrap();
+    clone_base_command(&mut cmd, &repo_paths)
+        .args(&["--metadata-only", "--target-names", "foo"])
+        .assert()
+        .failure();
+}
+
+#[test]
+// Ensure that, even when provided with arguments for targets, the `--metadata-only` argument
+// _only_ downloads metadata files
+fn clone_metadata_target_args() {
+    let repo_paths = RepoPaths::new();
+    let mut cmd = Command::cargo_bin("tuftool").unwrap();
+    clone_base_command(&mut cmd, &repo_paths)
+        .args(&[
+            "--metadata-only",
+            "--targets-url",
+            repo_paths.targets_base_url.as_str(),
+            "--targets-dir",
+            repo_paths.targets_outdir.path().to_str().unwrap(),
+        ])
+        .assert()
+        .success();
+
+    assert_all_metadata(&repo_paths.metadata_outdir);
+    assert!(repo_paths
+        .targets_outdir
+        .path()
+        .read_dir()
+        .unwrap()
+        .next()
+        .is_none())
+}
+
+#[test]
+// Ensure we can clone a subset of targets
+fn clone_subset_targets() {
+    let target_name = "file1.txt";
+    let repo_paths = RepoPaths::new();
+    let mut cmd = Command::cargo_bin("tuftool").unwrap();
+    clone_base_command(&mut cmd, &repo_paths)
+        .args(&[
+            "--targets-url",
+            repo_paths.targets_base_url.as_str(),
+            "--targets-dir",
+            repo_paths.targets_outdir.path().to_str().unwrap(),
+            "--target-names",
+            target_name,
+        ])
+        .assert()
+        .success();
+
+    assert_all_metadata(&repo_paths.metadata_outdir);
+    assert_target_match(&repo_paths.targets_outdir, target_name);
+
+    assert_eq!(
+        repo_paths.targets_outdir.path().read_dir().unwrap().count(),
+        1
+    );
+}
+
+#[test]
+// Ensure we can clone an entire repo
+fn clone_full_repo() {
+    let repo_paths = RepoPaths::new();
+    let mut cmd = Command::cargo_bin("tuftool").unwrap();
+    clone_base_command(&mut cmd, &repo_paths)
+        .args(&[
+            "--targets-url",
+            repo_paths.targets_base_url.as_str(),
+            "--targets-dir",
+            repo_paths.targets_outdir.path().to_str().unwrap(),
+        ])
+        .assert()
+        .success();
+
+    assert_all_metadata(&&repo_paths.metadata_outdir);
+
+    for f in &["file1.txt", "file2.txt", "file3.txt"] {
+        assert_target_match(&repo_paths.targets_outdir, f)
+    }
+}