From f47cfbdf5bdcae74ba7b042a31ade14c7f59ee77 Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Wed, 11 Aug 2021 21:17:02 +0000 Subject: [PATCH 1/3] tuftool: Add `download_root` module This change adds a `download_root` module, which abstracts the logic of downloading `root.json` out of `download` (the subcommand). It also makes the HTTP request a bit more robust by handling the initial request error as well as a bad response. Additionally, it avoids creating the file until after the request has succeeded to avoid creating cruft on the user's system. --- tuftool/src/download_root.rs | 51 ++++++++++++++++++++++++++++++++++++ tuftool/src/error.rs | 16 +++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tuftool/src/download_root.rs diff --git a/tuftool/src/download_root.rs b/tuftool/src/download_root.rs new file mode 100644 index 00000000..3c7e2419 --- /dev/null +++ b/tuftool/src/download_root.rs @@ -0,0 +1,51 @@ +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT OR Apache-2.0 +//! The `download_root` module owns the logic for downloading a given version of `root.json`. + +use crate::error::{self, Result}; +use snafu::ResultExt; +use std::fs::File; +use std::num::NonZeroU64; +use std::path::{Path, PathBuf}; +use url::Url; + +/// Download the given version of `root.json` +/// This is an unsafe operation, and doesn't establish trust. It should only be used for testing! +pub(crate) fn download_root

( + metadata_base_url: &Url, + version: NonZeroU64, + outdir: P, +) -> Result +where + P: AsRef, +{ + let name = format!("{}.root.json", version); + + let path = outdir.as_ref().join(&name); + let url = metadata_base_url.join(&name).context(error::UrlParse { + url: format!("{}/{}", metadata_base_url.as_str(), name), + })?; + root_warning(&path); + + let mut root_request = reqwest::blocking::get(url.as_str()) + .context(error::ReqwestGet)? + .error_for_status() + .context(error::BadResponse { url })?; + + let mut f = File::create(&path).context(error::OpenFile { path: &path })?; + root_request.copy_to(&mut f).context(error::ReqwestCopy)?; + + Ok(path) +} + +/// Print a very noticeable warning message about the unsafe nature of downloading `root.json` +/// without verification +fn root_warning>(path: P) { + #[rustfmt::skip] + eprintln!("\ +================================================================= +WARNING: Downloading root.json to {} +This is unsafe and will not establish trust, use only for testing +=================================================================", + path.as_ref().display()); +} diff --git a/tuftool/src/error.rs b/tuftool/src/error.rs index 8dba29a5..494c0552 100644 --- a/tuftool/src/error.rs +++ b/tuftool/src/error.rs @@ -243,6 +243,13 @@ pub(crate) enum Error { backtrace: Backtrace, }, + #[snafu(display("Response '{}' from '{}': {}", get_status_code(source), url, source))] + BadResponse { + url: String, + source: reqwest::Error, + backtrace: Backtrace, + }, + #[snafu(display("Failed to sign repository: {}", source))] SignRepo { source: tough::error::Error, @@ -357,3 +364,12 @@ pub(crate) enum Error { backtrace: Backtrace, }, } + +// Extracts the status code from a reqwest::Error and converts it to a string to be displayed +fn get_status_code(source: &reqwest::Error) -> String { + source + .status() + .as_ref() + .map_or("Unknown", |i| i.as_str()) + .to_string() +} From f1c886b0ce806e66c1cc25c1c3a77cf49c972477 Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Wed, 11 Aug 2021 22:05:43 +0000 Subject: [PATCH 2/3] tuftool: use `download_root` module in `download` This change updates the `download` module/subcommand to make use of the previously added `download_root` function. It also defines a default of "1" for the `root_version` argument. Previously, we effectively had this default in code by using `1.root.json` in the event the argument wasn't passed. It also has the nice side effect of not needing to deal with an `Option` for this argument. --- tuftool/src/download.rs | 41 ++++++----------------------------------- tuftool/src/main.rs | 1 + 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/tuftool/src/download.rs b/tuftool/src/download.rs index cadcbc4b..b21cb0aa 100644 --- a/tuftool/src/download.rs +++ b/tuftool/src/download.rs @@ -1,10 +1,11 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT OR Apache-2.0 +use crate::download_root::download_root; use crate::error::{self, Result}; use snafu::{OptionExt, ResultExt}; use std::fs::File; -use std::io::{self}; +use std::io; use std::num::NonZeroU64; use std::path::{Path, PathBuf}; use structopt::StructOpt; @@ -18,8 +19,8 @@ pub(crate) struct DownloadArgs { root: Option, /// Remote root.json version number - #[structopt(short = "v", long = "root-version")] - root_version: Option, + #[structopt(short = "v", long = "root-version", default_value = "1")] + root_version: NonZeroU64, /// TUF repository metadata base URL #[structopt(short = "m", long = "metadata-url")] @@ -45,16 +46,6 @@ pub(crate) struct DownloadArgs { allow_expired_repo: bool, } -fn root_warning>(path: P) { - #[rustfmt::skip] - eprintln!("\ -================================================================= -WARNING: Downloading root.json to {} -This is unsafe and will not establish trust, use only for testing -=================================================================", - path.as_ref().display()); -} - fn expired_repo_warning>(path: P) { #[rustfmt::skip] eprintln!("\ @@ -71,28 +62,8 @@ impl DownloadArgs { let root_path = if let Some(path) = &self.root { PathBuf::from(path) } else if self.allow_root_download { - let name = if let Some(version) = self.root_version { - format!("{}.root.json", version) - } else { - String::from("1.root.json") - }; - let path = std::env::current_dir() - .context(error::CurrentDir)? - .join(&name); - let url = self - .metadata_base_url - .join(&name) - .context(error::UrlParse { - url: self.metadata_base_url.as_str(), - })?; - root_warning(&path); - - let mut f = File::create(&path).context(error::OpenFile { path: &path })?; - reqwest::blocking::get(url.as_str()) - .context(error::ReqwestGet)? - .copy_to(&mut f) - .context(error::ReqwestCopy)?; - path + let outdir = std::env::current_dir().context(error::CurrentDir)?; + download_root(&self.metadata_base_url, self.root_version, outdir)? } else { eprintln!("No root.json available"); std::process::exit(1); diff --git a/tuftool/src/main.rs b/tuftool/src/main.rs index 39e702ec..d54a7631 100644 --- a/tuftool/src/main.rs +++ b/tuftool/src/main.rs @@ -18,6 +18,7 @@ mod create; mod create_role; mod datetime; mod download; +mod download_root; mod error; mod remove_key_role; mod remove_role; From e62cfee2067ebd1db899123636e9f44c5aeeda4a Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Thu, 12 Aug 2021 21:08:26 +0000 Subject: [PATCH 3/3] tuftool: Add `clone` subcommand This adds a `clone` subcommand to `tuftool`, allowing a user to download a fully functioning TUF repository. A user has the option to download a full repository, a subset of the targets, or just metadata. --- tuftool/src/clone.rs | 147 ++++++++++++++++++++++++ tuftool/src/error.rs | 6 + tuftool/src/main.rs | 6 +- tuftool/tests/clone_command.rs | 198 +++++++++++++++++++++++++++++++++ 4 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 tuftool/src/clone.rs create mode 100644 tuftool/tests/clone_command.rs diff --git a/tuftool/src/clone.rs b/tuftool/src/clone.rs new file mode 100644 index 00000000..8c9f7759 --- /dev/null +++ b/tuftool/src/clone.rs @@ -0,0 +1,147 @@ +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::common::UNUSED_URL; +use crate::download_root::download_root; +use crate::error::{self, Result}; +use snafu::ResultExt; +use std::fs::File; +use std::num::NonZeroU64; +use std::path::PathBuf; +use structopt::StructOpt; +use tough::{ExpirationEnforcement, RepositoryLoader}; +use url::Url; + +#[derive(Debug, StructOpt)] +pub(crate) struct CloneArgs { + /// Path to root.json file for the repository + #[structopt( + short = "r", + long = "root", + required_if("allow-root-download", "false") + )] + root: Option, + + /// Remote root.json version number + #[structopt(short = "v", long = "root-version", default_value = "1")] + root_version: NonZeroU64, + + /// TUF repository metadata base URL + #[structopt(short = "m", long = "metadata-url")] + metadata_base_url: Url, + + /// TUF repository targets base URL + #[structopt(short = "t", long = "targets-url", required_unless = "metadata-only")] + targets_base_url: Option, + + /// Allow downloading the root.json file (unsafe) + #[structopt(long)] + allow_root_download: bool, + + /// Allow repo download for expired metadata (unsafe) + #[structopt(long)] + allow_expired_repo: bool, + + /// Download only these targets, if specified + #[structopt(short = "n", long = "target-names", conflicts_with = "metadata-only")] + target_names: Vec, + + /// Output directory of targets + #[structopt(long, required_unless = "metadata-only")] + targets_dir: Option, + + /// Output directory of metadata + #[structopt(long)] + metadata_dir: PathBuf, + + /// Only download the repository metadata, not the targets + #[structopt(long, conflicts_with_all(&["target-names", "targets-dir", "targets-base-url"]))] + metadata_only: bool, +} + +#[rustfmt::skip] +fn expired_repo_warning() { + eprintln!("\ +================================================================= +WARNING: repo metadata is expired, meaning the owner hasn't verified its contents lately and it could be unsafe! +================================================================="); +} + +impl CloneArgs { + pub(crate) fn run(&self) -> Result<()> { + // Use local root.json or download from repository + let root_path = if let Some(path) = &self.root { + PathBuf::from(path) + } else if self.allow_root_download { + let outdir = std::env::current_dir().context(error::CurrentDir)?; + download_root(&self.metadata_base_url, self.root_version, outdir)? + } else { + eprintln!("No root.json available"); + std::process::exit(1); + }; + + // Structopt won't allow `targets_base_url` to be None when it is required. We require the + // user to supply `targets_base_url` in the case they actually plan to download targets. + // When downloading metadata, we don't ever need to access the targets URL, so we use a + // fake URL to satisfy the library. + let targets_base_url = self + .targets_base_url + .as_ref() + .unwrap_or(&Url::parse(UNUSED_URL).context(error::UrlParse { + url: UNUSED_URL.to_owned(), + })?) + .clone(); + + // Load repository + let expiration_enforcement = if self.allow_expired_repo { + expired_repo_warning(); + ExpirationEnforcement::Unsafe + } else { + ExpirationEnforcement::Safe + }; + let repository = RepositoryLoader::new( + File::open(&root_path).context(error::OpenRoot { path: &root_path })?, + self.metadata_base_url.clone(), + targets_base_url, + ) + .expiration_enforcement(expiration_enforcement) + .load() + .context(error::RepoLoad)?; + + // Clone the repository, downloading none, all, or a subset of targets + if self.metadata_only { + println!("Cloning repository metadata to {:?}", self.metadata_dir); + repository + .cache_metadata(&self.metadata_dir, true) + .context(error::CloneRepository)?; + } else { + // Similar to `targets_base_url, structopt's guard rails won't let us have a + // `targets_dir` that is None when the argument is required. We only require the user + // to supply a targets directory if they actually plan on downloading targets. + let targets_dir = self.targets_dir.as_ref().expect( + "Developer error: `targets_dir` is required unless downloading metadata only", + ); + + println!( + "Cloning repository:\n\tmetadata location: {:?}\n\ttargets location: {:?}", + self.metadata_dir, targets_dir + ); + if self.target_names.is_empty() { + repository + .cache(&self.metadata_dir, &targets_dir, None::<&[&str]>, true) + .context(error::CloneRepository)?; + } else { + repository + .cache( + &self.metadata_dir, + &targets_dir, + Some(self.target_names.as_slice()), + true, + ) + .context(error::CloneRepository)?; + } + }; + + Ok(()) + } +} diff --git a/tuftool/src/error.rs b/tuftool/src/error.rs index 494c0552..74daafdd 100644 --- a/tuftool/src/error.rs +++ b/tuftool/src/error.rs @@ -13,6 +13,12 @@ pub(crate) type Result = std::result::Result; #[derive(Debug, Snafu)] #[snafu(visibility = "pub(crate)")] pub(crate) enum Error { + #[snafu(display("Failed to clone repository: {}", source))] + CloneRepository { + source: tough::error::Error, + backtrace: Backtrace, + }, + #[snafu(display("Failed to run {}: {}", command_str, source))] CommandExec { command_str: String, diff --git a/tuftool/src/main.rs b/tuftool/src/main.rs index d54a7631..299227fe 100644 --- a/tuftool/src/main.rs +++ b/tuftool/src/main.rs @@ -13,6 +13,7 @@ mod add_key_role; mod add_role; +mod clone; mod common; mod create; mod create_role; @@ -77,7 +78,7 @@ impl Program { enum Command { /// Create a TUF repository Create(create::CreateArgs), - /// Download a TUF repository's resources + /// Download a TUF repository's targets Download(download::DownloadArgs), /// Update a TUF repository's metadata and optionally add targets Update(Box), @@ -85,6 +86,8 @@ enum Command { Root(root::Command), /// Delegation Commands Delegation(Delegation), + /// Clone a TUF repository, including metadata and some or all targets + Clone(clone::CloneArgs), } impl Command { @@ -95,6 +98,7 @@ impl Command { Command::Download(args) => args.run(), Command::Update(args) => args.run(), Command::Delegation(cmd) => cmd.run(), + Command::Clone(cmd) => cmd.run(), } } } diff --git a/tuftool/tests/clone_command.rs b/tuftool/tests/clone_command.rs new file mode 100644 index 00000000..a7924947 --- /dev/null +++ b/tuftool/tests/clone_command.rs @@ -0,0 +1,198 @@ +mod test_utils; + +use assert_cmd::Command; +use std::fs::read_to_string; +use std::path::PathBuf; +use tempfile::TempDir; +use test_utils::{dir_url, test_data}; +use url::Url; + +struct RepoPaths { + root_path: PathBuf, + metadata_base_url: Url, + targets_base_url: Url, + metadata_outdir: TempDir, + targets_outdir: TempDir, +} + +impl RepoPaths { + fn new() -> Self { + let base = test_data().join("tuf-reference-impl"); + RepoPaths { + root_path: base.join("metadata").join("1.root.json"), + metadata_base_url: dir_url(base.join("metadata")), + targets_base_url: dir_url(base.join("targets")), + metadata_outdir: TempDir::new().unwrap(), + targets_outdir: TempDir::new().unwrap(), + } + } +} + +enum FileType { + Metadata, + Target, +} + +/// Asserts that a target file is identical to the TUF reference example +fn assert_target_match(indir: &TempDir, filename: &str) { + assert_reference_file_match(indir, filename, FileType::Target) +} + +/// Asserts that a metadata file is identical to the TUF reference example +fn assert_metadata_match(indir: &TempDir, filename: &str) { + assert_reference_file_match(indir, filename, FileType::Metadata) +} + +/// Asserts that the named file in `indir` exactly matches the file in `tuf-reference-impl/` +fn assert_reference_file_match(indir: &TempDir, filename: &str, filetype: FileType) { + let got = read_to_string(indir.path().join(filename)).unwrap(); + + let ref_dir = match filetype { + FileType::Metadata => "metadata", + FileType::Target => "targets", + }; + let reference = read_to_string( + test_utils::test_data() + .join("tuf-reference-impl") + .join(ref_dir) + .join(filename), + ) + .unwrap(); + + assert_eq!(got, reference, "{} contents do not match.", filename); +} + +/// Asserts that all metadata files that should exist do and are identical to the reference example +fn assert_all_metadata(metadata_dir: &TempDir) { + for f in &[ + "snapshot.json", + "targets.json", + "timestamp.json", + "1.root.json", + "role1.json", + "role2.json", + ] { + assert_metadata_match(&metadata_dir, f) + } +} + +/// Given a `Command`, attach all the base args necessary for the `clone` subcommand +fn clone_base_command<'a>(cmd: &'a mut Command, repo_paths: &RepoPaths) -> &'a mut Command { + cmd.args(&[ + "clone", + "--root", + repo_paths.root_path.to_str().unwrap(), + "--metadata-url", + repo_paths.metadata_base_url.as_str(), + "--metadata-dir", + repo_paths.metadata_outdir.path().to_str().unwrap(), + ]) +} + +#[test] +// Ensure that we successfully clone all metadata +fn clone_metadata() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&["--metadata-only"]) + .assert() + .success(); + + assert_all_metadata(&repo_paths.metadata_outdir) +} + +#[test] +// Ensure that target arguments collide with the `--megadata-only` argument +fn clone_metadata_target_args_failure() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + // --target-names + clone_base_command(&mut cmd, &repo_paths) + .args(&["--metadata-only", "--target-names", "foo"]) + .assert() + .failure(); + + // --targets-url + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--metadata-only", + "--targets-url", + repo_paths.targets_base_url.as_str(), + ]) + .assert() + .failure(); + + // --targets-dir + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--metadata-only", + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + ]) + .assert() + .failure(); + + // all target args + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--metadata-only", + "--targets-url", + repo_paths.targets_base_url.as_str(), + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + "--target-names", + "foo", + ]) + .assert() + .failure(); +} + +#[test] +// Ensure we can clone a subset of targets +fn clone_subset_targets() { + let target_name = "file1.txt"; + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--targets-url", + repo_paths.targets_base_url.as_str(), + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + "--target-names", + target_name, + ]) + .assert() + .success(); + + assert_all_metadata(&repo_paths.metadata_outdir); + assert_target_match(&repo_paths.targets_outdir, target_name); + + assert_eq!( + repo_paths.targets_outdir.path().read_dir().unwrap().count(), + 1 + ); +} + +#[test] +// Ensure we can clone an entire repo +fn clone_full_repo() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--targets-url", + repo_paths.targets_base_url.as_str(), + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + ]) + .assert() + .success(); + + assert_all_metadata(&repo_paths.metadata_outdir); + + for f in &["file1.txt", "file2.txt", "file3.txt"] { + assert_target_match(&repo_paths.targets_outdir, f) + } +}