diff --git a/Cargo.lock b/Cargo.lock index a65273b..0d0e297 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -1018,9 +1018,8 @@ dependencies = [ [[package]] name = "ocidir" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1123c697592d4240224b7e09d50375c7da3f17320ed741c922f54b5377b79eb0" +version = "0.3.1" +source = "git+https://github.com/containers/ocidir-rs#a943d18a54806b7b6a5425bc9cacddd4d7f326d1" dependencies = [ "camino", "cap-std-ext", diff --git a/puzzlefs-lib/Cargo.toml b/puzzlefs-lib/Cargo.toml index e361200..bf61611 100644 --- a/puzzlefs-lib/Cargo.toml +++ b/puzzlefs-lib/Cargo.toml @@ -40,7 +40,7 @@ os_pipe = "1.1.2" tempfile = "3.10" openat = "0.1.21" zstd-seekable = "0.1.23" -ocidir = "0.3.0" +ocidir = {git="https://github.com/containers/ocidir-rs"} cap-std = "3.2.0" diff --git a/puzzlefs-lib/src/builder.rs b/puzzlefs-lib/src/builder.rs index d2eea81..5e3e5aa 100644 --- a/puzzlefs-lib/src/builder.rs +++ b/puzzlefs-lib/src/builder.rs @@ -502,14 +502,12 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu .find_manifest_with_tag(tag)? .ok_or_else(|| WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()))?; let config_digest = manifest.config().digest().digest(); - let config_digest_path = oci.blob_path().join(config_digest); - enable_verity_for_file(&oci.0.dir.open(config_digest_path)?)?; + let config_digest_path = Image::blob_path().join(config_digest); + enable_verity_for_file(&oci.0.dir().open(config_digest_path)?)?; for (content_addressed_file, verity_hash) in rootfs.get_verity_data()? { - let file_path = oci - .blob_path() - .join(Digest::new(&content_addressed_file).to_string()); - let fd = oci.0.dir.open(&file_path)?; + let file_path = Image::blob_path().join(Digest::new(&content_addressed_file).to_string()); + let fd = oci.0.dir().open(&file_path)?; if let Err(e) = fsverity_enable( fd.as_raw_fd(), FS_VERITY_BLOCK_SIZE_DEFAULT, @@ -564,8 +562,8 @@ pub mod tests { let md = image .0 - .dir - .symlink_metadata(image.blob_path().join(FILE_DIGEST)) + .dir() + .symlink_metadata(Image::blob_path().join(FILE_DIGEST)) .unwrap(); assert!(md.is_file()); @@ -603,14 +601,15 @@ pub mod tests { chunks[0].len, decompressor.get_uncompressed_length().unwrap() ); - Ok(()) } else { panic!("bad inode mode: {:?}", inodes[1].mode); - } + }; + image.0.fsck()?; + Ok::<(), anyhow::Error>(()) } #[test] - fn test_delta_generation() { + fn test_delta_generation() -> anyhow::Result<()> { let dir = tempdir().unwrap(); let image = Image::new(dir.path()).unwrap(); let tag = "test"; @@ -623,6 +622,7 @@ pub mod tests { delta_dir.join("SekienAkashita.jpg"), ) .unwrap(); + image.0.fsck()?; let new_tag = "test2"; let (_desc, image) = @@ -631,6 +631,7 @@ pub mod tests { assert_eq!(delta.metadatas.len(), 2); let image = Image::new(dir.path()).unwrap(); + image.0.fsck()?; let mut pfs = PuzzleFS::open(image, new_tag, None).unwrap(); assert_eq!(pfs.max_inode().unwrap(), 3); let mut walker = WalkPuzzleFS::walk(&mut pfs).unwrap(); @@ -651,6 +652,7 @@ pub mod tests { assert_eq!(foo_dir.inode.dir_entries().unwrap().len(), 0); assert!(walker.next().is_none()); + Ok(()) } fn do_vecs_match(a: &[T], b: &[T]) -> bool { @@ -662,8 +664,8 @@ pub mod tests { matching == a.len() } - fn get_image_blobs(image: &Image) -> Vec { - WalkDir::new(image.blob_path()) + fn get_image_blobs() -> Vec { + WalkDir::new(Image::blob_path()) .contents_first(false) .follow_links(false) .same_file_system(true) @@ -685,7 +687,7 @@ pub mod tests { for (i, image) in images.iter().enumerate() { build_test_fs(path, image, "test").unwrap(); - let ents = get_image_blobs(image); + let ents = get_image_blobs(); sha_suite.push(ents); if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) { @@ -708,7 +710,7 @@ pub mod tests { for (i, image) in images.iter().enumerate() { build_test_fs(&path[i], image, "test").unwrap(); - let ents = get_image_blobs(image); + let ents = get_image_blobs(); sha_suite.push(ents); if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) { diff --git a/puzzlefs-lib/src/oci.rs b/puzzlefs-lib/src/oci.rs index abc01df..158bef4 100644 --- a/puzzlefs-lib/src/oci.rs +++ b/puzzlefs-lib/src/oci.rs @@ -3,7 +3,6 @@ use std::any::Any; use std::backtrace::Backtrace; use std::fs; use std::io; -use std::io::Write; use std::io::{Read, Seek}; use std::path::{Path, PathBuf}; @@ -17,10 +16,7 @@ pub use crate::format::Digest; use crate::oci::media_types::{PuzzleFSMediaType, PUZZLEFS_ROOTFS, VERITY_ROOT_HASH_ANNOTATION}; use ocidir::oci_spec::image; pub use ocidir::oci_spec::image::Descriptor; -use ocidir::oci_spec::image::{ - DescriptorBuilder, ImageIndex, ImageManifest, ImageManifestBuilder, MediaType, Sha256Digest, -}; -use ocidir::oci_spec::OciSpecError; +use ocidir::oci_spec::image::{ImageIndex, ImageManifest, MediaType}; use ocidir::OciDir; use std::collections::HashMap; use std::str::FromStr; @@ -28,7 +24,6 @@ use std::str::FromStr; use std::io::Cursor; pub mod media_types; -const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name"; pub struct Image(pub OciDir); @@ -36,18 +31,22 @@ impl Image { pub fn new(oci_dir: &Path) -> Result { fs::create_dir_all(oci_dir)?; let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?; - let oci_dir = OciDir::ensure(&d)?; + let oci_dir = OciDir::ensure(d)?; Ok(Self(oci_dir)) } pub fn open(oci_dir: &Path) -> Result { let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?; - let oci_dir = OciDir::open(&d)?; + let blobs_dir = cap_std::fs::Dir::open_ambient_dir( + oci_dir.join(Self::blob_path()), + cap_std::ambient_authority(), + )?; + let oci_dir = OciDir::open_with_external_blobs(d, blobs_dir)?; Ok(Self(oci_dir)) } - pub fn blob_path(&self) -> PathBuf { + pub fn blob_path() -> PathBuf { // TODO: use BLOBDIR constant from ocidir after making it public PathBuf::from("blobs/sha256") } @@ -72,6 +71,7 @@ impl Image { let uncompressed_size = io::copy(&mut <&[u8]>::clone(&buf), &mut compressed)?; compressed.end()?; let compressed_size = compressed_data.get_ref().len() as u64; + let final_size = std::cmp::min(compressed_size, uncompressed_size); // store the uncompressed blob if the compressed version has bigger size let final_data = if compressed_blob && compressed_size >= uncompressed_size { @@ -90,7 +90,7 @@ impl Image { let fs_verity_digest = get_fs_verity_digest(&compressed_data.get_ref()[..])?; let mut descriptor = Descriptor::new( MediaType::Other(media_type_with_extension), - uncompressed_size, + final_size, image::Digest::from_str(&digest_string)?, ); // We need to store the PuzzleFS Rootfs verity digest as an annotation (obviously we cannot @@ -103,12 +103,12 @@ impl Image { ); descriptor.set_annotations(Some(annotations)); } - let path = self.blob_path().join(descriptor.digest().digest()); + let path = Self::blob_path().join(descriptor.digest().digest()); // avoid replacing the data blob so we don't drop fsverity data - if self.0.dir.exists(&path) { + if self.0.dir().exists(&path) { let mut hasher = Sha256::new(); - let mut file = self.0.dir.open(&path)?; + let mut file = self.0.dir().open(&path)?; io::copy(&mut file, &mut hasher)?; let existing_digest = hasher.finalize(); if existing_digest != digest { @@ -120,7 +120,7 @@ impl Image { .into()); } } else { - self.0.dir.write(&path, final_data)?; + self.0.dir().write(&path, final_data)?; } // Let's make the PuzzleFS image rootfs the first layer so it's easy to find @@ -136,7 +136,7 @@ impl Image { } fn open_raw_blob(&self, digest: &str, verity: Option<&[u8]>) -> io::Result { - let file = self.0.dir.open(self.blob_path().join(digest))?; + let file = self.0.blobs_dir().open(digest)?; if let Some(verity) = verity { check_fs_verity(&file, verity).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; } @@ -201,21 +201,10 @@ impl Image { Ok(file) } - // TODO: export this function from ocidr / find another way to avoid code duplication - fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool { - d.annotations() - .as_ref() - .and_then(|annos| annos.get(OCI_TAG_ANNOTATION)) - .filter(|tagval| tagval.as_str() == tag) - .is_some() - } - pub fn get_image_manifest_fd(&self, tag: &str) -> Result { - let index = self.get_index()?; - let image_manifest = index - .manifests() - .iter() - .find(|desc| Self::descriptor_is_tagged(desc, tag)) + let image_manifest = self + .0 + .find_manifest_descriptor_with_tag(tag)? .ok_or_else(|| { WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()) })?; @@ -270,39 +259,11 @@ impl Image { } pub fn get_index(&self) -> Result { - Ok(self - .0 - .read_index()? - .ok_or_else(|| OciSpecError::Other("missing OCI index".to_string()))?) + Ok(self.0.read_index()?) } pub fn get_empty_manifest(&self) -> Result { - // see https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor - let config = DescriptorBuilder::default() - .media_type(MediaType::EmptyJSON) - .size(2_u32) - .digest(Sha256Digest::from_str( - "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - )?) - .data("e30=") - .build()?; - - if !self.0.dir.exists( - self.blob_path() - .join("44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"), - ) { - let mut blob = self.0.create_blob()?; - blob.write_all("{}".as_bytes())?; - // TODO: blob.complete_verified_as(&config)? once https://github.com/containers/ocidir-rs/pull/18 is merged - blob.complete()?; - } - - let image_manifest = ImageManifestBuilder::default() - .schema_version(2_u32) - .config(config) - .layers(Vec::new()) - .build()?; - Ok(image_manifest) + Ok(self.0.new_empty_manifest()?.build()?) } } @@ -330,8 +291,8 @@ mod tests { let md = image .0 - .dir - .symlink_metadata(image.blob_path().join(DIGEST))?; + .dir() + .symlink_metadata(Image::blob_path().join(DIGEST))?; assert!(md.is_file()); Ok(()) }