From 6ee4818b0aaee1ada9eee5eeecce6f1d08e95adb Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Thu, 24 Oct 2024 08:23:29 -0700 Subject: [PATCH 01/12] Use fully qualified names from stone crate This is required to properly use cbindgen for the future C FFI bindings. cbindgen completely breaks when struct names collide since it doesn't consider fully qualified paths and there's no apparent workaround. --- boulder/src/package.rs | 4 +- boulder/src/package/analysis.rs | 13 +- boulder/src/package/collect.rs | 41 +++-- boulder/src/package/emit.rs | 5 +- boulder/src/package/emit/manifest.rs | 3 +- boulder/src/package/emit/manifest/binary.rs | 13 +- crates/stone/Cargo.toml | 4 + crates/stone/benches/read.rs | 3 +- crates/stone/src/ext.rs | 77 +++++++++ crates/stone/src/header/mod.rs | 60 +++---- crates/stone/src/header/v1.rs | 27 +-- crates/stone/src/lib.rs | 125 ++++---------- crates/stone/src/payload/attribute.rs | 12 +- crates/stone/src/payload/content.rs | 8 + crates/stone/src/payload/index.rs | 12 +- crates/stone/src/payload/layout.rs | 86 +++++----- crates/stone/src/payload/meta.rs | 180 +++++++++---------- crates/stone/src/payload/mod.rs | 85 +++++---- crates/stone/src/read/mod.rs | 181 +++++++++++++------- crates/stone/src/write.rs | 138 +++++++-------- crates/stone/src/write/digest.rs | 12 +- moss/src/cli/extract.rs | 21 ++- moss/src/cli/index.rs | 7 +- moss/src/cli/info.rs | 6 +- moss/src/cli/inspect.rs | 28 ++- moss/src/client/boot.rs | 22 ++- moss/src/client/cache.rs | 13 +- moss/src/client/mod.rs | 58 +++---- moss/src/client/verify.rs | 8 +- moss/src/db/layout/mod.rs | 55 +++--- moss/src/db/meta/mod.rs | 6 +- moss/src/dependency.rs | 28 +-- moss/src/package/meta.rs | 158 ++++++++++------- moss/src/registry/plugin/cobble.rs | 7 +- moss/src/repository/manager.rs | 9 +- 35 files changed, 823 insertions(+), 692 deletions(-) create mode 100644 crates/stone/src/ext.rs create mode 100644 crates/stone/src/payload/content.rs diff --git a/boulder/src/package.rs b/boulder/src/package.rs index 0b6f1fb8..333cd2d2 100644 --- a/boulder/src/package.rs +++ b/boulder/src/package.rs @@ -6,9 +6,9 @@ use std::{io, num::NonZeroU64}; use fs_err as fs; use itertools::Itertools; +use stone::StoneDigestWriterHasher; use thiserror::Error; -use stone::write::digest; use stone_recipe::{script, Package}; use crate::{build, container, timing, util, Macros, Paths, Recipe, Timing}; @@ -60,7 +60,7 @@ impl<'a> Packager<'a> { pub fn package(&self, timing: &mut Timing) -> Result<(), Error> { // Hasher used for calculating file digests - let mut hasher = digest::Hasher::new(); + let mut hasher = StoneDigestWriterHasher::new(); let timer = timing.begin(timing::Kind::Analyze); diff --git a/boulder/src/package/analysis.rs b/boulder/src/package/analysis.rs index e70f9888..db1ca2d7 100644 --- a/boulder/src/package/analysis.rs +++ b/boulder/src/package/analysis.rs @@ -9,7 +9,7 @@ use std::{ }; use moss::{Dependency, Provider}; -use stone::write::digest; +use stone::StoneDigestWriterHasher; use tui::{ProgressBar, ProgressStyle, Styled}; use crate::{Paths, Recipe}; @@ -25,12 +25,17 @@ pub struct Chain<'a> { recipe: &'a Recipe, paths: &'a Paths, collector: &'a Collector, - hasher: &'a mut digest::Hasher, + hasher: &'a mut StoneDigestWriterHasher, pub buckets: BTreeMap, } impl<'a> Chain<'a> { - pub fn new(paths: &'a Paths, recipe: &'a Recipe, collector: &'a Collector, hasher: &'a mut digest::Hasher) -> Self { + pub fn new( + paths: &'a Paths, + recipe: &'a Recipe, + collector: &'a Collector, + hasher: &'a mut StoneDigestWriterHasher, + ) -> Self { Self { handlers: vec![ Box::new(handler::ignore_blocked), @@ -144,7 +149,7 @@ impl Bucket { pub struct BucketMut<'a> { pub providers: &'a mut BTreeSet, pub dependencies: &'a mut BTreeSet, - pub hasher: &'a mut digest::Hasher, + pub hasher: &'a mut StoneDigestWriterHasher, pub recipe: &'a Recipe, pub paths: &'a Paths, } diff --git a/boulder/src/package/collect.rs b/boulder/src/package/collect.rs index 1c8ad0a4..057c0f84 100644 --- a/boulder/src/package/collect.rs +++ b/boulder/src/package/collect.rs @@ -12,8 +12,7 @@ use std::{ use fs_err as fs; use glob::Pattern; use nix::libc::{S_IFDIR, S_IRGRP, S_IROTH, S_IRWXU, S_IXGRP, S_IXOTH}; -use stone::payload::{layout, Layout}; -use stone::write::digest; +use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutBody, StonePayloadLayoutEntry}; use thiserror::Error; #[derive(Debug, Clone, Eq, PartialEq)] @@ -61,7 +60,7 @@ impl Collector { } /// Produce a [`PathInfo`] from the provided [`Path`] - pub fn path(&self, path: &Path, hasher: &mut digest::Hasher) -> Result { + pub fn path(&self, path: &Path, hasher: &mut StoneDigestWriterHasher) -> Result { let metadata = fs::metadata(path)?; self.path_with_metadata(path.to_path_buf(), &metadata, hasher) } @@ -70,7 +69,7 @@ impl Collector { &self, path: PathBuf, metadata: &Metadata, - hasher: &mut digest::Hasher, + hasher: &mut StoneDigestWriterHasher, ) -> Result { let target_path = Path::new("/").join(path.strip_prefix(&self.root).expect("path is ancestor of root")); @@ -85,7 +84,7 @@ impl Collector { pub fn enumerate_paths( &self, subdir: Option<(PathBuf, Metadata)>, - hasher: &mut digest::Hasher, + hasher: &mut StoneDigestWriterHasher, ) -> Result, Error> { let mut paths = vec![]; @@ -127,7 +126,7 @@ impl Collector { pub struct PathInfo { pub path: PathBuf, pub target_path: PathBuf, - pub layout: Layout, + pub layout: StonePayloadLayoutBody, pub size: u64, pub package: String, } @@ -137,7 +136,7 @@ impl PathInfo { path: PathBuf, target_path: PathBuf, metadata: &Metadata, - hasher: &mut digest::Hasher, + hasher: &mut StoneDigestWriterHasher, package: String, ) -> Result { let layout = layout_from_metadata(&path, &target_path, metadata, hasher)?; @@ -151,7 +150,7 @@ impl PathInfo { }) } - pub fn restat(&mut self, hasher: &mut digest::Hasher) -> Result<(), Error> { + pub fn restat(&mut self, hasher: &mut StoneDigestWriterHasher) -> Result<(), Error> { let metadata = fs::metadata(&self.path)?; self.layout = layout_from_metadata(&self.path, &self.target_path, &metadata, hasher)?; self.size = metadata.size(); @@ -159,11 +158,11 @@ impl PathInfo { } pub fn is_file(&self) -> bool { - matches!(self.layout.entry, layout::Entry::Regular(_, _)) + matches!(self.layout.entry, StonePayloadLayoutEntry::Regular(_, _)) } pub fn file_hash(&self) -> Option { - if let layout::Entry::Regular(hash, _) = &self.layout.entry { + if let StonePayloadLayoutEntry::Regular(hash, _) = &self.layout.entry { Some(*hash) } else { None @@ -188,8 +187,8 @@ fn layout_from_metadata( path: &Path, target_path: &Path, metadata: &Metadata, - hasher: &mut digest::Hasher, -) -> Result { + hasher: &mut StoneDigestWriterHasher, +) -> Result { // Strip /usr let target = target_path .strip_prefix("/usr") @@ -199,7 +198,7 @@ fn layout_from_metadata( let file_type = metadata.file_type(); - Ok(Layout { + Ok(StonePayloadLayoutBody { uid: metadata.uid(), gid: metadata.gid(), mode: metadata.mode(), @@ -207,21 +206,21 @@ fn layout_from_metadata( entry: if file_type.is_symlink() { let source = fs::read_link(path)?; - layout::Entry::Symlink(source.to_string_lossy().to_string(), target) + StonePayloadLayoutEntry::Symlink(source.to_string_lossy().to_string(), target) } else if file_type.is_dir() { - layout::Entry::Directory(target) + StonePayloadLayoutEntry::Directory(target) } else if file_type.is_char_device() { - layout::Entry::CharacterDevice(target) + StonePayloadLayoutEntry::CharacterDevice(target) } else if file_type.is_block_device() { - layout::Entry::BlockDevice(target) + StonePayloadLayoutEntry::BlockDevice(target) } else if file_type.is_fifo() { - layout::Entry::Fifo(target) + StonePayloadLayoutEntry::Fifo(target) } else if file_type.is_socket() { - layout::Entry::Socket(target) + StonePayloadLayoutEntry::Socket(target) } else { hasher.reset(); - let mut digest_writer = digest::Writer::new(io::sink(), hasher); + let mut digest_writer = StoneDigestWriter::new(io::sink(), hasher); let mut file = fs::File::open(path)?; // Copy bytes to null sink so we don't @@ -230,7 +229,7 @@ fn layout_from_metadata( let hash = hasher.digest128(); - layout::Entry::Regular(hash, target) + StonePayloadLayoutEntry::Regular(hash, target) }, }) } diff --git a/boulder/src/package/emit.rs b/boulder/src/package/emit.rs index f80d30b4..1fb9b298 100644 --- a/boulder/src/package/emit.rs +++ b/boulder/src/package/emit.rs @@ -10,6 +10,7 @@ use std::{ use fs_err::{self as fs, File}; use itertools::Itertools; use moss::{package::Meta, Dependency, Provider}; +use stone::{StoneHeaderV1FileType, StoneWriteError, StoneWriter}; use thiserror::Error; use tui::{ProgressBar, ProgressStyle, Styled}; @@ -172,7 +173,7 @@ fn emit_package(paths: &Paths, package: &Package<'_>) -> Result<(), Error> { let mut out_file = File::create(out_path)?; // Create stone binary writer - let mut writer = stone::Writer::new(&mut out_file, stone::header::v1::FileType::Binary)?; + let mut writer = StoneWriter::new(&mut out_file, StoneHeaderV1FileType::Binary)?; // Add metadata { @@ -233,7 +234,7 @@ fn emit_package(paths: &Paths, package: &Package<'_>) -> Result<(), Error> { #[derive(Debug, Error)] pub enum Error { #[error("stone binary writer")] - StoneBinaryWriter(#[from] stone::write::Error), + StoneBinaryWriter(#[from] StoneWriteError), #[error("manifest")] Manifest(#[from] manifest::Error), #[error("io")] diff --git a/boulder/src/package/emit/manifest.rs b/boulder/src/package/emit/manifest.rs index 69a5b601..aa562999 100644 --- a/boulder/src/package/emit/manifest.rs +++ b/boulder/src/package/emit/manifest.rs @@ -4,6 +4,7 @@ use std::{collections::BTreeSet, io, path::PathBuf}; +use stone::StoneWriteError; use thiserror::Error; use crate::{Architecture, Paths, Recipe}; @@ -69,7 +70,7 @@ impl<'a> Manifest<'a> { #[derive(Debug, Error)] pub enum Error { #[error("stone binary writer")] - StoneWriter(#[from] stone::write::Error), + StoneWriter(#[from] StoneWriteError), #[error("encode json")] Json(#[from] serde_json::Error), #[error("io")] diff --git a/boulder/src/package/emit/manifest/binary.rs b/boulder/src/package/emit/manifest/binary.rs index f707b2ca..52aa702f 100644 --- a/boulder/src/package/emit/manifest/binary.rs +++ b/boulder/src/package/emit/manifest/binary.rs @@ -6,10 +6,7 @@ use std::{collections::BTreeSet, path::Path}; use fs_err::File; use moss::Dependency; -use stone::{ - header::v1::FileType, - payload::{self, meta}, -}; +use stone::{StoneHeaderV1FileType, StonePayloadMetaBody, StonePayloadMetaKind, StonePayloadMetaTag, StoneWriter}; use super::Error; use crate::package::emit::Package; @@ -17,7 +14,7 @@ use crate::package::emit::Package; pub fn write(path: &Path, packages: &BTreeSet<&Package<'_>>, build_deps: &BTreeSet) -> Result<(), Error> { let mut output = File::create(path)?; - let mut writer = stone::Writer::new(&mut output, FileType::BuildManifest)?; + let mut writer = StoneWriter::new(&mut output, StoneHeaderV1FileType::BuildManifest)?; // Add each package for package in packages { @@ -29,9 +26,9 @@ pub fn write(path: &Path, packages: &BTreeSet<&Package<'_>>, build_deps: &BTreeS // Add build deps for name in build_deps { if let Ok(dep) = Dependency::from_name(name) { - payload.push(payload::Meta { - tag: meta::Tag::BuildDepends, - kind: meta::Kind::Dependency(dep.kind.into(), dep.name), + payload.push(StonePayloadMetaBody { + tag: StonePayloadMetaTag::BuildDepends, + kind: StonePayloadMetaKind::Dependency(dep.kind.into(), dep.name), }); } } diff --git a/crates/stone/Cargo.toml b/crates/stone/Cargo.toml index 1c6537e0..f0da5092 100644 --- a/crates/stone/Cargo.toml +++ b/crates/stone/Cargo.toml @@ -6,6 +6,10 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = [] +ffi = [] + [dependencies] strum.workspace = true thiserror.workspace = true diff --git a/crates/stone/benches/read.rs b/crates/stone/benches/read.rs index d13cfb6b..3a7ba675 100644 --- a/crates/stone/benches/read.rs +++ b/crates/stone/benches/read.rs @@ -9,6 +9,7 @@ use std::{ }; use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use stone::StoneDecodedPayload; fn read_unbuffered(path: impl AsRef) { read(File::open(path).unwrap()); @@ -23,7 +24,7 @@ fn read(reader: R) { let payloads = stone.payloads().unwrap().collect::, _>>().unwrap(); - if let Some(content) = payloads.iter().find_map(stone::read::PayloadKind::content) { + if let Some(content) = payloads.iter().find_map(StoneDecodedPayload::content) { stone.unpack_content(content, &mut sink()).unwrap(); } } diff --git a/crates/stone/src/ext.rs b/crates/stone/src/ext.rs new file mode 100644 index 00000000..5fc9875e --- /dev/null +++ b/crates/stone/src/ext.rs @@ -0,0 +1,77 @@ +use std::io::{Read, Result, Write}; + +pub trait ReadExt: Read { + fn read_u8(&mut self) -> Result { + let bytes = self.read_array::<1>()?; + Ok(bytes[0]) + } + + fn read_u16(&mut self) -> Result { + let bytes = self.read_array()?; + Ok(u16::from_be_bytes(bytes)) + } + + fn read_u32(&mut self) -> Result { + let bytes = self.read_array()?; + Ok(u32::from_be_bytes(bytes)) + } + + fn read_u64(&mut self) -> Result { + let bytes = self.read_array()?; + Ok(u64::from_be_bytes(bytes)) + } + + fn read_u128(&mut self) -> Result { + let bytes = self.read_array()?; + Ok(u128::from_be_bytes(bytes)) + } + + fn read_array(&mut self) -> Result<[u8; N]> { + let mut bytes = [0u8; N]; + self.read_exact(&mut bytes)?; + Ok(bytes) + } + + fn read_vec(&mut self, length: usize) -> Result> { + let mut bytes = vec![0u8; length]; + self.read_exact(&mut bytes)?; + Ok(bytes) + } + + fn read_string(&mut self, length: u64) -> Result { + let mut string = String::with_capacity(length as usize); + self.take(length).read_to_string(&mut string)?; + Ok(string) + } +} + +impl ReadExt for T {} + +pub trait WriteExt: Write { + fn write_u8(&mut self, item: u8) -> Result<()> { + self.write_array([item]) + } + + fn write_u16(&mut self, item: u16) -> Result<()> { + self.write_array(item.to_be_bytes()) + } + + fn write_u32(&mut self, item: u32) -> Result<()> { + self.write_array(item.to_be_bytes()) + } + + fn write_u64(&mut self, item: u64) -> Result<()> { + self.write_array(item.to_be_bytes()) + } + + fn write_u128(&mut self, item: u128) -> Result<()> { + self.write_array(item.to_be_bytes()) + } + + fn write_array(&mut self, bytes: [u8; N]) -> Result<()> { + self.write_all(&bytes)?; + Ok(()) + } +} + +impl WriteExt for T {} diff --git a/crates/stone/src/header/mod.rs b/crates/stone/src/header/mod.rs index cf50e392..e51aa4c8 100644 --- a/crates/stone/src/header/mod.rs +++ b/crates/stone/src/header/mod.rs @@ -6,17 +6,19 @@ use std::io::{self, Read, Write}; use thiserror::Error; -use crate::{ReadExt, WriteExt}; +use crate::ext::{ReadExt, WriteExt}; + +pub use self::v1::{StoneHeaderV1, StoneHeaderV1DecodeError, StoneHeaderV1FileType}; pub mod v1; /// Well defined magic field for a stone header -pub const STONE_MAGIC: &[u8; 4] = b"\0mos"; +pub const STONE_HEADER_MAGIC: &[u8; 4] = b"\0mos"; /// Format versions are defined as u32, to allow further mangling in future -#[repr(u32)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Version { +#[repr(u32)] +pub enum StoneHeaderVersion { V1 = 1, } @@ -28,7 +30,7 @@ pub enum Version { /// reader implementation, ensuring the container format is extensible well into /// the future #[repr(C)] -pub struct AgnosticHeader { +pub struct StoneAgnosticHeader { /// 4-bytes, BE (u32): Magic to quickly identify a stone file pub magic: [u8; 4], @@ -39,7 +41,7 @@ pub struct AgnosticHeader { pub version: [u8; 4], } -impl AgnosticHeader { +impl StoneAgnosticHeader { fn decode(mut reader: R) -> io::Result { let magic = reader.read_array()?; let data = reader.read_array()?; @@ -58,23 +60,23 @@ impl AgnosticHeader { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Header { - V1(v1::Header), +pub enum StoneHeader { + V1(StoneHeaderV1), } -impl Header { +impl StoneHeader { /// Size of the encoded header in bytes - pub const SIZE: usize = size_of::(); + pub const SIZE: usize = size_of::(); - pub fn version(&self) -> Version { + pub fn version(&self) -> StoneHeaderVersion { match self { - Header::V1(_) => Version::V1, + StoneHeader::V1(_) => StoneHeaderVersion::V1, } } pub fn num_payloads(&self) -> u16 { match self { - Header::V1(header) => header.num_payloads, + StoneHeader::V1(header) => header.num_payloads, } } @@ -82,54 +84,54 @@ impl Header { let version = u32::to_be_bytes(self.version() as u32); let data = match self { - Header::V1(v1) => v1.encode(), + StoneHeader::V1(v1) => v1.encode(), }; - AgnosticHeader { - magic: *STONE_MAGIC, + StoneAgnosticHeader { + magic: *STONE_HEADER_MAGIC, data, version, } .encode(writer) } - pub fn decode(reader: R) -> Result { - let header = AgnosticHeader::decode(reader)?; + pub fn decode(reader: R) -> Result { + let header = StoneAgnosticHeader::decode(reader)?; - if *STONE_MAGIC != header.magic { - return Err(DecodeError::InvalidMagic); + if *STONE_HEADER_MAGIC != header.magic { + return Err(StoneHeaderDecodeError::InvalidMagic); } let version = match u32::from_be_bytes(header.version) { - 1 => Version::V1, - v => return Err(DecodeError::UnknownVersion(v)), + 1 => StoneHeaderVersion::V1, + v => return Err(StoneHeaderDecodeError::UnknownVersion(v)), }; Ok(match version { - Version::V1 => Self::V1(v1::Header::decode(header.data)?), + StoneHeaderVersion::V1 => Self::V1(StoneHeaderV1::decode(header.data)?), }) } } #[derive(Debug, Error)] -pub enum DecodeError { - #[error("Header must be {} bytes long", size_of::())] +pub enum StoneHeaderDecodeError { + #[error("Header must be {} bytes long", size_of::())] NotEnoughBytes, #[error("Invalid magic")] InvalidMagic, #[error("Unknown version: {0}")] UnknownVersion(u32), #[error("v1 decode")] - V1(#[from] v1::DecodeError), + V1(#[from] StoneHeaderV1DecodeError), #[error("io")] Io(io::Error), } -impl From for DecodeError { +impl From for StoneHeaderDecodeError { fn from(error: io::Error) -> Self { match error.kind() { - io::ErrorKind::UnexpectedEof => DecodeError::NotEnoughBytes, - _ => DecodeError::Io(error), + io::ErrorKind::UnexpectedEof => StoneHeaderDecodeError::NotEnoughBytes, + _ => StoneHeaderDecodeError::Io(error), } } } diff --git a/crates/stone/src/header/v1.rs b/crates/stone/src/header/v1.rs index ff7ed32f..cdb65de6 100644 --- a/crates/stone/src/header/v1.rs +++ b/crates/stone/src/header/v1.rs @@ -11,9 +11,9 @@ const INTEGRITY_CHECK: [u8; 21] = [0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 4, 0, 0, 5, /// /// Some types are now legacy as we're going to use Ion to define them. /// -#[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FileType { +#[repr(u8)] +pub enum StoneHeaderV1FileType { /// Binary package Binary = 1, @@ -29,27 +29,28 @@ pub enum FileType { /// Header for the v1 format version #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Header { +#[repr(C)] +pub struct StoneHeaderV1 { pub num_payloads: u16, - pub file_type: FileType, + pub file_type: StoneHeaderV1FileType, } -impl Header { - pub fn decode(bytes: [u8; 24]) -> Result { +impl StoneHeaderV1 { + pub fn decode(bytes: [u8; 24]) -> Result { let (num_payloads, rest) = bytes.split_at(2); let (integrity_check, file_type) = rest.split_at(21); if integrity_check != INTEGRITY_CHECK { - return Err(DecodeError::Corrupt); + return Err(StoneHeaderV1DecodeError::Corrupt); } let num_payloads = u16::from_be_bytes(num_payloads.try_into().unwrap()); let file_type = match file_type[0] { - 1 => FileType::Binary, - 2 => FileType::Delta, - 3 => FileType::Repository, - 4 => FileType::BuildManifest, - f => return Err(DecodeError::UnknownFileType(f)), + 1 => StoneHeaderV1FileType::Binary, + 2 => StoneHeaderV1FileType::Delta, + 3 => StoneHeaderV1FileType::Repository, + 4 => StoneHeaderV1FileType::BuildManifest, + f => return Err(StoneHeaderV1DecodeError::UnknownFileType(f)), }; Ok(Self { @@ -73,7 +74,7 @@ impl Header { } #[derive(Debug, Error)] -pub enum DecodeError { +pub enum StoneHeaderV1DecodeError { #[error("Corrupt header, failed integrity check")] Corrupt, #[error("Unknown file type: {0}")] diff --git a/crates/stone/src/lib.rs b/crates/stone/src/lib.rs index 29c5846c..85ede379 100644 --- a/crates/stone/src/lib.rs +++ b/crates/stone/src/lib.rs @@ -2,93 +2,26 @@ // // SPDX-License-Identifier: MPL-2.0 -use std::io::{Read, Result, Write}; - -pub mod header; -pub mod payload; -pub mod read; -pub mod write; - -pub use self::header::Header; -pub use self::payload::Payload; -pub use self::read::{read, read_bytes, Reader}; -pub use self::write::Writer; - -pub trait ReadExt: Read { - fn read_u8(&mut self) -> Result { - let bytes = self.read_array::<1>()?; - Ok(bytes[0]) - } - - fn read_u16(&mut self) -> Result { - let bytes = self.read_array()?; - Ok(u16::from_be_bytes(bytes)) - } - - fn read_u32(&mut self) -> Result { - let bytes = self.read_array()?; - Ok(u32::from_be_bytes(bytes)) - } - - fn read_u64(&mut self) -> Result { - let bytes = self.read_array()?; - Ok(u64::from_be_bytes(bytes)) - } - - fn read_u128(&mut self) -> Result { - let bytes = self.read_array()?; - Ok(u128::from_be_bytes(bytes)) - } - - fn read_array(&mut self) -> Result<[u8; N]> { - let mut bytes = [0u8; N]; - self.read_exact(&mut bytes)?; - Ok(bytes) - } - - fn read_vec(&mut self, length: usize) -> Result> { - let mut bytes = vec![0u8; length]; - self.read_exact(&mut bytes)?; - Ok(bytes) - } - - fn read_string(&mut self, length: u64) -> Result { - let mut string = String::with_capacity(length as usize); - self.take(length).read_to_string(&mut string)?; - Ok(string) - } -} - -impl ReadExt for T {} - -pub trait WriteExt: Write { - fn write_u8(&mut self, item: u8) -> Result<()> { - self.write_array([item]) - } - - fn write_u16(&mut self, item: u16) -> Result<()> { - self.write_array(item.to_be_bytes()) - } - - fn write_u32(&mut self, item: u32) -> Result<()> { - self.write_array(item.to_be_bytes()) - } - - fn write_u64(&mut self, item: u64) -> Result<()> { - self.write_array(item.to_be_bytes()) - } - - fn write_u128(&mut self, item: u128) -> Result<()> { - self.write_array(item.to_be_bytes()) - } - - fn write_array(&mut self, bytes: [u8; N]) -> Result<()> { - self.write_all(&bytes)?; - Ok(()) - } -} - -impl WriteExt for T {} +pub(crate) mod ext; +mod header; +mod payload; +mod read; +mod write; + +pub use self::header::{ + StoneAgnosticHeader, StoneHeader, StoneHeaderDecodeError, StoneHeaderV1, StoneHeaderV1DecodeError, + StoneHeaderV1FileType, StoneHeaderVersion, STONE_HEADER_MAGIC, +}; +pub use self::payload::{ + StonePayload, StonePayloadAttributeBody, StonePayloadCompression, StonePayloadContentBody, StonePayloadDecodeError, + StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndexBody, StonePayloadKind, StonePayloadLayoutBody, + StonePayloadLayoutEntry, StonePayloadLayoutFileType, StonePayloadMetaBody, StonePayloadMetaDependency, + StonePayloadMetaKind, StonePayloadMetaTag, +}; +pub use self::read::{read, read_bytes, StoneDecodedPayload, StoneReadError, StoneReader}; +pub use self::write::{ + StoneContentWriter, StoneDigestWriter, StoneDigestWriterHasher, StoneWriteError, StoneWritePayload, StoneWriter, +}; #[cfg(test)] mod test { @@ -107,10 +40,10 @@ mod test { .unwrap() .collect::, _>>() .unwrap(); - let meta = payloads.iter().find_map(read::PayloadKind::meta).unwrap(); - let layouts = payloads.iter().find_map(read::PayloadKind::layout).unwrap(); - let indices = payloads.iter().find_map(read::PayloadKind::index).unwrap(); - let content = payloads.iter().find_map(read::PayloadKind::content).unwrap(); + let meta = payloads.iter().find_map(read::StoneDecodedPayload::meta).unwrap(); + let layouts = payloads.iter().find_map(read::StoneDecodedPayload::layout).unwrap(); + let indices = payloads.iter().find_map(read::StoneDecodedPayload::index).unwrap(); + let content = payloads.iter().find_map(read::StoneDecodedPayload::content).unwrap(); let mut content_buffer = vec![]; @@ -118,7 +51,7 @@ mod test { let mut out_stone = vec![]; let mut temp_content_buffer: Vec = vec![]; - let mut writer = Writer::new(&mut out_stone, header::v1::FileType::Binary) + let mut writer = StoneWriter::new(&mut out_stone, header::v1::StoneHeaderV1FileType::Binary) .unwrap() .with_content( Cursor::new(&mut temp_content_buffer), @@ -150,10 +83,10 @@ mod test { .unwrap() .collect::, _>>() .unwrap(); - let rt_meta = rt_payloads.iter().find_map(read::PayloadKind::meta).unwrap(); - let rt_layouts = rt_payloads.iter().find_map(read::PayloadKind::layout).unwrap(); - let rt_indices = rt_payloads.iter().find_map(read::PayloadKind::index).unwrap(); - let rt_content = rt_payloads.iter().find_map(read::PayloadKind::content).unwrap(); + let rt_meta = rt_payloads.iter().find_map(read::StoneDecodedPayload::meta).unwrap(); + let rt_layouts = rt_payloads.iter().find_map(read::StoneDecodedPayload::layout).unwrap(); + let rt_indices = rt_payloads.iter().find_map(read::StoneDecodedPayload::index).unwrap(); + let rt_content = rt_payloads.iter().find_map(read::StoneDecodedPayload::content).unwrap(); // Stored size / digest will be different since compression from boulder // isn't identical & we don't add null terminated strings diff --git a/crates/stone/src/payload/attribute.rs b/crates/stone/src/payload/attribute.rs index b37b1b55..4924ba5c 100644 --- a/crates/stone/src/payload/attribute.rs +++ b/crates/stone/src/payload/attribute.rs @@ -4,17 +4,17 @@ use std::io::{Read, Write}; -use super::{DecodeError, EncodeError, Record}; -use crate::{ReadExt, WriteExt}; +use super::{Record, StonePayloadDecodeError, StonePayloadEncodeError}; +use crate::ext::{ReadExt, WriteExt}; #[derive(Debug, Clone)] -pub struct Attribute { +pub struct StonePayloadAttributeBody { pub key: Vec, pub value: Vec, } -impl Record for Attribute { - fn decode(mut reader: R) -> Result { +impl Record for StonePayloadAttributeBody { + fn decode(mut reader: R) -> Result { let key_length = reader.read_u64()?; let value_length = reader.read_u64()?; @@ -24,7 +24,7 @@ impl Record for Attribute { Ok(Self { key, value }) } - fn encode(&self, writer: &mut W) -> Result<(), EncodeError> { + fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError> { writer.write_u64(self.key.len() as u64)?; writer.write_u64(self.value.len() as u64)?; writer.write_all(&self.key)?; diff --git a/crates/stone/src/payload/content.rs b/crates/stone/src/payload/content.rs new file mode 100644 index 00000000..ca70fec3 --- /dev/null +++ b/crates/stone/src/payload/content.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright © 2020-2024 Serpent OS Developers +// +// SPDX-License-Identifier: MPL-2.0 + +#[derive(Debug)] +pub struct StonePayloadContentBody { + pub offset: u64, +} diff --git a/crates/stone/src/payload/index.rs b/crates/stone/src/payload/index.rs index 0a482687..be31fea3 100644 --- a/crates/stone/src/payload/index.rs +++ b/crates/stone/src/payload/index.rs @@ -4,8 +4,8 @@ use std::io::{Read, Write}; -use super::{DecodeError, EncodeError, Record}; -use crate::{ReadExt, WriteExt}; +use super::{Record, StonePayloadDecodeError, StonePayloadEncodeError}; +use crate::ext::{ReadExt, WriteExt}; /// An IndexEntry (a series of sequential entries within the IndexPayload) /// record offsets to unique files within the ContentPayload when decompressed. @@ -13,7 +13,7 @@ use crate::{ReadExt, WriteExt}; /// This is used to split the file into the content store on disk before promoting /// to a transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Index { +pub struct StonePayloadIndexBody { /// Start pf the entry within the ContentPayload pub start: u64, @@ -24,8 +24,8 @@ pub struct Index { pub digest: u128, } -impl Record for Index { - fn decode(mut reader: R) -> Result { +impl Record for StonePayloadIndexBody { + fn decode(mut reader: R) -> Result { let start = reader.read_u64()?; let end = reader.read_u64()?; let digest = reader.read_u128()?; @@ -33,7 +33,7 @@ impl Record for Index { Ok(Self { start, end, digest }) } - fn encode(&self, writer: &mut W) -> Result<(), EncodeError> { + fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError> { writer.write_u64(self.start)?; writer.write_u64(self.end)?; writer.write_u128(self.digest)?; diff --git a/crates/stone/src/payload/layout.rs b/crates/stone/src/payload/layout.rs index af3a922f..46787cb9 100644 --- a/crates/stone/src/payload/layout.rs +++ b/crates/stone/src/payload/layout.rs @@ -4,14 +4,14 @@ use std::io::{Read, Write}; -use super::{DecodeError, EncodeError, Record}; -use crate::{ReadExt, WriteExt}; +use super::{StonePayloadDecodeError, StonePayloadEncodeError, Record}; +use crate::ext::{ReadExt, WriteExt}; /// Layout entries record their target file type so they can be rebuilt on /// the target installation. #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FileType { +pub enum StonePayloadLayoutFileType { /// Regular file Regular = 1, @@ -35,7 +35,7 @@ pub enum FileType { } #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Entry { +pub enum StonePayloadLayoutEntry { Regular(u128, String), Symlink(String, String), Directory(String), @@ -47,56 +47,56 @@ pub enum Entry { Socket(String), } -impl Entry { +impl StonePayloadLayoutEntry { fn source(&self) -> Vec { match self { - Entry::Regular(hash, _) => hash.to_be_bytes().to_vec(), - Entry::Symlink(source, _) => source.as_bytes().to_vec(), - Entry::Directory(_) => vec![], - Entry::CharacterDevice(_) => vec![], - Entry::BlockDevice(_) => vec![], - Entry::Fifo(_) => vec![], - Entry::Socket(_) => vec![], + StonePayloadLayoutEntry::Regular(hash, _) => hash.to_be_bytes().to_vec(), + StonePayloadLayoutEntry::Symlink(source, _) => source.as_bytes().to_vec(), + StonePayloadLayoutEntry::Directory(_) => vec![], + StonePayloadLayoutEntry::CharacterDevice(_) => vec![], + StonePayloadLayoutEntry::BlockDevice(_) => vec![], + StonePayloadLayoutEntry::Fifo(_) => vec![], + StonePayloadLayoutEntry::Socket(_) => vec![], } } pub fn target(&self) -> &str { match self { - Entry::Regular(_, target) => target, - Entry::Symlink(_, target) => target, - Entry::Directory(target) => target, - Entry::CharacterDevice(target) => target, - Entry::BlockDevice(target) => target, - Entry::Fifo(target) => target, - Entry::Socket(target) => target, + StonePayloadLayoutEntry::Regular(_, target) => target, + StonePayloadLayoutEntry::Symlink(_, target) => target, + StonePayloadLayoutEntry::Directory(target) => target, + StonePayloadLayoutEntry::CharacterDevice(target) => target, + StonePayloadLayoutEntry::BlockDevice(target) => target, + StonePayloadLayoutEntry::Fifo(target) => target, + StonePayloadLayoutEntry::Socket(target) => target, } } fn file_type(&self) -> u8 { match self { - Entry::Regular(..) => 1, - Entry::Symlink(..) => 2, - Entry::Directory(_) => 3, - Entry::CharacterDevice(_) => 4, - Entry::BlockDevice(_) => 5, - Entry::Fifo(_) => 6, - Entry::Socket(_) => 7, + StonePayloadLayoutEntry::Regular(..) => 1, + StonePayloadLayoutEntry::Symlink(..) => 2, + StonePayloadLayoutEntry::Directory(_) => 3, + StonePayloadLayoutEntry::CharacterDevice(_) => 4, + StonePayloadLayoutEntry::BlockDevice(_) => 5, + StonePayloadLayoutEntry::Fifo(_) => 6, + StonePayloadLayoutEntry::Socket(_) => 7, } } } // TODO: Strong types these fields #[derive(Debug, Clone, PartialEq, Eq)] -pub struct Layout { +pub struct StonePayloadLayoutBody { pub uid: u32, pub gid: u32, pub mode: u32, pub tag: u32, - pub entry: Entry, + pub entry: StonePayloadLayoutEntry, } -impl Record for Layout { - fn decode(mut reader: R) -> Result { +impl Record for StonePayloadLayoutBody { + fn decode(mut reader: R) -> Result { let uid = reader.read_u32()?; let gid = reader.read_u32()?; let mode = reader.read_u32()?; @@ -107,14 +107,14 @@ impl Record for Layout { let sanitize = |s: String| s.trim_end_matches('\0').to_owned(); let file_type = match reader.read_u8()? { - 1 => FileType::Regular, - 2 => FileType::Symlink, - 3 => FileType::Directory, - 4 => FileType::CharacterDevice, - 5 => FileType::BlockDevice, - 6 => FileType::Fifo, - 7 => FileType::Socket, - t => return Err(DecodeError::UnknownFileType(t)), + 1 => StonePayloadLayoutFileType::Regular, + 2 => StonePayloadLayoutFileType::Symlink, + 3 => StonePayloadLayoutFileType::Directory, + 4 => StonePayloadLayoutFileType::CharacterDevice, + 5 => StonePayloadLayoutFileType::BlockDevice, + 6 => StonePayloadLayoutFileType::Fifo, + 7 => StonePayloadLayoutFileType::Socket, + t => return Err(StonePayloadDecodeError::UnknownFileType(t)), }; let _padding = reader.read_array::<11>()?; @@ -122,16 +122,16 @@ impl Record for Layout { // Make the layout entry *usable* let entry = match file_type { // BUG: boulder stores xxh128 as le bytes not be - FileType::Regular => { + StonePayloadLayoutFileType::Regular => { let source = reader.read_vec(source_length as usize)?; let hash = u128::from_be_bytes(source.try_into().unwrap()); - Entry::Regular(hash, sanitize(reader.read_string(target_length as u64)?)) + StonePayloadLayoutEntry::Regular(hash, sanitize(reader.read_string(target_length as u64)?)) } - FileType::Symlink => Entry::Symlink( + StonePayloadLayoutFileType::Symlink => StonePayloadLayoutEntry::Symlink( sanitize(reader.read_string(source_length as u64)?), sanitize(reader.read_string(target_length as u64)?), ), - FileType::Directory => Entry::Directory(sanitize(reader.read_string(target_length as u64)?)), + StonePayloadLayoutFileType::Directory => StonePayloadLayoutEntry::Directory(sanitize(reader.read_string(target_length as u64)?)), _ => { if source_length > 0 { let _ = reader.read_vec(source_length as usize); @@ -149,7 +149,7 @@ impl Record for Layout { }) } - fn encode(&self, writer: &mut W) -> Result<(), EncodeError> { + fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError> { writer.write_u32(self.uid)?; writer.write_u32(self.gid)?; writer.write_u32(self.mode)?; diff --git a/crates/stone/src/payload/meta.rs b/crates/stone/src/payload/meta.rs index af0db71c..e275c612 100644 --- a/crates/stone/src/payload/meta.rs +++ b/crates/stone/src/payload/meta.rs @@ -4,23 +4,23 @@ use std::io::{Read, Write}; -use super::{DecodeError, EncodeError, Record}; -use crate::{ReadExt, WriteExt}; +use super::{Record, StonePayloadDecodeError, StonePayloadEncodeError}; +use crate::ext::{ReadExt, WriteExt}; /// The Meta payload contains a series of sequential records with /// strong types and context tags, i.e. their use such as Name. /// These record all metadata for every .stone packages and provide /// no content #[derive(Debug, Clone, PartialEq, Eq)] -pub struct Meta { - pub tag: Tag, - pub kind: Kind, +pub struct StonePayloadMetaBody { + pub tag: StonePayloadMetaTag, + pub kind: StonePayloadMetaKind, } #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] #[strum(serialize_all = "lowercase")] -pub enum Dependency { +pub enum StonePayloadMetaDependency { /// Just the plain name of a package #[strum(serialize = "name")] PackageName = 0, @@ -54,7 +54,7 @@ pub enum Dependency { #[repr(u8)] #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Kind { +pub enum StonePayloadMetaKind { Int8(i8), Uint8(u8), Int16(i16), @@ -64,34 +64,34 @@ pub enum Kind { Int64(i64), Uint64(u64), String(String), - Dependency(Dependency, String), - Provider(Dependency, String), + Dependency(StonePayloadMetaDependency, String), + Provider(StonePayloadMetaDependency, String), } -impl Kind { +impl StonePayloadMetaKind { fn size(&self) -> usize { match self { - Kind::Int8(_) => size_of::(), - Kind::Uint8(_) => size_of::(), - Kind::Int16(_) => size_of::(), - Kind::Uint16(_) => size_of::(), - Kind::Int32(_) => size_of::(), - Kind::Uint32(_) => size_of::(), - Kind::Int64(_) => size_of::(), - Kind::Uint64(_) => size_of::(), + StonePayloadMetaKind::Int8(_) => size_of::(), + StonePayloadMetaKind::Uint8(_) => size_of::(), + StonePayloadMetaKind::Int16(_) => size_of::(), + StonePayloadMetaKind::Uint16(_) => size_of::(), + StonePayloadMetaKind::Int32(_) => size_of::(), + StonePayloadMetaKind::Uint32(_) => size_of::(), + StonePayloadMetaKind::Int64(_) => size_of::(), + StonePayloadMetaKind::Uint64(_) => size_of::(), // nul terminator - Kind::String(s) => s.len() + 1, + StonePayloadMetaKind::String(s) => s.len() + 1, // Plus dep size & nul terminator - Kind::Dependency(_, s) => s.len() + 2, + StonePayloadMetaKind::Dependency(_, s) => s.len() + 2, // Plus dep size & nul terminator - Kind::Provider(_, s) => s.len() + 2, + StonePayloadMetaKind::Provider(_, s) => s.len() + 2, } } } #[repr(u16)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Tag { +pub enum StonePayloadMetaTag { // Name of the package Name = 1, // Architecture of the package @@ -135,48 +135,48 @@ pub enum Tag { } /// Helper to decode a dependency's encoded kind -fn decode_dependency(i: u8) -> Result { +fn decode_dependency(i: u8) -> Result { let result = match i { - 0 => Dependency::PackageName, - 1 => Dependency::SharedLibrary, - 2 => Dependency::PkgConfig, - 3 => Dependency::Interpreter, - 4 => Dependency::CMake, - 5 => Dependency::Python, - 6 => Dependency::Binary, - 7 => Dependency::SystemBinary, - 8 => Dependency::PkgConfig32, - _ => return Err(DecodeError::UnknownDependency(i)), + 0 => StonePayloadMetaDependency::PackageName, + 1 => StonePayloadMetaDependency::SharedLibrary, + 2 => StonePayloadMetaDependency::PkgConfig, + 3 => StonePayloadMetaDependency::Interpreter, + 4 => StonePayloadMetaDependency::CMake, + 5 => StonePayloadMetaDependency::Python, + 6 => StonePayloadMetaDependency::Binary, + 7 => StonePayloadMetaDependency::SystemBinary, + 8 => StonePayloadMetaDependency::PkgConfig32, + _ => return Err(StonePayloadDecodeError::UnknownDependency(i)), }; Ok(result) } -impl Record for Meta { - fn decode(mut reader: R) -> Result { +impl Record for StonePayloadMetaBody { + fn decode(mut reader: R) -> Result { let length = reader.read_u32()?; let tag = match reader.read_u16()? { - 1 => Tag::Name, - 2 => Tag::Architecture, - 3 => Tag::Version, - 4 => Tag::Summary, - 5 => Tag::Description, - 6 => Tag::Homepage, - 7 => Tag::SourceID, - 8 => Tag::Depends, - 9 => Tag::Provides, - 10 => Tag::Conflicts, - 11 => Tag::Release, - 12 => Tag::License, - 13 => Tag::BuildRelease, - 14 => Tag::PackageURI, - 15 => Tag::PackageHash, - 16 => Tag::PackageSize, - 17 => Tag::BuildDepends, - 18 => Tag::SourceURI, - 19 => Tag::SourcePath, - 20 => Tag::SourceRef, - t => return Err(DecodeError::UnknownMetaTag(t)), + 1 => StonePayloadMetaTag::Name, + 2 => StonePayloadMetaTag::Architecture, + 3 => StonePayloadMetaTag::Version, + 4 => StonePayloadMetaTag::Summary, + 5 => StonePayloadMetaTag::Description, + 6 => StonePayloadMetaTag::Homepage, + 7 => StonePayloadMetaTag::SourceID, + 8 => StonePayloadMetaTag::Depends, + 9 => StonePayloadMetaTag::Provides, + 10 => StonePayloadMetaTag::Conflicts, + 11 => StonePayloadMetaTag::Release, + 12 => StonePayloadMetaTag::License, + 13 => StonePayloadMetaTag::BuildRelease, + 14 => StonePayloadMetaTag::PackageURI, + 15 => StonePayloadMetaTag::PackageHash, + 16 => StonePayloadMetaTag::PackageSize, + 17 => StonePayloadMetaTag::BuildDepends, + 18 => StonePayloadMetaTag::SourceURI, + 19 => StonePayloadMetaTag::SourcePath, + 20 => StonePayloadMetaTag::SourceRef, + t => return Err(StonePayloadDecodeError::UnknownMetaTag(t)), }; let kind = reader.read_u8()?; @@ -186,44 +186,44 @@ impl Record for Meta { let sanitize = |s: String| s.trim_end_matches('\0').to_owned(); let kind = match kind { - 1 => Kind::Int8(reader.read_u8()? as i8), - 2 => Kind::Uint8(reader.read_u8()?), - 3 => Kind::Int16(reader.read_u16()? as i16), - 4 => Kind::Uint16(reader.read_u16()?), - 5 => Kind::Int32(reader.read_u32()? as i32), - 6 => Kind::Uint32(reader.read_u32()?), - 7 => Kind::Int64(reader.read_u64()? as i64), - 8 => Kind::Uint64(reader.read_u64()?), - 9 => Kind::String(sanitize(reader.read_string(length as u64)?)), - 10 => Kind::Dependency( + 1 => StonePayloadMetaKind::Int8(reader.read_u8()? as i8), + 2 => StonePayloadMetaKind::Uint8(reader.read_u8()?), + 3 => StonePayloadMetaKind::Int16(reader.read_u16()? as i16), + 4 => StonePayloadMetaKind::Uint16(reader.read_u16()?), + 5 => StonePayloadMetaKind::Int32(reader.read_u32()? as i32), + 6 => StonePayloadMetaKind::Uint32(reader.read_u32()?), + 7 => StonePayloadMetaKind::Int64(reader.read_u64()? as i64), + 8 => StonePayloadMetaKind::Uint64(reader.read_u64()?), + 9 => StonePayloadMetaKind::String(sanitize(reader.read_string(length as u64)?)), + 10 => StonePayloadMetaKind::Dependency( // DependencyKind u8 subtracted from length decode_dependency(reader.read_u8()?)?, sanitize(reader.read_string(length as u64 - 1)?), ), - 11 => Kind::Provider( + 11 => StonePayloadMetaKind::Provider( // DependencyKind u8 subtracted from length decode_dependency(reader.read_u8()?)?, sanitize(reader.read_string(length as u64 - 1)?), ), - k => return Err(DecodeError::UnknownMetaKind(k)), + k => return Err(StonePayloadDecodeError::UnknownMetaKind(k)), }; Ok(Self { tag, kind }) } - fn encode(&self, writer: &mut W) -> Result<(), EncodeError> { + fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError> { let kind = match self.kind { - Kind::Int8(_) => 1, - Kind::Uint8(_) => 2, - Kind::Int16(_) => 3, - Kind::Uint16(_) => 4, - Kind::Int32(_) => 5, - Kind::Uint32(_) => 6, - Kind::Int64(_) => 7, - Kind::Uint64(_) => 8, - Kind::String(_) => 9, - Kind::Dependency(_, _) => 10, - Kind::Provider(_, _) => 11, + StonePayloadMetaKind::Int8(_) => 1, + StonePayloadMetaKind::Uint8(_) => 2, + StonePayloadMetaKind::Int16(_) => 3, + StonePayloadMetaKind::Uint16(_) => 4, + StonePayloadMetaKind::Int32(_) => 5, + StonePayloadMetaKind::Uint32(_) => 6, + StonePayloadMetaKind::Int64(_) => 7, + StonePayloadMetaKind::Uint64(_) => 8, + StonePayloadMetaKind::String(_) => 9, + StonePayloadMetaKind::Dependency(_, _) => 10, + StonePayloadMetaKind::Provider(_, _) => 11, }; writer.write_u32(self.kind.size() as u32)?; @@ -233,19 +233,19 @@ impl Record for Meta { writer.write_array::<1>([0])?; match &self.kind { - Kind::Int8(i) => writer.write_u8(*i as u8)?, - Kind::Uint8(i) => writer.write_u8(*i)?, - Kind::Int16(i) => writer.write_u16(*i as u16)?, - Kind::Uint16(i) => writer.write_u16(*i)?, - Kind::Int32(i) => writer.write_u32(*i as u32)?, - Kind::Uint32(i) => writer.write_u32(*i)?, - Kind::Int64(i) => writer.write_u64(*i as u64)?, - Kind::Uint64(i) => writer.write_u64(*i)?, - Kind::String(s) => { + StonePayloadMetaKind::Int8(i) => writer.write_u8(*i as u8)?, + StonePayloadMetaKind::Uint8(i) => writer.write_u8(*i)?, + StonePayloadMetaKind::Int16(i) => writer.write_u16(*i as u16)?, + StonePayloadMetaKind::Uint16(i) => writer.write_u16(*i)?, + StonePayloadMetaKind::Int32(i) => writer.write_u32(*i as u32)?, + StonePayloadMetaKind::Uint32(i) => writer.write_u32(*i)?, + StonePayloadMetaKind::Int64(i) => writer.write_u64(*i as u64)?, + StonePayloadMetaKind::Uint64(i) => writer.write_u64(*i)?, + StonePayloadMetaKind::String(s) => { writer.write_all(s.as_bytes())?; writer.write_u8(b'\0')?; } - Kind::Dependency(dep, s) | Kind::Provider(dep, s) => { + StonePayloadMetaKind::Dependency(dep, s) | StonePayloadMetaKind::Provider(dep, s) => { writer.write_u8(*dep as u8)?; writer.write_all(s.as_bytes())?; writer.write_u8(b'\0')?; diff --git a/crates/stone/src/payload/mod.rs b/crates/stone/src/payload/mod.rs index 186bcd70..b3857fc5 100644 --- a/crates/stone/src/payload/mod.rs +++ b/crates/stone/src/payload/mod.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: MPL-2.0 mod attribute; +mod content; mod index; pub mod layout; pub mod meta; @@ -11,15 +12,18 @@ use std::io::{self, Read, Write}; use thiserror::Error; -pub use self::attribute::Attribute; -pub use self::index::Index; -pub use self::layout::Layout; -pub use self::meta::Meta; -use crate::{ReadExt, WriteExt}; +use crate::ext::{ReadExt, WriteExt}; +pub use self::attribute::StonePayloadAttributeBody; +pub use self::content::StonePayloadContentBody; +pub use self::index::StonePayloadIndexBody; +pub use self::layout::{StonePayloadLayoutBody, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; +pub use self::meta::{StonePayloadMetaBody, StonePayloadMetaDependency, StonePayloadMetaKind, StonePayloadMetaTag}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] +#[strum(serialize_all = "kebab-case")] #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Kind { +pub enum StonePayloadKind { // The Metadata store Meta = 1, // File store, i.e. hash indexed @@ -34,9 +38,10 @@ pub enum Kind { Dumb = 6, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] +#[strum(serialize_all = "kebab-case")] #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Compression { +pub enum StonePayloadCompression { // Payload has no compression None = 1, // Payload uses ZSTD compression @@ -44,21 +49,19 @@ pub enum Compression { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Header { +#[repr(C)] +pub struct StonePayloadHeader { pub stored_size: u64, pub plain_size: u64, pub checksum: [u8; 8], pub num_records: usize, pub version: u16, - pub kind: Kind, - pub compression: Compression, + pub kind: StonePayloadKind, + pub compression: StonePayloadCompression, } -impl Header { - /// Size of the encoded payload header in bytes - pub const SIZE: usize = 8 + 8 + 8 + 4 + 2 + 1 + 1; - - pub fn decode(mut reader: R) -> Result { +impl StonePayloadHeader { + pub fn decode(mut reader: R) -> Result { let stored_size = reader.read_u64()?; let plain_size = reader.read_u64()?; let checksum = reader.read_array()?; @@ -66,19 +69,19 @@ impl Header { let version = reader.read_u16()?; let kind = match reader.read_u8()? { - 1 => Kind::Meta, - 2 => Kind::Content, - 3 => Kind::Layout, - 4 => Kind::Index, - 5 => Kind::Attributes, - 6 => Kind::Dumb, - k => return Err(DecodeError::UnknownKind(k)), + 1 => StonePayloadKind::Meta, + 2 => StonePayloadKind::Content, + 3 => StonePayloadKind::Layout, + 4 => StonePayloadKind::Index, + 5 => StonePayloadKind::Attributes, + 6 => StonePayloadKind::Dumb, + k => return Err(StonePayloadDecodeError::UnknownKind(k)), }; let compression = match reader.read_u8()? { - 1 => Compression::None, - 2 => Compression::Zstd, - d => return Err(DecodeError::UnknownCompression(d)), + 1 => StonePayloadCompression::None, + 2 => StonePayloadCompression::Zstd, + d => return Err(StonePayloadDecodeError::UnknownCompression(d)), }; Ok(Self { @@ -92,7 +95,7 @@ impl Header { }) } - pub fn encode(&self, writer: &mut W) -> Result<(), EncodeError> { + pub fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError> { writer.write_u64(self.stored_size)?; writer.write_u64(self.plain_size)?; writer.write_array(self.checksum)?; @@ -105,13 +108,16 @@ impl Header { } } -pub trait Record: Sized { - fn decode(reader: R) -> Result; - fn encode(&self, writer: &mut W) -> Result<(), EncodeError>; +pub(crate) trait Record: Sized { + fn decode(reader: R) -> Result; + fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError>; fn size(&self) -> usize; } -pub fn decode_records(mut reader: R, num_records: usize) -> Result, DecodeError> { +pub(crate) fn decode_records( + mut reader: R, + num_records: usize, +) -> Result, StonePayloadDecodeError> { let mut records = Vec::with_capacity(num_records); for _ in 0..num_records { @@ -121,25 +127,28 @@ pub fn decode_records(mut reader: R, num_records: usize) -> Ok(records) } -pub fn encode_records(writer: &mut W, records: &[T]) -> Result<(), EncodeError> { +pub(crate) fn encode_records( + writer: &mut W, + records: &[T], +) -> Result<(), StonePayloadEncodeError> { for record in records { record.encode(writer)?; } Ok(()) } -pub fn records_total_size(records: &[T]) -> usize { +pub(crate) fn records_total_size(records: &[T]) -> usize { records.iter().map(T::size).sum() } #[derive(Debug, Clone)] -pub struct Payload { - pub header: Header, +pub struct StonePayload { + pub header: StonePayloadHeader, pub body: T, } #[derive(Debug, Error)] -pub enum DecodeError { +pub enum StonePayloadDecodeError { #[error("Unknown header type: {0}")] UnknownKind(u8), #[error("Unknown header compression: {0}")] @@ -157,7 +166,7 @@ pub enum DecodeError { } #[derive(Debug, Error)] -pub enum EncodeError { +pub enum StonePayloadEncodeError { #[error("io")] Io(#[from] io::Error), } diff --git a/crates/stone/src/read/mod.rs b/crates/stone/src/read/mod.rs index 75495744..29ef94cf 100644 --- a/crates/stone/src/read/mod.rs +++ b/crates/stone/src/read/mod.rs @@ -1,73 +1,118 @@ // SPDX-FileCopyrightText: Copyright © 2020-2024 Serpent OS Developers // // SPDX-License-Identifier: MPL-2.0 +#![allow(dead_code)] use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; use thiserror::Error; -use crate::payload::{Attribute, Compression, Index, Layout, Meta}; -use crate::{header, Payload}; -use crate::{payload, Header}; +use crate::{ + payload, StoneHeader, StoneHeaderDecodeError, StonePayload, StonePayloadAttributeBody, StonePayloadCompression, + StonePayloadContentBody, StonePayloadDecodeError, StonePayloadHeader, StonePayloadIndexBody, StonePayloadKind, + StonePayloadLayoutBody, StonePayloadMetaBody, +}; use self::zstd::Zstd; mod digest; mod zstd; -pub fn read(mut reader: R) -> Result, Error> { - let header = Header::decode(&mut reader).map_err(Error::HeaderDecode)?; +pub fn read(mut reader: R) -> Result, StoneReadError> { + let header = StoneHeader::decode(&mut reader).map_err(StoneReadError::HeaderDecode)?; - Ok(Reader { + Ok(StoneReader { header, reader, hasher: digest::Hasher::new(), + + #[cfg(feature = "ffi")] + next_payload: 0, }) } -pub fn read_bytes(bytes: &[u8]) -> Result>, Error> { +pub fn read_bytes(bytes: &[u8]) -> Result>, StoneReadError> { read(Cursor::new(bytes)) } -pub struct Reader { - pub header: Header, +pub struct StoneReader { + pub header: StoneHeader, reader: R, hasher: digest::Hasher, + + #[cfg(feature = "ffi")] + next_payload: u16, } -impl Reader { - pub fn payloads(&mut self) -> Result> + '_, Error> { - if self.reader.stream_position()? != Header::SIZE as u64 { +impl StoneReader { + pub fn payloads( + &mut self, + ) -> Result> + '_, StoneReadError> { + if self.reader.stream_position()? != StoneHeader::SIZE as u64 { // Rewind to start of payloads - self.reader.seek(SeekFrom::Start(Header::SIZE as u64))?; + self.reader.seek(SeekFrom::Start(StoneHeader::SIZE as u64))?; + } + + #[cfg(feature = "ffi")] + { + self.next_payload = self.header.num_payloads(); } Ok((0..self.header.num_payloads()) - .flat_map(|_| PayloadKind::decode(&mut self.reader, &mut self.hasher).transpose())) + .flat_map(|_| StoneDecodedPayload::decode(&mut self.reader, &mut self.hasher).transpose())) } - pub fn unpack_content(&mut self, content: &Payload, writer: &mut W) -> Result<(), Error> + pub fn unpack_content( + &mut self, + content: &StonePayload, + writer: &mut W, + ) -> Result<(), StoneReadError> where W: Write, { self.reader.seek(SeekFrom::Start(content.body.offset))?; - self.hasher.reset(); - let mut hashed = digest::Reader::new(&mut self.reader, &mut self.hasher); - let mut framed = (&mut hashed).take(content.header.stored_size); - let mut reader = PayloadReader::new(&mut framed, content.header.compression)?; + unpack_payload(self, writer, &content.header) + } +} - io::copy(&mut reader, writer)?; +#[cfg(feature = "ffi")] +impl StoneReader { + pub fn next_payload_header(&mut self) -> Result, StoneReadError> { + if self.next_payload < self.header.num_payloads() { + let header = StonePayloadHeader::decode(&mut self.reader)?; - // Validate checksum - validate_checksum(&self.hasher, &content.header)?; + self.next_payload += 1; - Ok(()) + Ok(Some(header)) + } else { + Ok(None) + } + } + + pub fn unpack_payload(&mut self, header: &StonePayloadHeader, writer: &mut W) -> Result<(), StoneReadError> + where + W: Write, + { + unpack_payload(self, writer, header) } } -#[derive(Debug, Clone, Copy)] -pub struct Content { - offset: u64, +fn unpack_payload( + reader: &mut StoneReader, + writer: &mut W, + header: &StonePayloadHeader, +) -> Result<(), StoneReadError> { + reader.hasher.reset(); + + let mut hashed = digest::Reader::new(&mut reader.reader, &mut reader.hasher); + let mut framed = (&mut hashed).take(header.stored_size); + + io::copy(&mut PayloadReader::new(&mut framed, header.compression)?, writer)?; + + // Validate checksum + validate_checksum(&reader.hasher, header)?; + + Ok(()) } enum PayloadReader { @@ -76,10 +121,10 @@ enum PayloadReader { } impl PayloadReader { - fn new(reader: R, compression: Compression) -> Result { + fn new(reader: R, compression: StonePayloadCompression) -> Result { Ok(match compression { - Compression::None => PayloadReader::Plain(reader), - Compression::Zstd => PayloadReader::Zstd(Zstd::new(reader)?), + StonePayloadCompression::None => PayloadReader::Plain(reader), + StonePayloadCompression::Zstd => PayloadReader::Zstd(Zstd::new(reader)?), }) } } @@ -94,78 +139,78 @@ impl Read for PayloadReader { } #[derive(Debug)] -pub enum PayloadKind { - Meta(Payload>), - Attributes(Payload>), - Layout(Payload>), - Index(Payload>), - Content(Payload), +pub enum StoneDecodedPayload { + Meta(StonePayload>), + Attributes(StonePayload>), + Layout(StonePayload>), + Index(StonePayload>), + Content(StonePayload), } -impl PayloadKind { - fn decode(mut reader: R, hasher: &mut digest::Hasher) -> Result, Error> { - match payload::Header::decode(&mut reader) { +impl StoneDecodedPayload { + fn decode(mut reader: R, hasher: &mut digest::Hasher) -> Result, StoneReadError> { + match StonePayloadHeader::decode(&mut reader) { Ok(header) => { hasher.reset(); let mut hashed = digest::Reader::new(&mut reader, hasher); let mut framed = (&mut hashed).take(header.stored_size); let payload = match header.kind { - payload::Kind::Meta => PayloadKind::Meta(Payload { + StonePayloadKind::Meta => StoneDecodedPayload::Meta(StonePayload { header, body: payload::decode_records( PayloadReader::new(&mut framed, header.compression)?, header.num_records, )?, }), - payload::Kind::Layout => PayloadKind::Layout(Payload { + StonePayloadKind::Layout => StoneDecodedPayload::Layout(StonePayload { header, body: payload::decode_records( PayloadReader::new(&mut framed, header.compression)?, header.num_records, )?, }), - payload::Kind::Index => PayloadKind::Index(Payload { + StonePayloadKind::Index => StoneDecodedPayload::Index(StonePayload { header, body: payload::decode_records( PayloadReader::new(&mut framed, header.compression)?, header.num_records, )?, }), - payload::Kind::Attributes => PayloadKind::Attributes(Payload { + StonePayloadKind::Attributes => StoneDecodedPayload::Attributes(StonePayload { header, body: payload::decode_records( PayloadReader::new(&mut framed, header.compression)?, header.num_records, )?, }), - payload::Kind::Content => { + StonePayloadKind::Content => { let offset = reader.stream_position()?; // Skip past, these are read by user later reader.seek(SeekFrom::Current(header.stored_size as i64))?; - PayloadKind::Content(Payload { + StoneDecodedPayload::Content(StonePayload { header, - body: Content { offset }, + body: StonePayloadContentBody { offset }, }) } - payload::Kind::Dumb => unimplemented!("??"), + StonePayloadKind::Dumb => unimplemented!("??"), }; // Validate hash for non-content payloads - if !matches!(header.kind, payload::Kind::Content) { + if !matches!(header.kind, StonePayloadKind::Content) { validate_checksum(hasher, &header)?; } Ok(Some(payload)) } - Err(payload::DecodeError::Io(error)) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(None), - Err(error) => Err(Error::PayloadDecode(error)), + Err(StonePayloadDecodeError::Io(error)) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(None), + Err(error) => Err(StoneReadError::PayloadDecode(error)), } } - pub fn meta(&self) -> Option<&Payload>> { + pub fn meta(&self) -> Option<&StonePayload>> { if let Self::Meta(meta) = self { Some(meta) } else { @@ -173,7 +218,7 @@ impl PayloadKind { } } - pub fn attributes(&self) -> Option<&Payload>> { + pub fn attributes(&self) -> Option<&StonePayload>> { if let Self::Attributes(attributes) = self { Some(attributes) } else { @@ -181,7 +226,7 @@ impl PayloadKind { } } - pub fn layout(&self) -> Option<&Payload>> { + pub fn layout(&self) -> Option<&StonePayload>> { if let Self::Layout(layouts) = self { Some(layouts) } else { @@ -189,7 +234,7 @@ impl PayloadKind { } } - pub fn index(&self) -> Option<&Payload>> { + pub fn index(&self) -> Option<&StonePayload>> { if let Self::Index(indices) = self { Some(indices) } else { @@ -197,7 +242,7 @@ impl PayloadKind { } } - pub fn content(&self) -> Option<&Payload> { + pub fn content(&self) -> Option<&StonePayload> { if let Self::Content(content) = self { Some(content) } else { @@ -206,25 +251,25 @@ impl PayloadKind { } } -fn validate_checksum(hasher: &digest::Hasher, header: &payload::Header) -> Result<(), Error> { +fn validate_checksum(hasher: &digest::Hasher, header: &StonePayloadHeader) -> Result<(), StoneReadError> { let got = hasher.digest(); let expected = u64::from_be_bytes(header.checksum); if got != expected { - Err(Error::PayloadChecksum { got, expected }) + Err(StoneReadError::PayloadChecksum { got, expected }) } else { Ok(()) } } #[derive(Debug, Error)] -pub enum Error { +pub enum StoneReadError { #[error("Multiple content payloads not allowed")] MultipleContent, #[error("header decode")] - HeaderDecode(#[from] header::DecodeError), + HeaderDecode(#[from] StoneHeaderDecodeError), #[error("payload decode")] - PayloadDecode(#[from] payload::DecodeError), + PayloadDecode(#[from] StonePayloadDecodeError), #[error("payload checksum mismatch: got {got:02x}, expected {expected:02x}")] PayloadChecksum { got: u64, expected: u64 }, #[error("io")] @@ -235,7 +280,7 @@ pub enum Error { mod test { use xxhash_rust::xxh3::xxh3_128; - use crate::payload::layout::Entry; + use crate::{StoneHeaderVersion, StonePayloadLayoutEntry}; use super::*; @@ -248,14 +293,14 @@ mod test { #[test] fn read_header() { let stone = read_bytes(&BASH_TEST_STONE).expect("valid stone"); - assert_eq!(stone.header.version(), header::Version::V1); + assert_eq!(stone.header.version(), StoneHeaderVersion::V1); } #[test] fn read_bash_completion() { let mut stone = read_bytes(include_bytes!("../../../../test/bash-completion-2.11-1-1-x86_64.stone")).expect("valid stone"); - assert_eq!(stone.header.version(), header::Version::V1); + assert_eq!(stone.header.version(), StoneHeaderVersion::V1); let payloads = stone .payloads() @@ -265,22 +310,26 @@ mod test { let mut unpacked_content = vec![]; - if let Some(content) = payloads.iter().find_map(PayloadKind::content) { + if let Some(content) = payloads.iter().find_map(StoneDecodedPayload::content) { stone .unpack_content(content, &mut unpacked_content) .expect("valid content"); - for index in payloads.iter().filter_map(PayloadKind::index).flat_map(|p| &p.body) { + for index in payloads + .iter() + .filter_map(StoneDecodedPayload::index) + .flat_map(|p| &p.body) + { let content = &unpacked_content[index.start as usize..index.end as usize]; let digest = xxh3_128(content); assert_eq!(digest, index.digest); payloads .iter() - .filter_map(PayloadKind::layout) + .filter_map(StoneDecodedPayload::layout) .flat_map(|p| &p.body) .find(|layout| { - if let Entry::Regular(digest, _) = &layout.entry { + if let StonePayloadLayoutEntry::Regular(digest, _) = &layout.entry { return *digest == index.digest; } false diff --git a/crates/stone/src/write.rs b/crates/stone/src/write.rs index f655ee88..c7c1c547 100644 --- a/crates/stone/src/write.rs +++ b/crates/stone/src/write.rs @@ -6,37 +6,39 @@ use std::io::{self, Read, Seek, SeekFrom, Write}; use thiserror::Error; use crate::{ - header, - payload::{self, Attribute, Index, Layout, Meta}, - Header, + payload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StonePayloadAttributeBody, StonePayloadCompression, + StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndexBody, StonePayloadKind, StonePayloadLayoutBody, + StonePayloadMetaBody, }; +pub use self::digest::{StoneDigestWriter, StoneDigestWriterHasher}; + pub mod digest; mod zstd; -pub struct Writer { +pub struct StoneWriter { writer: W, content: T, - file_type: header::v1::FileType, + file_type: StoneHeaderV1FileType, payloads: Vec, - payload_hasher: digest::Hasher, + payload_hasher: StoneDigestWriterHasher, // TODO: Allow plain encoding? encoder: zstd::Encoder, } -impl Writer { - pub fn new(writer: W, file_type: header::v1::FileType) -> Result { +impl StoneWriter { + pub fn new(writer: W, file_type: StoneHeaderV1FileType) -> Result { Ok(Self { writer, content: (), file_type, payloads: vec![], - payload_hasher: digest::Hasher::new(), + payload_hasher: StoneDigestWriterHasher::new(), encoder: zstd::Encoder::new()?, }) } - pub fn add_payload<'a>(&mut self, payload: impl Into>) -> Result<(), Error> { + pub fn add_payload<'a>(&mut self, payload: impl Into>) -> Result<(), StoneWriteError> { self.payloads.push(encode_payload( payload.into().into(), &mut self.payload_hasher, @@ -50,20 +52,20 @@ impl Writer { buffer: B, pledged_size: Option, num_workers: u32, - ) -> Result>, Error> { + ) -> Result>, StoneWriteError> { let mut encoder = zstd::Encoder::new()?; encoder.set_pledged_size(pledged_size)?; encoder.set_num_workers(num_workers)?; - Ok(Writer { + Ok(StoneWriter { writer: self.writer, - content: Content { + content: StoneContentWriter { buffer, plain_size: 0, stored_size: 0, indices: vec![], - index_hasher: digest::Hasher::new(), - buffer_hasher: digest::Hasher::new(), + index_hasher: StoneDigestWriterHasher::new(), + buffer_hasher: StoneDigestWriterHasher::new(), encoder, }, file_type: self.file_type, @@ -73,17 +75,17 @@ impl Writer { }) } - pub fn finalize(mut self) -> Result<(), Error> { + pub fn finalize(mut self) -> Result<(), StoneWriteError> { finalize::<_, io::Empty>(&mut self.writer, self.file_type, self.payloads, None) } } -impl Writer> +impl StoneWriter> where W: Write, B: Read + Write + Seek, { - pub fn add_payload<'a>(&mut self, payload: impl Into>) -> Result<(), Error> { + pub fn add_payload<'a>(&mut self, payload: impl Into>) -> Result<(), StoneWriteError> { self.payloads.push(encode_payload( payload.into().into(), &mut self.payload_hasher, @@ -92,7 +94,7 @@ where Ok(()) } - pub fn add_content(&mut self, content: &mut R) -> Result<(), Error> { + pub fn add_content(&mut self, content: &mut R) -> Result<(), StoneWriteError> { // Reset index hasher for this file self.content.index_hasher.reset(); @@ -106,9 +108,9 @@ where // // Bytes -> index digest -> compression -> buffer checksum -> buffer let mut payload_checksum_writer = - digest::Writer::new(&mut self.content.buffer, &mut self.content.buffer_hasher); + StoneDigestWriter::new(&mut self.content.buffer, &mut self.content.buffer_hasher); let mut zstd_writer = zstd::Writer::new(&mut payload_checksum_writer, &mut self.content.encoder); - let mut index_digest_writer = digest::Writer::new(&mut zstd_writer, &mut self.content.index_hasher); + let mut index_digest_writer = StoneDigestWriter::new(&mut zstd_writer, &mut self.content.index_hasher); io::copy(content, &mut index_digest_writer)?; @@ -127,15 +129,15 @@ where let end = self.content.plain_size; // Add index data - self.content.indices.push(Index { start, end, digest }); + self.content.indices.push(StonePayloadIndexBody { start, end, digest }); Ok(()) } - pub fn finalize(mut self) -> Result<(), Error> { + pub fn finalize(mut self) -> Result<(), StoneWriteError> { // Finish frame & get content payload checksum let checksum = { - let mut writer = digest::Writer::new(&mut self.content.buffer, &mut self.content.buffer_hasher); + let mut writer = StoneDigestWriter::new(&mut self.content.buffer, &mut self.content.buffer_hasher); self.content.encoder.finish(&mut writer)?; writer.flush()?; self.content.stored_size += writer.bytes as u64; @@ -158,37 +160,37 @@ where } } -pub struct Content { +pub struct StoneContentWriter { buffer: B, plain_size: u64, stored_size: u64, - indices: Vec, + indices: Vec, /// Used to generate un-compressed digest of file /// contents used for [`Index`] - index_hasher: digest::Hasher, + index_hasher: StoneDigestWriterHasher, /// Used to generate compressed digest of file /// contents used for content payload header - buffer_hasher: digest::Hasher, + buffer_hasher: StoneDigestWriterHasher, encoder: zstd::Encoder, } struct EncodedPayload { - header: payload::Header, + header: StonePayloadHeader, content: Vec, } -pub enum Payload<'a> { - Meta(&'a [Meta]), - Attributes(&'a [Attribute]), - Layout(&'a [Layout]), +pub enum StoneWritePayload<'a> { + Meta(&'a [StonePayloadMetaBody]), + Attributes(&'a [StonePayloadAttributeBody]), + Layout(&'a [StonePayloadLayoutBody]), } -impl<'a> From> for InnerPayload<'a> { - fn from(payload: Payload<'a>) -> Self { +impl<'a> From> for InnerPayload<'a> { + fn from(payload: StoneWritePayload<'a>) -> Self { match payload { - Payload::Meta(payload) => InnerPayload::Meta(payload), - Payload::Attributes(payload) => InnerPayload::Attributes(payload), - Payload::Layout(payload) => InnerPayload::Layout(payload), + StoneWritePayload::Meta(payload) => InnerPayload::Meta(payload), + StoneWritePayload::Attributes(payload) => InnerPayload::Attributes(payload), + StoneWritePayload::Layout(payload) => InnerPayload::Layout(payload), } } } @@ -197,10 +199,10 @@ impl<'a> From> for InnerPayload<'a> { /// doesn't support passing in `Index` payloads /// since it's a side-effect of [`Writer::add_content`] enum InnerPayload<'a> { - Meta(&'a [Meta]), - Attributes(&'a [Attribute]), - Layout(&'a [Layout]), - Index(&'a [Index]), + Meta(&'a [StonePayloadMetaBody]), + Attributes(&'a [StonePayloadAttributeBody]), + Layout(&'a [StonePayloadLayoutBody]), + Index(&'a [StonePayloadIndexBody]), } impl InnerPayload<'_> { @@ -222,7 +224,7 @@ impl InnerPayload<'_> { } } - fn encode(&self, writer: &mut W) -> Result<(), Error> { + fn encode(&self, writer: &mut W) -> Result<(), StoneWriteError> { match self { InnerPayload::Meta(records) => payload::encode_records(writer, records)?, InnerPayload::Attributes(records) => payload::encode_records(writer, records)?, @@ -232,39 +234,39 @@ impl InnerPayload<'_> { Ok(()) } - fn kind(&self) -> payload::Kind { + fn kind(&self) -> StonePayloadKind { match self { - InnerPayload::Meta(_) => payload::Kind::Meta, - InnerPayload::Attributes(_) => payload::Kind::Attributes, - InnerPayload::Layout(_) => payload::Kind::Layout, - InnerPayload::Index(_) => payload::Kind::Index, + InnerPayload::Meta(_) => StonePayloadKind::Meta, + InnerPayload::Attributes(_) => StonePayloadKind::Attributes, + InnerPayload::Layout(_) => StonePayloadKind::Layout, + InnerPayload::Index(_) => StonePayloadKind::Index, } } } -impl<'a> From<&'a [Meta]> for Payload<'a> { - fn from(payload: &'a [Meta]) -> Self { +impl<'a> From<&'a [StonePayloadMetaBody]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadMetaBody]) -> Self { Self::Meta(payload) } } -impl<'a> From<&'a [Attribute]> for Payload<'a> { - fn from(payload: &'a [Attribute]) -> Self { +impl<'a> From<&'a [StonePayloadAttributeBody]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadAttributeBody]) -> Self { Self::Attributes(payload) } } -impl<'a> From<&'a [Layout]> for Payload<'a> { - fn from(payload: &'a [Layout]) -> Self { +impl<'a> From<&'a [StonePayloadLayoutBody]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadLayoutBody]) -> Self { Self::Layout(payload) } } fn encode_payload( payload: InnerPayload<'_>, - hasher: &mut digest::Hasher, + hasher: &mut StoneDigestWriterHasher, encoder: &mut zstd::Encoder, -) -> Result { +) -> Result { // Reset hasher (it's used across all payloads) hasher.reset(); // Set pledged size @@ -273,7 +275,7 @@ fn encode_payload( let mut content = vec![]; // Checksum is on compressed body so we wrap it inside zstd writer - let mut hashed_writer = digest::Writer::new(&mut content, hasher); + let mut hashed_writer = StoneDigestWriter::new(&mut content, hasher); let mut zstd_writer = zstd::Writer::new(&mut hashed_writer, encoder); payload.encode(&mut zstd_writer)?; @@ -284,14 +286,14 @@ fn encode_payload( let stored_size = hashed_writer.bytes as u64; - let header = payload::Header { + let header = StonePayloadHeader { stored_size, plain_size, checksum: hasher.digest().to_be_bytes(), num_records: payload.num_records(), version: 1, kind: payload.kind(), - compression: payload::Compression::Zstd, + compression: StonePayloadCompression::Zstd, }; Ok(EncodedPayload { header, content }) @@ -299,12 +301,12 @@ fn encode_payload( fn finalize( writer: &mut W, - file_type: header::v1::FileType, + file_type: StoneHeaderV1FileType, payloads: Vec, - content: Option<(Content, u64)>, -) -> Result<(), Error> { + content: Option<(StoneContentWriter, u64)>, +) -> Result<(), StoneWriteError> { // Write header - Header::V1(header::v1::Header { + StoneHeader::V1(StoneHeaderV1 { num_payloads: payloads.len() as u16 + content.is_some().then_some(1).unwrap_or_default(), file_type, }) @@ -318,14 +320,14 @@ fn finalize( // Write content payload header + buffer if let Some((mut content, checksum)) = content { - payload::Header { + StonePayloadHeader { stored_size: content.stored_size, plain_size: content.plain_size, checksum: checksum.to_be_bytes(), num_records: 0, version: 1, - kind: payload::Kind::Content, - compression: payload::Compression::Zstd, + kind: StonePayloadKind::Content, + compression: StonePayloadCompression::Zstd, } .encode(writer)?; // Seek to beginning & copy content buffer @@ -339,9 +341,9 @@ fn finalize( } #[derive(Debug, Error)] -pub enum Error { +pub enum StoneWriteError { #[error("payload encode")] - PayloadEncode(#[from] payload::EncodeError), + PayloadEncode(#[from] StonePayloadEncodeError), #[error("io")] Io(#[from] io::Error), } diff --git a/crates/stone/src/write/digest.rs b/crates/stone/src/write/digest.rs index cf59b3b9..524555b3 100644 --- a/crates/stone/src/write/digest.rs +++ b/crates/stone/src/write/digest.rs @@ -6,19 +6,19 @@ use std::io::Write; use xxhash_rust::xxh3::Xxh3; -pub type Hasher = Xxh3; +pub type StoneDigestWriterHasher = Xxh3; -pub struct Writer<'a, W: Write> { +pub struct StoneDigestWriter<'a, W: Write> { inner: W, - hasher: &'a mut Hasher, + hasher: &'a mut StoneDigestWriterHasher, pub bytes: usize, } -impl<'a, W> Writer<'a, W> +impl<'a, W> StoneDigestWriter<'a, W> where W: Write, { - pub fn new(writer: W, hasher: &'a mut Hasher) -> Self { + pub fn new(writer: W, hasher: &'a mut StoneDigestWriterHasher) -> Self { Self { inner: writer, hasher, @@ -27,7 +27,7 @@ where } } -impl Write for Writer<'_, W> +impl Write for StoneDigestWriter<'_, W> where W: Write, { diff --git a/moss/src/cli/extract.rs b/moss/src/cli/extract.rs index c1e28548..7213c8d9 100644 --- a/moss/src/cli/extract.rs +++ b/moss/src/cli/extract.rs @@ -11,7 +11,7 @@ use std::{ use clap::{arg, ArgMatches, Command}; use fs_err::{self as fs, File}; use moss::package::{self, MissingMetaFieldError}; -use stone::{payload::layout, read::PayloadKind}; +use stone::{StoneDecodedPayload, StonePayloadLayoutEntry, StoneReadError}; use thiserror::{self, Error}; use tui::{ProgressBar, ProgressStyle}; @@ -43,9 +43,12 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { let mut reader = stone::read(rdr).map_err(Error::Format)?; let payloads = reader.payloads()?.collect::, _>>()?; - let content = payloads.iter().find_map(PayloadKind::content); - let layouts = payloads.iter().find_map(PayloadKind::layout); - let meta = payloads.iter().find_map(PayloadKind::meta).ok_or(Error::MissingMeta)?; + let content = payloads.iter().find_map(StoneDecodedPayload::content); + let layouts = payloads.iter().find_map(StoneDecodedPayload::layout); + let meta = payloads + .iter() + .find_map(StoneDecodedPayload::meta) + .ok_or(Error::MissingMeta)?; let pkg = package::Meta::from_stone_payload(&meta.body).map_err(Error::MalformedMeta)?; let extraction_root = PathBuf::from(pkg.id().to_string()); @@ -73,7 +76,7 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { // Extract all indices from the `.stoneContent` into hash-indexed unique files payloads .iter() - .filter_map(PayloadKind::index) + .filter_map(StoneDecodedPayload::index) .flat_map(|p| &p.body) .map(|idx| { // Split file reader over index range @@ -95,7 +98,7 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { if let Some(layouts) = layouts { for layout in &layouts.body { match &layout.entry { - layout::Entry::Regular(id, target) => { + StonePayloadLayoutEntry::Regular(id, target) => { let store_path = content_store.join(format!("{id:02x}")); let target_disk = extraction_root.join("usr").join(target); @@ -107,7 +110,7 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { // link from CA store fs::hard_link(store_path, target_disk)?; } - layout::Entry::Symlink(source, target) => { + StonePayloadLayoutEntry::Symlink(source, target) => { let target_disk = extraction_root.join("usr").join(target); let directory_target = target_disk.parent().unwrap(); @@ -117,7 +120,7 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { // join the link path to the directory target for relative joinery symlink(source, target_disk)?; } - layout::Entry::Directory(target) => { + StonePayloadLayoutEntry::Directory(target) => { let target_disk = extraction_root.join("usr").join(target); // TODO: Fix perms! fs::create_dir_all(target_disk)?; @@ -146,5 +149,5 @@ pub enum Error { IO(#[from] std::io::Error), #[error("stone format")] - Format(#[from] stone::read::Error), + Format(#[from] StoneReadError), } diff --git a/moss/src/cli/index.rs b/moss/src/cli/index.rs index c52dbc64..9fb20237 100644 --- a/moss/src/cli/index.rs +++ b/moss/src/cli/index.rs @@ -16,6 +16,7 @@ use moss::{ }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use sha2::{Digest, Sha256}; +use stone::{StoneHeaderV1FileType, StoneReadError, StoneWriteError, StoneWriter}; use thiserror::Error; use tui::{MultiProgress, ProgressBar, ProgressStyle, Styled}; @@ -95,7 +96,7 @@ fn write_index(dir: &Path, map: BTreeMap, total_progress: & let mut file = fs::File::create(dir.join("stone.index"))?; - let mut writer = stone::Writer::new(&mut file, stone::header::v1::FileType::Repository)?; + let mut writer = StoneWriter::new(&mut file, StoneHeaderV1FileType::Repository)?; for (_, meta) in map { let payload = meta.to_stone_payload(); @@ -193,10 +194,10 @@ pub enum Error { Io(#[from] io::Error), #[error("stone read")] - StoneRead(#[from] stone::read::Error), + StoneRead(#[from] StoneReadError), #[error("stone write")] - StoneWrite(#[from] stone::write::Error), + StoneWrite(#[from] StoneWriteError), #[error("package {0} has two files with the same release {1}")] DuplicateRelease(package::Name, u64), diff --git a/moss/src/cli/info.rs b/moss/src/cli/info.rs index 729e2cc5..70cc2252 100644 --- a/moss/src/cli/info.rs +++ b/moss/src/cli/info.rs @@ -10,7 +10,7 @@ use moss::{ package::Flags, Installation, Package, Provider, }; -use stone::payload::layout; +use stone::StonePayloadLayoutEntry; use thiserror::Error; use tui::{Styled, TermSize}; use vfs::tree::BlitFile; @@ -179,8 +179,8 @@ fn print_files(vfs: vfs::Tree) { let path = file.path(); let meta = match &file.layout.entry { - layout::Entry::Regular(hash, _) => Some(format!(" ({hash:2x})")), - layout::Entry::Symlink(source, _) => Some(format!(" -> {source}")), + StonePayloadLayoutEntry::Regular(hash, _) => Some(format!(" ({hash:2x})")), + StonePayloadLayoutEntry::Symlink(source, _) => Some(format!(" -> {source}")), _ => None, }; diff --git a/moss/src/cli/inspect.rs b/moss/src/cli/inspect.rs index c6dad34f..a206ca86 100644 --- a/moss/src/cli/inspect.rs +++ b/moss/src/cli/inspect.rs @@ -5,9 +5,7 @@ use clap::{arg, ArgMatches, Command}; use fs_err::File; use std::path::PathBuf; -use stone::payload::layout; -use stone::payload::meta; -use stone::read::PayloadKind; +use stone::{StoneDecodedPayload, StonePayloadLayoutEntry, StonePayloadMetaKind, StonePayloadMetaTag, StoneReadError}; use thiserror::Error; const COLUMN_WIDTH: usize = 20; @@ -50,30 +48,30 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { let mut cnfls = vec![]; match payload { - PayloadKind::Layout(l) => layouts = l.body, - PayloadKind::Meta(meta) => { + StoneDecodedPayload::Layout(l) => layouts = l.body, + StoneDecodedPayload::Meta(meta) => { println!(); for record in meta.body { let name = format!("{:?}", record.tag); match &record.kind { - meta::Kind::Provider(k, p) if record.tag == meta::Tag::Provides => { + StonePayloadMetaKind::Provider(k, p) if record.tag == StonePayloadMetaTag::Provides => { provs.push(format!("{k}({p})")); } - meta::Kind::Provider(k, p) if record.tag == meta::Tag::Conflicts => { + StonePayloadMetaKind::Provider(k, p) if record.tag == StonePayloadMetaTag::Conflicts => { cnfls.push(format!("{k}({p})")); } - meta::Kind::Dependency(k, d) => { + StonePayloadMetaKind::Dependency(k, d) => { deps.push(format!("{k}({d})")); } - meta::Kind::String(s) => { + StonePayloadMetaKind::String(s) => { println!("{name:COLUMN_WIDTH$} : {s}"); } - meta::Kind::Int64(i) => { + StonePayloadMetaKind::Int64(i) => { println!("{name:COLUMN_WIDTH$} : {i}"); } - meta::Kind::Uint64(i) => { + StonePayloadMetaKind::Uint64(i) => { println!("{name:COLUMN_WIDTH$} : {i}"); } _ => { @@ -108,13 +106,13 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { println!("\n{:COLUMN_WIDTH$} :", "Layout entries"); for layout in layouts { match layout.entry { - layout::Entry::Regular(hash, target) => { + StonePayloadLayoutEntry::Regular(hash, target) => { println!(" - /usr/{target} - [Regular] {hash:032x}"); } - layout::Entry::Directory(target) => { + StonePayloadLayoutEntry::Directory(target) => { println!(" - /usr/{target} [Directory]"); } - layout::Entry::Symlink(source, target) => { + StonePayloadLayoutEntry::Symlink(source, target) => { println!(" - /usr/{target} -> {source} [Symlink]"); } _ => unreachable!(), @@ -133,5 +131,5 @@ pub enum Error { IO(#[from] std::io::Error), #[error("stone format")] - Format(#[from] stone::read::Error), + Format(#[from] StoneReadError), } diff --git a/moss/src/client/boot.rs b/moss/src/client/boot.rs index 452f122f..b1ad10bc 100644 --- a/moss/src/client/boot.rs +++ b/moss/src/client/boot.rs @@ -18,7 +18,7 @@ use blsforme::{ use fnmatch::Pattern; use fs_err as fs; use itertools::Itertools; -use stone::payload::layout::{self, Layout}; +use stone::{StonePayloadLayoutBody, StonePayloadLayoutEntry}; use thiserror::{self, Error}; use crate::{db, package::Id, Installation, State}; @@ -51,7 +51,7 @@ pub enum Error { #[derive(Debug)] struct KernelCandidate { path: PathBuf, - _layout: Layout, + _layout: StonePayloadLayoutBody, } impl AsRef for KernelCandidate { @@ -61,13 +61,17 @@ impl AsRef for KernelCandidate { } /// From a given set of input paths, produce a set of match pairs -/// This is applied against the given system root -fn kernel_files_from_state<'a>(layouts: &'a [(Id, Layout)], pattern: &'a Pattern) -> Vec { +/// NOTE: This only works for a *new* blit and doesn't retroactively +/// sync old kernels! +fn kernel_files_from_state<'a>( + layouts: &'a [(Id, StonePayloadLayoutBody)], + pattern: &'a Pattern, +) -> Vec { let mut kernel_entries = vec![]; for (_, path) in layouts.iter() { match &path.entry { - layout::Entry::Regular(_, target) => { + StonePayloadLayoutEntry::Regular(_, target) => { if pattern.match_path(target).is_some() { kernel_entries.push(KernelCandidate { path: PathBuf::from("usr").join(target), @@ -75,7 +79,7 @@ fn kernel_files_from_state<'a>(layouts: &'a [(Id, Layout)], pattern: &'a Pattern }); } } - layout::Entry::Symlink(_, target) => { + StonePayloadLayoutEntry::Symlink(_, target) => { if pattern.match_path(target).is_some() { kernel_entries.push(KernelCandidate { path: PathBuf::from("usr").join(target), @@ -93,13 +97,13 @@ fn kernel_files_from_state<'a>(layouts: &'a [(Id, Layout)], pattern: &'a Pattern /// Find bootloader assets in the new state fn boot_files_from_new_state<'a>( install: &Installation, - layouts: &'a [(Id, Layout)], + layouts: &'a [(Id, StonePayloadLayoutBody)], pattern: &'a Pattern, ) -> Vec { let mut rets = vec![]; for (_, path) in layouts.iter() { - if let layout::Entry::Regular(_, target) = &path.entry { + if let StonePayloadLayoutEntry::Regular(_, target) = &path.entry { if pattern.match_path(target).is_some() { rets.push(install.root.join("usr").join(target)); } @@ -110,7 +114,7 @@ fn boot_files_from_new_state<'a>( } /// Grab all layouts for the provided state, mapped to package id -fn layouts_for_state(client: &Client, state: &State) -> Result, db::Error> { +fn layouts_for_state(client: &Client, state: &State) -> Result, db::Error> { client.layout_db.query(state.selections.iter().map(|s| &s.package)) } diff --git a/moss/src/client/cache.rs b/moss/src/client/cache.rs index bf4e4842..c8d30578 100644 --- a/moss/src/client/cache.rs +++ b/moss/src/client/cache.rs @@ -12,12 +12,11 @@ use std::{ }; use futures_util::StreamExt; +use stone::{StoneDecodedPayload, StonePayloadIndexBody, StoneReadError}; use thiserror::Error; use tokio::io::AsyncWriteExt; use url::Url; -use stone::{payload, read::PayloadKind}; - use crate::{package, request, Installation}; /// Synchronized set of assets that are currently being @@ -124,7 +123,7 @@ pub struct Download { /// Upon fetch completion we have this unpacked asset bound with /// an open reader pub struct UnpackedAsset { - pub payloads: Vec, + pub payloads: Vec, } impl Download { @@ -186,7 +185,7 @@ impl Download { let payloads = reader.payloads()?.collect::, _>>()?; let indices = payloads .iter() - .filter_map(PayloadKind::index) + .filter_map(StoneDecodedPayload::index) .flat_map(|p| &p.body) .collect::>(); @@ -198,7 +197,7 @@ impl Download { let content = payloads .iter() - .find_map(PayloadKind::content) + .find_map(StoneDecodedPayload::content) .ok_or(Error::MissingContent)?; let content_file = OpenOptions::new() @@ -258,7 +257,7 @@ impl Download { } /// Returns true if all assets already exist in the installation -fn check_assets_exist(indices: &[&payload::Index], installation: &Installation) -> bool { +fn check_assets_exist(indices: &[&StonePayloadIndexBody], installation: &Installation) -> bool { indices.iter().all(|index| { let path = asset_path(installation, &format!("{:02x}", index.digest)); path.exists() @@ -306,7 +305,7 @@ pub enum Error { #[error("Malformed download hash: {0}")] MalformedHash(String), #[error("stone format")] - Format(#[from] stone::read::Error), + Format(#[from] StoneReadError), #[error("invalid url")] InvalidUrl(#[from] url::ParseError), #[error("request")] diff --git a/moss/src/client/mod.rs b/moss/src/client/mod.rs index b3932ebd..b6aaa20e 100644 --- a/moss/src/client/mod.rs +++ b/moss/src/client/mod.rs @@ -26,7 +26,7 @@ use nix::{ unistd::{close, linkat, mkdir, symlinkat}, }; use postblit::TriggerScope; -use stone::{payload::layout, read::PayloadKind}; +use stone::{StoneDecodedPayload, StonePayloadLayoutBody, StonePayloadLayoutEntry}; use thiserror::Error; use tui::{MultiProgress, ProgressBar, ProgressStyle, Styled}; use vfs::tree::{builder::TreeBuilder, BlitFile, Element}; @@ -557,7 +557,7 @@ impl Client { layout_db.batch_add(cached.iter().flat_map(|(p, u)| { u.payloads .iter() - .flat_map(PayloadKind::layout) + .flat_map(StoneDecodedPayload::layout) .flat_map(|p| p.body.as_slice()) .map(|layout| (&p.id, layout)) }))?; @@ -603,7 +603,7 @@ impl Client { /// /// This functionality is core to all moss filesystem transactions, forming the entire /// staging logic. For all the [`crate::package::Id`] present in the staging state, - /// query their stored [`stone::payload::Layout`] and cache into a [`vfs::Tree`]. + /// query their stored [`StonePayloadLayoutBody`] and cache into a [`vfs::Tree`]. /// /// The new `/usr` filesystem is written in optimal order to a staging tree by making /// use of the "at" family of functions (`mkdirat`, `linkat`, etc) with relative directory @@ -697,7 +697,7 @@ impl Client { /// * `item` - New inode being recorded fn blit_element_item(&self, parent: RawFd, cache: RawFd, subpath: &str, item: &PendingFile) -> Result<(), Error> { match &item.layout.entry { - layout::Entry::Regular(id, _) => { + StonePayloadLayoutEntry::Regular(id, _) => { let hash = format!("{id:02x}"); let directory = if hash.len() >= 10 { PathBuf::from(&hash[..2]).join(&hash[2..4]).join(&hash[4..6]) @@ -740,18 +740,18 @@ impl Client { } } } - layout::Entry::Symlink(source, _) => { + StonePayloadLayoutEntry::Symlink(source, _) => { symlinkat(source.as_str(), Some(parent), subpath)?; } - layout::Entry::Directory(_) => { + StonePayloadLayoutEntry::Directory(_) => { mkdirat(parent, subpath, Mode::from_bits_truncate(item.layout.mode))?; } // unimplemented - layout::Entry::CharacterDevice(_) => todo!(), - layout::Entry::BlockDevice(_) => todo!(), - layout::Entry::Fifo(_) => todo!(), - layout::Entry::Socket(_) => todo!(), + StonePayloadLayoutEntry::CharacterDevice(_) => todo!(), + StonePayloadLayoutEntry::BlockDevice(_) => todo!(), + StonePayloadLayoutEntry::Fifo(_) => todo!(), + StonePayloadLayoutEntry::Socket(_) => todo!(), }; Ok(()) @@ -840,15 +840,15 @@ pub struct PendingFile { pub id: package::Id, /// Corresponding layout entry, describing the inode - pub layout: layout::Layout, + pub layout: StonePayloadLayoutBody, } impl BlitFile for PendingFile { /// Match internal kind to minimalist vfs kind fn kind(&self) -> vfs::tree::Kind { match &self.layout.entry { - layout::Entry::Symlink(source, _) => vfs::tree::Kind::Symlink(source.clone()), - layout::Entry::Directory(_) => vfs::tree::Kind::Directory, + StonePayloadLayoutEntry::Symlink(source, _) => vfs::tree::Kind::Symlink(source.clone()), + StonePayloadLayoutEntry::Directory(_) => vfs::tree::Kind::Directory, _ => vfs::tree::Kind::Regular, } } @@ -861,13 +861,13 @@ impl BlitFile for PendingFile { /// Resolve the target path, including the missing `/usr` prefix fn path(&self) -> String { let result = match &self.layout.entry { - layout::Entry::Regular(_, target) => target.clone(), - layout::Entry::Symlink(_, target) => target.clone(), - layout::Entry::Directory(target) => target.clone(), - layout::Entry::CharacterDevice(target) => target.clone(), - layout::Entry::BlockDevice(target) => target.clone(), - layout::Entry::Fifo(target) => target.clone(), - layout::Entry::Socket(target) => target.clone(), + StonePayloadLayoutEntry::Regular(_, target) => target.clone(), + StonePayloadLayoutEntry::Symlink(_, target) => target.clone(), + StonePayloadLayoutEntry::Directory(target) => target.clone(), + StonePayloadLayoutEntry::CharacterDevice(target) => target.clone(), + StonePayloadLayoutEntry::BlockDevice(target) => target.clone(), + StonePayloadLayoutEntry::Fifo(target) => target.clone(), + StonePayloadLayoutEntry::Socket(target) => target.clone(), }; vfs::path::join("/usr", &result) @@ -877,13 +877,13 @@ impl BlitFile for PendingFile { fn cloned_to(&self, path: String) -> Self { let mut new = self.clone(); new.layout.entry = match &self.layout.entry { - layout::Entry::Regular(source, _) => layout::Entry::Regular(*source, path), - layout::Entry::Symlink(source, _) => layout::Entry::Symlink(source.clone(), path), - layout::Entry::Directory(_) => layout::Entry::Directory(path), - layout::Entry::CharacterDevice(_) => layout::Entry::CharacterDevice(path), - layout::Entry::BlockDevice(_) => layout::Entry::BlockDevice(path), - layout::Entry::Fifo(_) => layout::Entry::Fifo(path), - layout::Entry::Socket(_) => layout::Entry::Socket(path), + StonePayloadLayoutEntry::Regular(source, _) => StonePayloadLayoutEntry::Regular(*source, path), + StonePayloadLayoutEntry::Symlink(source, _) => StonePayloadLayoutEntry::Symlink(source.clone(), path), + StonePayloadLayoutEntry::Directory(_) => StonePayloadLayoutEntry::Directory(path), + StonePayloadLayoutEntry::CharacterDevice(_) => StonePayloadLayoutEntry::CharacterDevice(path), + StonePayloadLayoutEntry::BlockDevice(_) => StonePayloadLayoutEntry::BlockDevice(path), + StonePayloadLayoutEntry::Fifo(_) => StonePayloadLayoutEntry::Fifo(path), + StonePayloadLayoutEntry::Socket(_) => StonePayloadLayoutEntry::Socket(path), }; new } @@ -893,12 +893,12 @@ impl From for PendingFile { fn from(value: String) -> Self { PendingFile { id: Default::default(), - layout: layout::Layout { + layout: StonePayloadLayoutBody { uid: 0, gid: 0, mode: 0o755, tag: 0, - entry: layout::Entry::Directory(value), + entry: StonePayloadLayoutEntry::Directory(value), }, } } diff --git a/moss/src/client/verify.rs b/moss/src/client/verify.rs index 95bbb57d..f2f8c6bc 100644 --- a/moss/src/client/verify.rs +++ b/moss/src/client/verify.rs @@ -7,7 +7,7 @@ use std::{collections::BTreeSet, fmt, io, path::PathBuf}; use itertools::Itertools; use fs_err as fs; -use stone::{payload::layout, write::digest}; +use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutEntry}; use tui::{ dialoguer::{theme::ColorfulTheme, Confirm}, ProgressBar, ProgressStyle, Styled, @@ -29,7 +29,7 @@ pub fn verify(client: &Client, yes: bool, verbose: bool) -> Result<(), client::E let unique_assets = layouts .into_iter() .filter_map(|(package, layout)| { - if let layout::Entry::Regular(hash, file) = layout.entry { + if let StonePayloadLayoutEntry::Regular(hash, file) = layout.entry { Some((format!("{hash:02x}"), (package, file))) } else { None @@ -38,7 +38,7 @@ pub fn verify(client: &Client, yes: bool, verbose: bool) -> Result<(), client::E .into_group_map(); let mut issues = vec![]; - let mut hasher = digest::Hasher::new(); + let mut hasher = StoneDigestWriterHasher::new(); let pb = ProgressBar::new(unique_assets.len() as u64) .with_message("Verifying") @@ -77,7 +77,7 @@ pub fn verify(client: &Client, yes: bool, verbose: bool) -> Result<(), client::E hasher.reset(); - let mut digest_writer = digest::Writer::new(io::sink(), &mut hasher); + let mut digest_writer = StoneDigestWriter::new(io::sink(), &mut hasher); let mut file = fs::File::open(&path)?; // Copy bytes to null sink so we don't diff --git a/moss/src/db/layout/mod.rs b/moss/src/db/layout/mod.rs index 9202a4cd..a023bccc 100644 --- a/moss/src/db/layout/mod.rs +++ b/moss/src/db/layout/mod.rs @@ -6,8 +6,7 @@ use diesel::prelude::*; use diesel::{Connection as _, SqliteConnection}; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use std::collections::BTreeSet; - -use stone::payload; +use stone::{StonePayloadLayoutBody, StonePayloadLayoutEntry}; use crate::package; @@ -38,7 +37,7 @@ impl Database { pub fn query<'a>( &self, packages: impl IntoIterator, - ) -> Result, Error> { + ) -> Result, Error> { self.conn.exec(|conn| { let packages = packages.into_iter().map(AsRef::::as_ref).collect::>(); @@ -59,7 +58,7 @@ impl Database { }) } - pub fn all(&self) -> Result, Error> { + pub fn all(&self) -> Result, Error> { self.conn.exec(|conn| { model::layout::table .select(model::Layout::as_select()) @@ -84,13 +83,13 @@ impl Database { }) } - pub fn add(&self, package: &package::Id, layout: &payload::Layout) -> Result<(), Error> { + pub fn add(&self, package: &package::Id, layout: &StonePayloadLayoutBody) -> Result<(), Error> { self.batch_add(vec![(package, layout)]) } pub fn batch_add<'a>( &self, - layouts: impl IntoIterator, + layouts: impl IntoIterator, ) -> Result<(), Error> { self.conn.exclusive_tx(|tx| { let mut ids = vec![]; @@ -149,12 +148,12 @@ fn batch_remove_impl(packages: &[&str], tx: &mut SqliteConnection) -> Result<(), Ok(()) } -fn map_layout(result: QueryResult) -> Result<(package::Id, payload::Layout), Error> { +fn map_layout(result: QueryResult) -> Result<(package::Id, StonePayloadLayoutBody), Error> { let row = result?; let entry = decode_entry(row.entry_type, row.entry_value1, row.entry_value2).ok_or(Error::LayoutEntryDecode)?; - let layout = payload::Layout { + let layout = StonePayloadLayoutBody { uid: row.uid as u32, gid: row.gid as u32, mode: row.mode as u32, @@ -169,37 +168,33 @@ fn decode_entry( entry_type: String, entry_value1: Option, entry_value2: Option, -) -> Option { - use payload::layout::Entry; - +) -> Option { match entry_type.as_str() { "regular" => { let hash = entry_value1?.parse::().ok()?; let name = entry_value2?; - Some(Entry::Regular(hash, name)) + Some(StonePayloadLayoutEntry::Regular(hash, name)) } - "symlink" => Some(Entry::Symlink(entry_value1?, entry_value2?)), - "directory" => Some(Entry::Directory(entry_value1?)), - "character-device" => Some(Entry::CharacterDevice(entry_value1?)), - "block-device" => Some(Entry::BlockDevice(entry_value1?)), - "fifo" => Some(Entry::Fifo(entry_value1?)), - "socket" => Some(Entry::Socket(entry_value1?)), + "symlink" => Some(StonePayloadLayoutEntry::Symlink(entry_value1?, entry_value2?)), + "directory" => Some(StonePayloadLayoutEntry::Directory(entry_value1?)), + "character-device" => Some(StonePayloadLayoutEntry::CharacterDevice(entry_value1?)), + "block-device" => Some(StonePayloadLayoutEntry::BlockDevice(entry_value1?)), + "fifo" => Some(StonePayloadLayoutEntry::Fifo(entry_value1?)), + "socket" => Some(StonePayloadLayoutEntry::Socket(entry_value1?)), _ => None, } } -fn encode_entry(entry: payload::layout::Entry) -> (&'static str, Option, Option) { - use payload::layout::Entry; - +fn encode_entry(entry: StonePayloadLayoutEntry) -> (&'static str, Option, Option) { match entry { - Entry::Regular(hash, name) => ("regular", Some(hash.to_string()), Some(name)), - Entry::Symlink(a, b) => ("symlink", Some(a), Some(b)), - Entry::Directory(name) => ("directory", Some(name), None), - Entry::CharacterDevice(name) => ("character-device", Some(name), None), - Entry::BlockDevice(name) => ("block-device", Some(name), None), - Entry::Fifo(name) => ("fifo", Some(name), None), - Entry::Socket(name) => ("socket", Some(name), None), + StonePayloadLayoutEntry::Regular(hash, name) => ("regular", Some(hash.to_string()), Some(name)), + StonePayloadLayoutEntry::Symlink(a, b) => ("symlink", Some(a), Some(b)), + StonePayloadLayoutEntry::Directory(name) => ("directory", Some(name), None), + StonePayloadLayoutEntry::CharacterDevice(name) => ("character-device", Some(name), None), + StonePayloadLayoutEntry::BlockDevice(name) => ("block-device", Some(name), None), + StonePayloadLayoutEntry::Fifo(name) => ("fifo", Some(name), None), + StonePayloadLayoutEntry::Socket(name) => ("socket", Some(name), None), } } @@ -241,7 +236,7 @@ mod model { #[cfg(test)] mod test { - use stone::read::PayloadKind; + use stone::StoneDecodedPayload; use super::*; @@ -256,7 +251,7 @@ mod test { let payloads = stone.payloads().unwrap().collect::, _>>().unwrap(); let layouts = payloads .iter() - .filter_map(PayloadKind::layout) + .filter_map(StoneDecodedPayload::layout) .flat_map(|p| &p.body) .map(|layout| (package::Id::from("test".to_owned()), layout)) .collect::>(); diff --git a/moss/src/db/meta/mod.rs b/moss/src/db/meta/mod.rs index c0090e20..53b8c93f 100644 --- a/moss/src/db/meta/mod.rs +++ b/moss/src/db/meta/mod.rs @@ -467,7 +467,7 @@ mod model { #[cfg(test)] mod test { - use stone::read::PayloadKind; + use stone::StoneDecodedPayload; use crate::dependency::Kind; @@ -482,7 +482,7 @@ mod test { let mut stone = stone::read_bytes(bash_completion).unwrap(); let payloads = stone.payloads().unwrap().collect::, _>>().unwrap(); - let meta_payload = payloads.iter().find_map(PayloadKind::meta).unwrap(); + let meta_payload = payloads.iter().find_map(StoneDecodedPayload::meta).unwrap(); let meta = Meta::from_stone_payload(&meta_payload.body).unwrap(); let id = package::Id::from("test".to_owned()); @@ -527,7 +527,7 @@ mod test { let mut stone = stone::read_bytes(italian_pizza).unwrap(); let payloads = stone.payloads().unwrap().collect::, _>>().unwrap(); - let meta_payload = payloads.iter().find_map(PayloadKind::meta).unwrap(); + let meta_payload = payloads.iter().find_map(StoneDecodedPayload::meta).unwrap(); let meta = Meta::from_stone_payload(&meta_payload.body).unwrap(); db.add(package::Id::from(meta.id()), meta.clone()).unwrap(); diff --git a/moss/src/dependency.rs b/moss/src/dependency.rs index 13c31eca..43f61fdb 100644 --- a/moss/src/dependency.rs +++ b/moss/src/dependency.rs @@ -23,7 +23,7 @@ use std::str::FromStr; use derive_more::Display; -use stone::payload; +use stone::StonePayloadMetaDependency; use thiserror::Error; /// Every dependency @@ -62,24 +62,24 @@ pub enum Kind { } /// Convert payload dependency types to our internal representation -impl From for Kind { - fn from(dependency: payload::meta::Dependency) -> Self { +impl From for Kind { + fn from(dependency: StonePayloadMetaDependency) -> Self { match dependency { - payload::meta::Dependency::PackageName => Kind::PackageName, - payload::meta::Dependency::SharedLibrary => Kind::SharedLibrary, - payload::meta::Dependency::PkgConfig => Kind::PkgConfig, - payload::meta::Dependency::Interpreter => Kind::Interpreter, - payload::meta::Dependency::CMake => Kind::CMake, - payload::meta::Dependency::Python => Kind::Python, - payload::meta::Dependency::Binary => Kind::Binary, - payload::meta::Dependency::SystemBinary => Kind::SystemBinary, - payload::meta::Dependency::PkgConfig32 => Kind::PkgConfig32, + StonePayloadMetaDependency::PackageName => Kind::PackageName, + StonePayloadMetaDependency::SharedLibrary => Kind::SharedLibrary, + StonePayloadMetaDependency::PkgConfig => Kind::PkgConfig, + StonePayloadMetaDependency::Interpreter => Kind::Interpreter, + StonePayloadMetaDependency::CMake => Kind::CMake, + StonePayloadMetaDependency::Python => Kind::Python, + StonePayloadMetaDependency::Binary => Kind::Binary, + StonePayloadMetaDependency::SystemBinary => Kind::SystemBinary, + StonePayloadMetaDependency::PkgConfig32 => Kind::PkgConfig32, } } } -/// Convert our [`Kind`] into a [`payload::meta::Dependency]`` -impl From for payload::meta::Dependency { +/// Convert our [`Kind`] into a [`Dependency]`` +impl From for StonePayloadMetaDependency { fn from(kind: Kind) -> Self { match kind { Kind::PackageName => Self::PackageName, diff --git a/moss/src/package/meta.rs b/moss/src/package/meta.rs index ae3eba19..95191ffc 100644 --- a/moss/src/package/meta.rs +++ b/moss/src/package/meta.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use derive_more::{AsRef, Display, From, Into}; -use stone::payload; +use stone::{StonePayloadMetaBody, StonePayloadMetaKind, StonePayloadMetaTag}; use thiserror::Error; use crate::{dependency, Dependency, Provider}; @@ -62,23 +62,23 @@ pub struct Meta { } impl Meta { - pub fn from_stone_payload(payload: &[payload::Meta]) -> Result { - let name = find_meta_string(payload, payload::meta::Tag::Name)?; - let version_identifier = find_meta_string(payload, payload::meta::Tag::Version)?; - let source_release = find_meta_u64(payload, payload::meta::Tag::Release)?; - let build_release = find_meta_u64(payload, payload::meta::Tag::BuildRelease)?; - let architecture = find_meta_string(payload, payload::meta::Tag::Architecture)?; - let summary = find_meta_string(payload, payload::meta::Tag::Summary)?; - let description = find_meta_string(payload, payload::meta::Tag::Description)?; - let source_id = find_meta_string(payload, payload::meta::Tag::SourceID)?; - let homepage = find_meta_string(payload, payload::meta::Tag::Homepage)?; - let uri = find_meta_string(payload, payload::meta::Tag::PackageURI).ok(); - let hash = find_meta_string(payload, payload::meta::Tag::PackageHash).ok(); - let download_size = find_meta_u64(payload, payload::meta::Tag::PackageSize).ok(); + pub fn from_stone_payload(payload: &[StonePayloadMetaBody]) -> Result { + let name = find_meta_string(payload, StonePayloadMetaTag::Name)?; + let version_identifier = find_meta_string(payload, StonePayloadMetaTag::Version)?; + let source_release = find_meta_u64(payload, StonePayloadMetaTag::Release)?; + let build_release = find_meta_u64(payload, StonePayloadMetaTag::BuildRelease)?; + let architecture = find_meta_string(payload, StonePayloadMetaTag::Architecture)?; + let summary = find_meta_string(payload, StonePayloadMetaTag::Summary)?; + let description = find_meta_string(payload, StonePayloadMetaTag::Description)?; + let source_id = find_meta_string(payload, StonePayloadMetaTag::SourceID)?; + let homepage = find_meta_string(payload, StonePayloadMetaTag::Homepage)?; + let uri = find_meta_string(payload, StonePayloadMetaTag::PackageURI).ok(); + let hash = find_meta_string(payload, StonePayloadMetaTag::PackageHash).ok(); + let download_size = find_meta_u64(payload, StonePayloadMetaTag::PackageSize).ok(); let licenses = payload .iter() - .filter_map(|meta| meta_string(meta, payload::meta::Tag::License)) + .filter_map(|meta| meta_string(meta, StonePayloadMetaTag::License)) .collect(); let dependencies = payload.iter().filter_map(meta_dependency).collect(); let providers = payload @@ -112,48 +112,90 @@ impl Meta { }) } - pub fn to_stone_payload(self) -> Vec { - use payload::meta::{Kind, Tag}; - + pub fn to_stone_payload(self) -> Vec { vec![ - (Tag::Name, Kind::String(self.name.to_string())), - (Tag::Version, Kind::String(self.version_identifier)), - (Tag::Release, Kind::Uint64(self.source_release)), - (Tag::BuildRelease, Kind::Uint64(self.build_release)), - (Tag::Architecture, Kind::String(self.architecture)), - (Tag::Summary, Kind::String(self.summary)), - (Tag::Description, Kind::String(self.description)), - (Tag::SourceID, Kind::String(self.source_id)), - (Tag::Homepage, Kind::String(self.homepage)), + ( + StonePayloadMetaTag::Name, + StonePayloadMetaKind::String(self.name.to_string()), + ), + ( + StonePayloadMetaTag::Version, + StonePayloadMetaKind::String(self.version_identifier), + ), + ( + StonePayloadMetaTag::Release, + StonePayloadMetaKind::Uint64(self.source_release), + ), + ( + StonePayloadMetaTag::BuildRelease, + StonePayloadMetaKind::Uint64(self.build_release), + ), + ( + StonePayloadMetaTag::Architecture, + StonePayloadMetaKind::String(self.architecture), + ), + (StonePayloadMetaTag::Summary, StonePayloadMetaKind::String(self.summary)), + ( + StonePayloadMetaTag::Description, + StonePayloadMetaKind::String(self.description), + ), + ( + StonePayloadMetaTag::SourceID, + StonePayloadMetaKind::String(self.source_id), + ), + ( + StonePayloadMetaTag::Homepage, + StonePayloadMetaKind::String(self.homepage), + ), ] .into_iter() - .chain(self.uri.map(|uri| (Tag::PackageURI, Kind::String(uri)))) - .chain(self.hash.map(|hash| (Tag::PackageHash, Kind::String(hash)))) - .chain(self.download_size.map(|size| (Tag::PackageSize, Kind::Uint64(size)))) .chain( - self.licenses - .into_iter() - .map(|license| (Tag::License, Kind::String(license))), + self.uri + .map(|uri| (StonePayloadMetaTag::PackageURI, StonePayloadMetaKind::String(uri))), + ) + .chain( + self.hash + .map(|hash| (StonePayloadMetaTag::PackageHash, StonePayloadMetaKind::String(hash))), + ) + .chain( + self.download_size + .map(|size| (StonePayloadMetaTag::PackageSize, StonePayloadMetaKind::Uint64(size))), ) .chain( - self.dependencies + self.licenses .into_iter() - .map(|dep| (Tag::Depends, Kind::Dependency(dep.kind.into(), dep.name))), + .map(|license| (StonePayloadMetaTag::License, StonePayloadMetaKind::String(license))), ) + .chain(self.dependencies.into_iter().map(|dep| { + ( + StonePayloadMetaTag::Depends, + StonePayloadMetaKind::Dependency(dep.kind.into(), dep.name), + ) + })) .chain( self.providers .into_iter() // We re-add this on ingestion / it's implied .filter(|provider| provider.kind != dependency::Kind::PackageName) - .map(|provider| (Tag::Provides, Kind::Provider(provider.kind.into(), provider.name))), + .map(|provider| { + ( + StonePayloadMetaTag::Provides, + StonePayloadMetaKind::Provider(provider.kind.into(), provider.name), + ) + }), ) .chain( self.conflicts .into_iter() // We re-add this on ingestion / it's implied - .map(|conflict| (Tag::Conflicts, Kind::Provider(conflict.kind.into(), conflict.name))), + .map(|conflict| { + ( + StonePayloadMetaTag::Conflicts, + StonePayloadMetaKind::Provider(conflict.kind.into(), conflict.name), + ) + }), ) - .map(|(tag, kind)| payload::Meta { tag, kind }) + .map(|(tag, kind)| StonePayloadMetaBody { tag, kind }) .collect() } @@ -166,29 +208,29 @@ impl Meta { } } -fn find_meta_string(meta: &[payload::Meta], tag: payload::meta::Tag) -> Result { +fn find_meta_string(meta: &[StonePayloadMetaBody], tag: StonePayloadMetaTag) -> Result { meta.iter() .find_map(|meta| meta_string(meta, tag)) .ok_or(MissingMetaFieldError(tag)) } -fn find_meta_u64(meta: &[payload::Meta], tag: payload::meta::Tag) -> Result { +fn find_meta_u64(meta: &[StonePayloadMetaBody], tag: StonePayloadMetaTag) -> Result { meta.iter() .find_map(|meta| meta_u64(meta, tag)) .ok_or(MissingMetaFieldError(tag)) } -fn meta_u64(meta: &payload::Meta, tag: payload::meta::Tag) -> Option { +fn meta_u64(meta: &StonePayloadMetaBody, tag: StonePayloadMetaTag) -> Option { if meta.tag == tag { Some(match meta.kind { - payload::meta::Kind::Int8(i) => i as _, - payload::meta::Kind::Uint8(i) => i as _, - payload::meta::Kind::Int16(i) => i as _, - payload::meta::Kind::Uint16(i) => i as _, - payload::meta::Kind::Int32(i) => i as _, - payload::meta::Kind::Uint32(i) => i as _, - payload::meta::Kind::Int64(i) => i as _, - payload::meta::Kind::Uint64(i) => i, + StonePayloadMetaKind::Int8(i) => i as _, + StonePayloadMetaKind::Uint8(i) => i as _, + StonePayloadMetaKind::Int16(i) => i as _, + StonePayloadMetaKind::Uint16(i) => i as _, + StonePayloadMetaKind::Int32(i) => i as _, + StonePayloadMetaKind::Uint32(i) => i as _, + StonePayloadMetaKind::Int64(i) => i as _, + StonePayloadMetaKind::Uint64(i) => i, _ => return None, }) } else { @@ -196,15 +238,15 @@ fn meta_u64(meta: &payload::Meta, tag: payload::meta::Tag) -> Option { } } -fn meta_string(meta: &payload::Meta, tag: payload::meta::Tag) -> Option { +fn meta_string(meta: &StonePayloadMetaBody, tag: StonePayloadMetaTag) -> Option { match (meta.tag, &meta.kind) { - (meta_tag, payload::meta::Kind::String(value)) if meta_tag == tag => Some(value.clone()), + (meta_tag, StonePayloadMetaKind::String(value)) if meta_tag == tag => Some(value.clone()), _ => None, } } -fn meta_dependency(meta: &payload::Meta) -> Option { - if let payload::meta::Kind::Dependency(kind, name) = meta.kind.clone() { +fn meta_dependency(meta: &StonePayloadMetaBody) -> Option { + if let StonePayloadMetaKind::Dependency(kind, name) = meta.kind.clone() { Some(Dependency { kind: dependency::Kind::from(kind), name, @@ -214,9 +256,9 @@ fn meta_dependency(meta: &payload::Meta) -> Option { } } -fn meta_provider(meta: &payload::Meta) -> Option { +fn meta_provider(meta: &StonePayloadMetaBody) -> Option { match (meta.tag, meta.kind.clone()) { - (payload::meta::Tag::Provides, payload::meta::Kind::Provider(kind, name)) => Some(Provider { + (StonePayloadMetaTag::Provides, StonePayloadMetaKind::Provider(kind, name)) => Some(Provider { kind: dependency::Kind::from(kind), name: name.clone(), }), @@ -224,9 +266,9 @@ fn meta_provider(meta: &payload::Meta) -> Option { } } -fn meta_conflict(meta: &payload::Meta) -> Option { +fn meta_conflict(meta: &StonePayloadMetaBody) -> Option { match (meta.tag, meta.kind.clone()) { - (payload::meta::Tag::Conflicts, payload::meta::Kind::Provider(kind, name)) => Some(Provider { + (StonePayloadMetaTag::Conflicts, StonePayloadMetaKind::Provider(kind, name)) => Some(Provider { kind: dependency::Kind::from(kind), name: name.clone(), }), @@ -236,4 +278,4 @@ fn meta_conflict(meta: &payload::Meta) -> Option { #[derive(Debug, Error)] #[error("Missing metadata field: {0:?}")] -pub struct MissingMetaFieldError(pub payload::meta::Tag); +pub struct MissingMetaFieldError(pub StonePayloadMetaTag); diff --git a/moss/src/registry/plugin/cobble.rs b/moss/src/registry/plugin/cobble.rs index e3c05ee3..bc5b9bb3 100644 --- a/moss/src/registry/plugin/cobble.rs +++ b/moss/src/registry/plugin/cobble.rs @@ -7,10 +7,9 @@ use std::io; use std::path::PathBuf; use fs_err::File; +use stone::{StoneDecodedPayload, StoneReadError}; use thiserror::Error; -use stone::read::PayloadKind; - use crate::package::{self, meta, Meta, MissingMetaFieldError, Package}; use crate::Provider; @@ -32,7 +31,7 @@ impl Cobble { // Grab the metapayload let metadata = payloads .find_map(|result| { - if let Ok(PayloadKind::Meta(meta)) = result { + if let Ok(StoneDecodedPayload::Meta(meta)) = result { Some(meta) } else { None @@ -114,7 +113,7 @@ pub enum Error { MissingMetaPayload, #[error("stone read")] - StoneRead(#[from] stone::read::Error), + StoneRead(#[from] StoneReadError), #[error("io")] Io(#[from] io::Error), diff --git a/moss/src/repository/manager.rs b/moss/src/repository/manager.rs index 288f34e8..9d198849 100644 --- a/moss/src/repository/manager.rs +++ b/moss/src/repository/manager.rs @@ -9,6 +9,7 @@ use std::time::Duration; use fs_err::{self as fs, File}; use futures_util::{stream, StreamExt, TryStreamExt}; +use stone::{StoneDecodedPayload, StonePayloadMetaTag, StoneReadError}; use thiserror::Error; use xxhash_rust::xxh3::xxh3_64; @@ -343,7 +344,7 @@ fn update_meta_db(state: &repository::Cached, index_path: &Path) -> Result<(), E let packages = payloads .into_iter() .filter_map(|payload| { - if let stone::read::PayloadKind::Meta(meta) = payload { + if let StoneDecodedPayload::Meta(meta) = payload { Some(meta) } else { None @@ -356,7 +357,7 @@ fn update_meta_db(state: &repository::Cached, index_path: &Path) -> Result<(), E let hash = meta .hash .clone() - .ok_or(Error::MissingMetaField(stone::payload::meta::Tag::PackageHash))?; + .ok_or(Error::MissingMetaField(StonePayloadMetaTag::PackageHash))?; let id = package::Id::from(hash); Ok((id, meta)) @@ -374,7 +375,7 @@ pub enum Error { #[error("Can't modify repos when using explicit configs")] ExplicitUnsupported, #[error("Missing metadata field: {0:?}")] - MissingMetaField(stone::payload::meta::Tag), + MissingMetaField(StonePayloadMetaTag), #[error("create directory")] CreateDir(#[source] io::Error), #[error("remove directory")] @@ -384,7 +385,7 @@ pub enum Error { #[error("open index file")] OpenIndex(#[source] io::Error), #[error("read index file")] - ReadStone(#[from] stone::read::Error), + ReadStone(#[from] StoneReadError), #[error("meta db")] Database(#[from] meta::Error), #[error("save config")] From 8fdc4b783fa8bc6e37a68332806f007531ffd47c Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Thu, 24 Oct 2024 21:40:34 -0700 Subject: [PATCH 02/12] Add initial stone ffi crate & test --- Cargo.lock | 8 + Cargo.toml | 46 +++- boulder/src/package/collect.rs | 8 +- boulder/src/package/emit/manifest/binary.rs | 4 +- crates/stone-ffi/Cargo.toml | 22 ++ crates/stone-ffi/cbindgen.toml | 21 ++ crates/stone-ffi/src/lib.rs | 259 ++++++++++++++++++++ crates/stone-ffi/src/payload.rs | 106 ++++++++ crates/stone-ffi/test.c | 128 ++++++++++ crates/stone/src/header/v1.rs | 3 +- crates/stone/src/lib.rs | 6 +- crates/stone/src/payload/attribute.rs | 4 +- crates/stone/src/payload/content.rs | 2 +- crates/stone/src/payload/index.rs | 4 +- crates/stone/src/payload/layout.rs | 32 +-- crates/stone/src/payload/meta.rs | 4 +- crates/stone/src/payload/mod.rs | 10 +- crates/stone/src/read/mod.rs | 85 ++++--- crates/stone/src/write.rs | 36 +-- justfile | 8 + moss/src/client/boot.rs | 13 +- moss/src/client/cache.rs | 4 +- moss/src/client/mod.rs | 6 +- moss/src/db/layout/mod.rs | 14 +- moss/src/package/meta.rs | 22 +- 25 files changed, 713 insertions(+), 142 deletions(-) create mode 100644 crates/stone-ffi/Cargo.toml create mode 100644 crates/stone-ffi/cbindgen.toml create mode 100644 crates/stone-ffi/src/lib.rs create mode 100644 crates/stone-ffi/src/payload.rs create mode 100644 crates/stone-ffi/test.c diff --git a/Cargo.lock b/Cargo.lock index 60cbeb36..29d4bcca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2637,6 +2637,14 @@ dependencies = [ "zstd", ] +[[package]] +name = "stone-ffi" +version = "0.24.7" +dependencies = [ + "libc", + "stone", +] + [[package]] name = "stone_recipe" version = "0.24.7" diff --git a/Cargo.toml b/Cargo.toml index 183f6b3a..bb4fa6dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,6 @@ [workspace] -members = [ - "boulder", - "moss", - "crates/*", -] -default-members = [ - "moss" -] +members = ["boulder", "moss", "crates/*"] +default-members = ["moss"] resolver = "2" [workspace.package] @@ -23,9 +17,17 @@ clap_complete = "4.5.37" clap_mangen = "0.2.24" criterion = { version = "0.5.1", features = ["html_reports"] } crossterm = "0.28.1" -derive_more = { version = "1.0.0", features = ["as_ref", "display", "from", "into"] } +derive_more = { version = "1.0.0", features = [ + "as_ref", + "display", + "from", + "into", +] } dialoguer = "0.11.0" -diesel = { version = "2.2.1", features = ["sqlite", "returning_clauses_for_sqlite_3_35"] } +diesel = { version = "2.2.1", features = [ + "sqlite", + "returning_clauses_for_sqlite_3_35", +] } diesel_migrations = "2.2.0" dirs = "5.0.1" elf = "0.7.4" @@ -39,11 +41,29 @@ indextree = "4.6.1" libsqlite3-sys = { version = "0.30.1", features = ["bundled"] } log = "0.4.22" nom = "7.1.3" -nix = { version = "0.27.1", features = ["user", "fs", "sched", "process", "mount", "hostname", "signal", "term"] } +nix = { version = "0.27.1", features = [ + "user", + "fs", + "sched", + "process", + "mount", + "hostname", + "signal", + "term", +] } petgraph = "0.6.5" rayon = "1.10.0" regex = "1.10.5" -reqwest = { version = "0.12.5", default-features = false, features = ["brotli", "charset", "deflate", "gzip", "http2", "rustls-tls", "stream", "zstd"] } +reqwest = { version = "0.12.5", default-features = false, features = [ + "brotli", + "charset", + "deflate", + "gzip", + "http2", + "rustls-tls", + "stream", + "zstd", +] } serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.120" serde_yaml = "0.9.34" @@ -59,6 +79,8 @@ xxhash-rust = { version = "0.8.11", features = ["xxh3"] } zstd = { version = "0.13.2", features = ["zstdmt"] } mailparse = "0.15.0" zbus = "5.1.1" +libc = "0.2.62" +cbindgen = "0.26" [workspace.lints.rust] rust_2018_idioms = { level = "warn", priority = -1 } diff --git a/boulder/src/package/collect.rs b/boulder/src/package/collect.rs index 057c0f84..6f38d89d 100644 --- a/boulder/src/package/collect.rs +++ b/boulder/src/package/collect.rs @@ -12,7 +12,7 @@ use std::{ use fs_err as fs; use glob::Pattern; use nix::libc::{S_IFDIR, S_IRGRP, S_IROTH, S_IRWXU, S_IXGRP, S_IXOTH}; -use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutBody, StonePayloadLayoutEntry}; +use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayout, StonePayloadLayoutEntry}; use thiserror::Error; #[derive(Debug, Clone, Eq, PartialEq)] @@ -126,7 +126,7 @@ impl Collector { pub struct PathInfo { pub path: PathBuf, pub target_path: PathBuf, - pub layout: StonePayloadLayoutBody, + pub layout: StonePayloadLayout, pub size: u64, pub package: String, } @@ -188,7 +188,7 @@ fn layout_from_metadata( target_path: &Path, metadata: &Metadata, hasher: &mut StoneDigestWriterHasher, -) -> Result { +) -> Result { // Strip /usr let target = target_path .strip_prefix("/usr") @@ -198,7 +198,7 @@ fn layout_from_metadata( let file_type = metadata.file_type(); - Ok(StonePayloadLayoutBody { + Ok(StonePayloadLayout { uid: metadata.uid(), gid: metadata.gid(), mode: metadata.mode(), diff --git a/boulder/src/package/emit/manifest/binary.rs b/boulder/src/package/emit/manifest/binary.rs index 52aa702f..461f01ca 100644 --- a/boulder/src/package/emit/manifest/binary.rs +++ b/boulder/src/package/emit/manifest/binary.rs @@ -6,7 +6,7 @@ use std::{collections::BTreeSet, path::Path}; use fs_err::File; use moss::Dependency; -use stone::{StoneHeaderV1FileType, StonePayloadMetaBody, StonePayloadMetaKind, StonePayloadMetaTag, StoneWriter}; +use stone::{StoneHeaderV1FileType, StonePayloadMeta, StonePayloadMetaKind, StonePayloadMetaTag, StoneWriter}; use super::Error; use crate::package::emit::Package; @@ -26,7 +26,7 @@ pub fn write(path: &Path, packages: &BTreeSet<&Package<'_>>, build_deps: &BTreeS // Add build deps for name in build_deps { if let Ok(dep) = Dependency::from_name(name) { - payload.push(StonePayloadMetaBody { + payload.push(StonePayloadMeta { tag: StonePayloadMetaTag::BuildDepends, kind: StonePayloadMetaKind::Dependency(dep.kind.into(), dep.name), }); diff --git a/crates/stone-ffi/Cargo.toml b/crates/stone-ffi/Cargo.toml new file mode 100644 index 00000000..9e108e4b --- /dev/null +++ b/crates/stone-ffi/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "stone-ffi" +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[features] +capi = [] + +[dependencies] +stone = { path = "../stone", features = ["ffi"] } + +libc.workspace = true + +[package.metadata.capi.header] +name = "stone" + +[package.metadata.capi.pkg_config] +name = "stone" + +[package.metadata.capi.library] +name = "stone" diff --git a/crates/stone-ffi/cbindgen.toml b/crates/stone-ffi/cbindgen.toml new file mode 100644 index 00000000..2f6c68c8 --- /dev/null +++ b/crates/stone-ffi/cbindgen.toml @@ -0,0 +1,21 @@ +include_guard = "STONE_H" +language = "C" + +header = """ +// SPDX-FileCopyrightText: Copyright © 2020-2024 Serpent OS Developers +// +// SPDX-License-Identifier: MPL-2.0 +""" + +[enum] +prefix_with_name = true +rename_variants = "ScreamingSnakeCase" + +[export] +include = ["StonePayloadLayoutRecord"] + +[export.rename] + +[parse] +parse_deps = true +include = ['stone'] diff --git a/crates/stone-ffi/src/lib.rs b/crates/stone-ffi/src/lib.rs new file mode 100644 index 00000000..7b369180 --- /dev/null +++ b/crates/stone-ffi/src/lib.rs @@ -0,0 +1,259 @@ +// SPDX-FileCopyrightText: Copyright © 2020-2024 Serpent OS Developers +// +// SPDX-License-Identifier: MPL-2.0 +#![allow(clippy::missing_safety_doc)] +#![allow(non_camel_case_types)] + +use std::{ + fs::File, + io::{Cursor, Write}, + os::fd::FromRawFd, + ptr::NonNull, + slice, +}; + +use libc::{c_int, size_t}; +use stone::{ + StoneDecodedPayload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StoneHeaderVersion, + StonePayloadCompression, StonePayloadHeader, StonePayloadKind, StonePayloadLayoutFileType, StoneReadError, +}; + +pub use self::payload::{StonePayload, StonePayloadLayoutRecord}; + +mod payload; + +#[derive(Debug)] +#[repr(C)] +pub struct StoneString { + pub buf: *const u8, + pub size: size_t, +} + +impl StoneString { + pub fn new(s: &str) -> Self { + Self { + buf: s.as_ptr(), + size: s.len(), + } + } +} + +pub enum StoneReader<'a> { + File(stone::StoneReader), + Buffer(stone::StoneReader>), +} + +impl<'a> StoneReader<'a> { + pub fn header(&self) -> &StoneHeader { + match self { + StoneReader::File(reader) => &reader.header, + StoneReader::Buffer(reader) => &reader.header, + } + } + + pub fn next_payload(&mut self) -> Result, StoneReadError> { + match self { + StoneReader::File(reader) => reader.next_payload(), + StoneReader::Buffer(reader) => reader.next_payload(), + } + } + + pub fn unpack_content( + &mut self, + content: &stone::StonePayload, + writer: &mut W, + ) -> Result<(), StoneReadError> + where + W: Write, + { + match self { + StoneReader::File(reader) => reader.unpack_content(content, writer), + StoneReader::Buffer(reader) => reader.unpack_content(content, writer), + } + } +} + +fn fallible(f: impl FnOnce() -> Result<(), Box>) -> c_int { + if f().is_err() { + -1 + } else { + 0 + } +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_read_file( + file: c_int, + reader_ptr: *mut *mut StoneReader, + version: *mut StoneHeaderVersion, +) -> c_int { + fallible(|| { + // TODO: Errors + let reader_ptr = NonNull::new(reader_ptr).ok_or("")?; + let mut version = NonNull::new(version).ok_or("")?; + + let reader = stone::read(File::from_raw_fd(file)).map(StoneReader::File)?; + + *version.as_mut() = reader.header().version(); + *reader_ptr.as_ptr() = Box::into_raw(Box::new(reader)); + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_read_buf( + buf: *const u8, + len: usize, + reader_ptr: *mut *mut StoneReader, + version: *mut StoneHeaderVersion, +) -> c_int { + fallible(|| { + let buf = NonNull::new(buf as *mut _).ok_or("")?; + let reader_ptr = NonNull::new(reader_ptr).ok_or("")?; + let mut version = NonNull::new(version).ok_or("")?; + + let reader = stone::read_bytes(slice::from_raw_parts(buf.as_ptr(), len)).map(StoneReader::Buffer)?; + + *version.as_mut() = reader.header().version(); + *reader_ptr.as_ptr() = Box::into_raw(Box::new(reader)); + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_header_v1(reader: *const StoneReader, header: *mut StoneHeaderV1) -> c_int { + fallible(|| { + let reader = NonNull::new(reader as *mut StoneReader).ok_or("")?; + let mut header = NonNull::new(header).ok_or("")?; + + match reader.as_ref().header() { + StoneHeader::V1(v1) => *header.as_mut() = *v1, + } + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_next_payload( + reader: *mut StoneReader, + payload_ptr: *mut *mut StonePayload, +) -> c_int { + fallible(|| { + let mut reader = NonNull::new(reader).ok_or("")?; + let payload_ptr = NonNull::new(payload_ptr).ok_or("")?; + + if let Some(payload) = reader.as_mut().next_payload()? { + *payload_ptr.as_ptr() = Box::into_raw(Box::new(payload.into())); + } else { + Err("no more payloads")?; + } + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_unpack_content_payload( + reader: *mut StoneReader, + payload: *const StonePayload, + data: *mut u8, +) -> c_int { + fallible(|| { + let mut reader = NonNull::new(reader).ok_or("")?; + let payload = NonNull::new(payload as *mut StonePayload).ok_or("")?; + let data = NonNull::new(data).ok_or("")?; + + let mut cursor = Cursor::new(slice::from_raw_parts_mut( + data.as_ptr(), + payload.as_ref().header().stored_size as usize, + )); + + if let StoneDecodedPayload::Content(content) = &payload.as_ref().decoded { + reader.as_mut().unpack_content(content, &mut cursor)?; + } else { + Err("incorrect payload kind")?; + } + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_destroy(reader: *mut StoneReader) { + let Some(reader) = NonNull::new(reader) else { + return; + }; + + drop(Box::from_raw(reader.as_ptr())); +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_header(payload: *const StonePayload, header: *mut StonePayloadHeader) -> c_int { + fallible(|| { + let payload = NonNull::new(payload as *mut StonePayload).ok_or("")?; + let mut header = NonNull::new(header).ok_or("")?; + + *header.as_mut() = *payload.as_ref().header(); + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_next_layout_record( + payload: *mut StonePayload, + record: *mut StonePayloadLayoutRecord, +) -> c_int { + fallible(|| { + let mut payload = NonNull::new(payload).ok_or("")?; + let mut record = NonNull::new(record).ok_or("")?; + + *record.as_mut() = payload.as_mut().next_layout_record().ok_or("")?; + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_destroy(payload: *mut StonePayload) { + let Some(payload) = NonNull::new(payload) else { + return; + }; + + drop(Box::from_raw(payload.as_ptr())); +} + +#[no_mangle] +pub unsafe extern "C" fn stone_format_header_v1_file_type(file_type: StoneHeaderV1FileType, buf: *mut u8) { + fill_c_string(buf, file_type); +} + +#[no_mangle] +pub unsafe extern "C" fn stone_format_payload_compression(compression: StonePayloadCompression, buf: *mut u8) { + fill_c_string(buf, compression); +} + +#[no_mangle] +pub unsafe extern "C" fn stone_format_payload_kind(kind: StonePayloadKind, buf: *mut u8) { + fill_c_string(buf, kind); +} + +#[no_mangle] +pub unsafe extern "C" fn stone_format_payload_layout_file_type(file_type: StonePayloadLayoutFileType, buf: *mut u8) { + fill_c_string(buf, file_type); +} + +unsafe fn fill_c_string(buf: *mut u8, content: impl ToString) { + let Some(buf) = NonNull::new(buf) else { + return; + }; + + let content = content.to_string(); + let buf = slice::from_raw_parts_mut(buf.as_ptr(), content.len() + 1); + + buf[0..content.len()].copy_from_slice(content.as_bytes()); + buf[content.len()] = b'\0'; +} diff --git a/crates/stone-ffi/src/payload.rs b/crates/stone-ffi/src/payload.rs new file mode 100644 index 00000000..f5058ab5 --- /dev/null +++ b/crates/stone-ffi/src/payload.rs @@ -0,0 +1,106 @@ +use std::mem::ManuallyDrop; + +use stone::{StoneDecodedPayload, StonePayloadHeader, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; + +use super::StoneString; + +pub struct StonePayload { + pub decoded: StoneDecodedPayload, + next_record: usize, +} + +impl StonePayload { + pub fn header(&self) -> &StonePayloadHeader { + self.decoded.header() + } + + pub fn next_layout_record(&mut self) -> Option { + if self.next_record >= self.header().num_records { + return None; + } + + self.next_record += 1; + + let payload = self.decoded.layout()?; + let record = payload.body.get(self.next_record)?; + + Some(StonePayloadLayoutRecord { + uid: record.uid, + gid: record.gid, + mode: record.mode, + tag: record.tag, + file_type: record.entry.file_type(), + file_payload: match &record.entry { + StonePayloadLayoutEntry::Regular(hash, name) => StonePayloadLayoutFilePayload { + regular: ManuallyDrop::new(StonePayloadLayoutFileRegular { + hash: hash.to_be_bytes(), + name: StoneString::new(name), + }), + }, + StonePayloadLayoutEntry::Symlink(source, target) => StonePayloadLayoutFilePayload { + symlink: ManuallyDrop::new(StonePayloadLayoutFileSymlink { + source: StoneString::new(source), + target: StoneString::new(target), + }), + }, + StonePayloadLayoutEntry::Directory(name) => StonePayloadLayoutFilePayload { + directory: ManuallyDrop::new(StoneString::new(name)), + }, + StonePayloadLayoutEntry::CharacterDevice(name) => StonePayloadLayoutFilePayload { + character_device: ManuallyDrop::new(StoneString::new(name)), + }, + StonePayloadLayoutEntry::BlockDevice(name) => StonePayloadLayoutFilePayload { + block_device: ManuallyDrop::new(StoneString::new(name)), + }, + StonePayloadLayoutEntry::Fifo(name) => StonePayloadLayoutFilePayload { + fifo: ManuallyDrop::new(StoneString::new(name)), + }, + StonePayloadLayoutEntry::Socket(name) => StonePayloadLayoutFilePayload { + socket: ManuallyDrop::new(StoneString::new(name)), + }, + }, + }) + } +} + +impl From for StonePayload { + fn from(decoded: StoneDecodedPayload) -> Self { + Self { + decoded, + next_record: 0, + } + } +} + +#[repr(C)] +pub struct StonePayloadLayoutRecord { + pub uid: u32, + pub gid: u32, + pub mode: u32, + pub tag: u32, + pub file_type: StonePayloadLayoutFileType, + pub file_payload: StonePayloadLayoutFilePayload, +} + +#[repr(C)] +pub union StonePayloadLayoutFilePayload { + regular: ManuallyDrop, + symlink: ManuallyDrop, + directory: ManuallyDrop, + character_device: ManuallyDrop, + block_device: ManuallyDrop, + fifo: ManuallyDrop, + socket: ManuallyDrop, +} + +#[repr(C)] +pub struct StonePayloadLayoutFileRegular { + pub hash: [u8; 16], + pub name: StoneString, +} + +#[repr(C)] +pub struct StonePayloadLayoutFileSymlink { + pub source: StoneString, + pub target: StoneString, +} diff --git a/crates/stone-ffi/test.c b/crates/stone-ffi/test.c new file mode 100644 index 00000000..bafb6986 --- /dev/null +++ b/crates/stone-ffi/test.c @@ -0,0 +1,128 @@ +#include +#include +#include +#include +#include +#include + +// 32 byte stone header +static uint8_t HEADER_BUF[] = {0x00, 0x6d, 0x6f, 0x73, 0x00, 0x04, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x05, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x01}; + +void print_header_v1(StoneHeaderV1 *header) { + char file_type[100]; + + stone_format_header_v1_file_type(header->file_type, file_type); + + printf("StoneHeaderV1 {\n"); + printf(" num_payloads: %d\n", header->num_payloads); + printf(" file_type: %s\n", file_type); + printf("}\n"); +} + +void print_payload_header(StonePayloadHeader *header) { + char compression[100]; + char kind[100]; + + stone_format_payload_compression(header->compression, compression); + stone_format_payload_kind(header->kind, &(kind[0])); + + printf("StonePayload {\n"); + printf(" kind: %s\n", kind); + printf(" plain_size: %d\n", header->plain_size); + printf(" stored_size: %d\n", header->stored_size); + printf(" compression: %s\n", compression); + printf(" num_records: %d\n", header->num_records); + printf(" version: %d\n", header->version); + printf("}\n"); +} + +void print_payload_layout_record(StonePayloadLayoutRecord *record) { + char file_type[100]; + + stone_format_payload_layout_file_type(record->file_type, file_type); + + printf("StonePayloadLayoutRecord {\n"); + printf(" uid: %d\n", record->uid); + printf(" gid: %d\n", record->gid); + printf(" mode: %d\n", record->mode); + printf(" tag: %d\n", record->tag); + printf(" file_type: %s\n", file_type); + + switch (record->file_type) { + case STONE_PAYLOAD_LAYOUT_FILE_TYPE_REGULAR: { + printf(" hash: "); + for (size_t i = 0; i < 16; i++) { + printf("%.02x", record->file_payload.regular.hash[i]); + } + printf("\n name: %.*s\n", record->file_payload.regular.name.size, + record->file_payload.regular.name.buf); + break; + } + case STONE_PAYLOAD_LAYOUT_FILE_TYPE_SYMLINK: { + printf(" source: %.*s\n", record->file_payload.symlink.source.size, + record->file_payload.symlink.source.buf); + printf(" target: %.*s\n", record->file_payload.symlink.target.size, + record->file_payload.symlink.target.buf); + break; + } + default: + } + + printf("}\n"); +} + +void process_reader(StoneReader *reader, StoneHeaderVersion version) { + StoneHeaderV1 header; + StonePayload *payload; + + assert(version == STONE_HEADER_VERSION_V1); + + stone_reader_header_v1(reader, &header); + print_header_v1(&header); + + while (stone_reader_next_payload(reader, &payload) >= 0) { + StonePayloadHeader payload_header; + + stone_payload_header(payload, &payload_header); + print_payload_header(&payload_header); + + switch (payload_header.kind) { + case STONE_PAYLOAD_KIND_LAYOUT: { + StonePayloadLayoutRecord record; + + while (stone_payload_next_layout_record(payload, &record) >= 0) { + print_payload_layout_record(&record); + } + + break; + } + default: + } + + stone_payload_destroy(payload); + } + + stone_reader_destroy(reader); +} + +int main(int argc, char *argv[]) { + FILE *fptr; + StoneReader *reader; + StoneHeaderVersion version; + char *file = "./test/bash-completion-2.11-1-1-x86_64.stone"; + + printf("Reading stone from '%s'\n\n", file); + fptr = fopen(file, "r"); + stone_reader_read_file(fileno(fptr), &reader, &version); + process_reader(reader, version); + + printf("\n"); + printf("Reading stone header from buffer\n\n"); + stone_reader_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); + process_reader(reader, version); + + return 0; +} diff --git a/crates/stone/src/header/v1.rs b/crates/stone/src/header/v1.rs index cdb65de6..dfe551a7 100644 --- a/crates/stone/src/header/v1.rs +++ b/crates/stone/src/header/v1.rs @@ -11,7 +11,8 @@ const INTEGRITY_CHECK: [u8; 21] = [0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 4, 0, 0, 5, /// /// Some types are now legacy as we're going to use Ion to define them. /// -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] +#[strum(serialize_all = "kebab-case")] #[repr(u8)] pub enum StoneHeaderV1FileType { /// Binary package diff --git a/crates/stone/src/lib.rs b/crates/stone/src/lib.rs index 85ede379..233dbb3b 100644 --- a/crates/stone/src/lib.rs +++ b/crates/stone/src/lib.rs @@ -13,9 +13,9 @@ pub use self::header::{ StoneHeaderV1FileType, StoneHeaderVersion, STONE_HEADER_MAGIC, }; pub use self::payload::{ - StonePayload, StonePayloadAttributeBody, StonePayloadCompression, StonePayloadContentBody, StonePayloadDecodeError, - StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndexBody, StonePayloadKind, StonePayloadLayoutBody, - StonePayloadLayoutEntry, StonePayloadLayoutFileType, StonePayloadMetaBody, StonePayloadMetaDependency, + StonePayload, StonePayloadAttribute, StonePayloadCompression, StonePayloadContent, StonePayloadDecodeError, + StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndex, StonePayloadKind, StonePayloadLayout, + StonePayloadLayoutEntry, StonePayloadLayoutFileType, StonePayloadMeta, StonePayloadMetaDependency, StonePayloadMetaKind, StonePayloadMetaTag, }; pub use self::read::{read, read_bytes, StoneDecodedPayload, StoneReadError, StoneReader}; diff --git a/crates/stone/src/payload/attribute.rs b/crates/stone/src/payload/attribute.rs index 4924ba5c..8091ed41 100644 --- a/crates/stone/src/payload/attribute.rs +++ b/crates/stone/src/payload/attribute.rs @@ -8,12 +8,12 @@ use super::{Record, StonePayloadDecodeError, StonePayloadEncodeError}; use crate::ext::{ReadExt, WriteExt}; #[derive(Debug, Clone)] -pub struct StonePayloadAttributeBody { +pub struct StonePayloadAttribute { pub key: Vec, pub value: Vec, } -impl Record for StonePayloadAttributeBody { +impl Record for StonePayloadAttribute { fn decode(mut reader: R) -> Result { let key_length = reader.read_u64()?; let value_length = reader.read_u64()?; diff --git a/crates/stone/src/payload/content.rs b/crates/stone/src/payload/content.rs index ca70fec3..71b9e0b9 100644 --- a/crates/stone/src/payload/content.rs +++ b/crates/stone/src/payload/content.rs @@ -3,6 +3,6 @@ // SPDX-License-Identifier: MPL-2.0 #[derive(Debug)] -pub struct StonePayloadContentBody { +pub struct StonePayloadContent { pub offset: u64, } diff --git a/crates/stone/src/payload/index.rs b/crates/stone/src/payload/index.rs index be31fea3..52d78aad 100644 --- a/crates/stone/src/payload/index.rs +++ b/crates/stone/src/payload/index.rs @@ -13,7 +13,7 @@ use crate::ext::{ReadExt, WriteExt}; /// This is used to split the file into the content store on disk before promoting /// to a transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct StonePayloadIndexBody { +pub struct StonePayloadIndex { /// Start pf the entry within the ContentPayload pub start: u64, @@ -24,7 +24,7 @@ pub struct StonePayloadIndexBody { pub digest: u128, } -impl Record for StonePayloadIndexBody { +impl Record for StonePayloadIndex { fn decode(mut reader: R) -> Result { let start = reader.read_u64()?; let end = reader.read_u64()?; diff --git a/crates/stone/src/payload/layout.rs b/crates/stone/src/payload/layout.rs index 46787cb9..dfc8f819 100644 --- a/crates/stone/src/payload/layout.rs +++ b/crates/stone/src/payload/layout.rs @@ -4,13 +4,14 @@ use std::io::{Read, Write}; -use super::{StonePayloadDecodeError, StonePayloadEncodeError, Record}; +use super::{Record, StonePayloadDecodeError, StonePayloadEncodeError}; use crate::ext::{ReadExt, WriteExt}; /// Layout entries record their target file type so they can be rebuilt on /// the target installation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] +#[strum(serialize_all = "kebab-case")] #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StonePayloadLayoutFileType { /// Regular file Regular = 1, @@ -72,22 +73,21 @@ impl StonePayloadLayoutEntry { } } - fn file_type(&self) -> u8 { + pub fn file_type(&self) -> StonePayloadLayoutFileType { match self { - StonePayloadLayoutEntry::Regular(..) => 1, - StonePayloadLayoutEntry::Symlink(..) => 2, - StonePayloadLayoutEntry::Directory(_) => 3, - StonePayloadLayoutEntry::CharacterDevice(_) => 4, - StonePayloadLayoutEntry::BlockDevice(_) => 5, - StonePayloadLayoutEntry::Fifo(_) => 6, - StonePayloadLayoutEntry::Socket(_) => 7, + StonePayloadLayoutEntry::Regular(..) => StonePayloadLayoutFileType::Regular, + StonePayloadLayoutEntry::Symlink(..) => StonePayloadLayoutFileType::Symlink, + StonePayloadLayoutEntry::Directory(_) => StonePayloadLayoutFileType::Directory, + StonePayloadLayoutEntry::CharacterDevice(_) => StonePayloadLayoutFileType::CharacterDevice, + StonePayloadLayoutEntry::BlockDevice(_) => StonePayloadLayoutFileType::BlockDevice, + StonePayloadLayoutEntry::Fifo(_) => StonePayloadLayoutFileType::Fifo, + StonePayloadLayoutEntry::Socket(_) => StonePayloadLayoutFileType::Socket, } } } -// TODO: Strong types these fields #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StonePayloadLayoutBody { +pub struct StonePayloadLayout { pub uid: u32, pub gid: u32, pub mode: u32, @@ -95,7 +95,7 @@ pub struct StonePayloadLayoutBody { pub entry: StonePayloadLayoutEntry, } -impl Record for StonePayloadLayoutBody { +impl Record for StonePayloadLayout { fn decode(mut reader: R) -> Result { let uid = reader.read_u32()?; let gid = reader.read_u32()?; @@ -131,7 +131,9 @@ impl Record for StonePayloadLayoutBody { sanitize(reader.read_string(source_length as u64)?), sanitize(reader.read_string(target_length as u64)?), ), - StonePayloadLayoutFileType::Directory => StonePayloadLayoutEntry::Directory(sanitize(reader.read_string(target_length as u64)?)), + StonePayloadLayoutFileType::Directory => { + StonePayloadLayoutEntry::Directory(sanitize(reader.read_string(target_length as u64)?)) + } _ => { if source_length > 0 { let _ = reader.read_vec(source_length as usize); @@ -160,7 +162,7 @@ impl Record for StonePayloadLayoutBody { writer.write_u16(source.len() as u16)?; writer.write_u16(target.len() as u16)?; - writer.write_u8(self.entry.file_type())?; + writer.write_u8(self.entry.file_type() as u8)?; writer.write_array([0; 11])?; writer.write_all(&source)?; writer.write_all(target.as_bytes())?; diff --git a/crates/stone/src/payload/meta.rs b/crates/stone/src/payload/meta.rs index e275c612..20845d06 100644 --- a/crates/stone/src/payload/meta.rs +++ b/crates/stone/src/payload/meta.rs @@ -12,7 +12,7 @@ use crate::ext::{ReadExt, WriteExt}; /// These record all metadata for every .stone packages and provide /// no content #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StonePayloadMetaBody { +pub struct StonePayloadMeta { pub tag: StonePayloadMetaTag, pub kind: StonePayloadMetaKind, } @@ -151,7 +151,7 @@ fn decode_dependency(i: u8) -> Result(mut reader: R) -> Result { let length = reader.read_u32()?; diff --git a/crates/stone/src/payload/mod.rs b/crates/stone/src/payload/mod.rs index b3857fc5..d6a519d4 100644 --- a/crates/stone/src/payload/mod.rs +++ b/crates/stone/src/payload/mod.rs @@ -14,11 +14,11 @@ use thiserror::Error; use crate::ext::{ReadExt, WriteExt}; -pub use self::attribute::StonePayloadAttributeBody; -pub use self::content::StonePayloadContentBody; -pub use self::index::StonePayloadIndexBody; -pub use self::layout::{StonePayloadLayoutBody, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; -pub use self::meta::{StonePayloadMetaBody, StonePayloadMetaDependency, StonePayloadMetaKind, StonePayloadMetaTag}; +pub use self::attribute::StonePayloadAttribute; +pub use self::content::StonePayloadContent; +pub use self::index::StonePayloadIndex; +pub use self::layout::{StonePayloadLayout, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; +pub use self::meta::{StonePayloadMeta, StonePayloadMetaDependency, StonePayloadMetaKind, StonePayloadMetaTag}; #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] #[strum(serialize_all = "kebab-case")] diff --git a/crates/stone/src/read/mod.rs b/crates/stone/src/read/mod.rs index 29ef94cf..5695415d 100644 --- a/crates/stone/src/read/mod.rs +++ b/crates/stone/src/read/mod.rs @@ -7,9 +7,9 @@ use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; use thiserror::Error; use crate::{ - payload, StoneHeader, StoneHeaderDecodeError, StonePayload, StonePayloadAttributeBody, StonePayloadCompression, - StonePayloadContentBody, StonePayloadDecodeError, StonePayloadHeader, StonePayloadIndexBody, StonePayloadKind, - StonePayloadLayoutBody, StonePayloadMetaBody, + payload, StoneHeader, StoneHeaderDecodeError, StonePayload, StonePayloadAttribute, StonePayloadCompression, + StonePayloadContent, StonePayloadDecodeError, StonePayloadHeader, StonePayloadIndex, StonePayloadKind, + StonePayloadLayout, StonePayloadMeta, }; use self::zstd::Zstd; @@ -63,56 +63,43 @@ impl StoneReader { pub fn unpack_content( &mut self, - content: &StonePayload, + content: &StonePayload, writer: &mut W, ) -> Result<(), StoneReadError> where W: Write, { self.reader.seek(SeekFrom::Start(content.body.offset))?; + self.hasher.reset(); - unpack_payload(self, writer, &content.header) + let mut hashed = digest::Reader::new(&mut self.reader, &mut self.hasher); + let mut framed = (&mut hashed).take(content.header.stored_size); + + io::copy( + &mut PayloadReader::new(&mut framed, content.header.compression)?, + writer, + )?; + + // Validate checksum + validate_checksum(&self.hasher, &content.header)?; + + Ok(()) } } #[cfg(feature = "ffi")] impl StoneReader { - pub fn next_payload_header(&mut self) -> Result, StoneReadError> { + pub fn next_payload(&mut self) -> Result, StoneReadError> { if self.next_payload < self.header.num_payloads() { - let header = StonePayloadHeader::decode(&mut self.reader)?; + let payload = StoneDecodedPayload::decode(&mut self.reader, &mut self.hasher)?; self.next_payload += 1; - Ok(Some(header)) + Ok(payload) } else { Ok(None) } } - - pub fn unpack_payload(&mut self, header: &StonePayloadHeader, writer: &mut W) -> Result<(), StoneReadError> - where - W: Write, - { - unpack_payload(self, writer, header) - } -} - -fn unpack_payload( - reader: &mut StoneReader, - writer: &mut W, - header: &StonePayloadHeader, -) -> Result<(), StoneReadError> { - reader.hasher.reset(); - - let mut hashed = digest::Reader::new(&mut reader.reader, &mut reader.hasher); - let mut framed = (&mut hashed).take(header.stored_size); - - io::copy(&mut PayloadReader::new(&mut framed, header.compression)?, writer)?; - - // Validate checksum - validate_checksum(&reader.hasher, header)?; - - Ok(()) } enum PayloadReader { @@ -140,14 +127,24 @@ impl Read for PayloadReader { #[derive(Debug)] pub enum StoneDecodedPayload { - Meta(StonePayload>), - Attributes(StonePayload>), - Layout(StonePayload>), - Index(StonePayload>), - Content(StonePayload), + Meta(StonePayload>), + Attributes(StonePayload>), + Layout(StonePayload>), + Index(StonePayload>), + Content(StonePayload), } impl StoneDecodedPayload { + pub fn header(&self) -> &StonePayloadHeader { + match self { + StoneDecodedPayload::Meta(payload) => &payload.header, + StoneDecodedPayload::Attributes(payload) => &payload.header, + StoneDecodedPayload::Layout(payload) => &payload.header, + StoneDecodedPayload::Index(payload) => &payload.header, + StoneDecodedPayload::Content(payload) => &payload.header, + } + } + fn decode(mut reader: R, hasher: &mut digest::Hasher) -> Result, StoneReadError> { match StonePayloadHeader::decode(&mut reader) { Ok(header) => { @@ -192,7 +189,7 @@ impl StoneDecodedPayload { StoneDecodedPayload::Content(StonePayload { header, - body: StonePayloadContentBody { offset }, + body: StonePayloadContent { offset }, }) } StonePayloadKind::Dumb => unimplemented!("??"), @@ -210,7 +207,7 @@ impl StoneDecodedPayload { } } - pub fn meta(&self) -> Option<&StonePayload>> { + pub fn meta(&self) -> Option<&StonePayload>> { if let Self::Meta(meta) = self { Some(meta) } else { @@ -218,7 +215,7 @@ impl StoneDecodedPayload { } } - pub fn attributes(&self) -> Option<&StonePayload>> { + pub fn attributes(&self) -> Option<&StonePayload>> { if let Self::Attributes(attributes) = self { Some(attributes) } else { @@ -226,7 +223,7 @@ impl StoneDecodedPayload { } } - pub fn layout(&self) -> Option<&StonePayload>> { + pub fn layout(&self) -> Option<&StonePayload>> { if let Self::Layout(layouts) = self { Some(layouts) } else { @@ -234,7 +231,7 @@ impl StoneDecodedPayload { } } - pub fn index(&self) -> Option<&StonePayload>> { + pub fn index(&self) -> Option<&StonePayload>> { if let Self::Index(indices) = self { Some(indices) } else { @@ -242,7 +239,7 @@ impl StoneDecodedPayload { } } - pub fn content(&self) -> Option<&StonePayload> { + pub fn content(&self) -> Option<&StonePayload> { if let Self::Content(content) = self { Some(content) } else { diff --git a/crates/stone/src/write.rs b/crates/stone/src/write.rs index c7c1c547..79d5fe96 100644 --- a/crates/stone/src/write.rs +++ b/crates/stone/src/write.rs @@ -6,9 +6,9 @@ use std::io::{self, Read, Seek, SeekFrom, Write}; use thiserror::Error; use crate::{ - payload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StonePayloadAttributeBody, StonePayloadCompression, - StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndexBody, StonePayloadKind, StonePayloadLayoutBody, - StonePayloadMetaBody, + payload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StonePayloadAttribute, StonePayloadCompression, + StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndex, StonePayloadKind, StonePayloadLayout, + StonePayloadMeta, }; pub use self::digest::{StoneDigestWriter, StoneDigestWriterHasher}; @@ -129,7 +129,7 @@ where let end = self.content.plain_size; // Add index data - self.content.indices.push(StonePayloadIndexBody { start, end, digest }); + self.content.indices.push(StonePayloadIndex { start, end, digest }); Ok(()) } @@ -164,7 +164,7 @@ pub struct StoneContentWriter { buffer: B, plain_size: u64, stored_size: u64, - indices: Vec, + indices: Vec, /// Used to generate un-compressed digest of file /// contents used for [`Index`] index_hasher: StoneDigestWriterHasher, @@ -180,9 +180,9 @@ struct EncodedPayload { } pub enum StoneWritePayload<'a> { - Meta(&'a [StonePayloadMetaBody]), - Attributes(&'a [StonePayloadAttributeBody]), - Layout(&'a [StonePayloadLayoutBody]), + Meta(&'a [StonePayloadMeta]), + Attributes(&'a [StonePayloadAttribute]), + Layout(&'a [StonePayloadLayout]), } impl<'a> From> for InnerPayload<'a> { @@ -199,10 +199,10 @@ impl<'a> From> for InnerPayload<'a> { /// doesn't support passing in `Index` payloads /// since it's a side-effect of [`Writer::add_content`] enum InnerPayload<'a> { - Meta(&'a [StonePayloadMetaBody]), - Attributes(&'a [StonePayloadAttributeBody]), - Layout(&'a [StonePayloadLayoutBody]), - Index(&'a [StonePayloadIndexBody]), + Meta(&'a [StonePayloadMeta]), + Attributes(&'a [StonePayloadAttribute]), + Layout(&'a [StonePayloadLayout]), + Index(&'a [StonePayloadIndex]), } impl InnerPayload<'_> { @@ -244,20 +244,20 @@ impl InnerPayload<'_> { } } -impl<'a> From<&'a [StonePayloadMetaBody]> for StoneWritePayload<'a> { - fn from(payload: &'a [StonePayloadMetaBody]) -> Self { +impl<'a> From<&'a [StonePayloadMeta]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadMeta]) -> Self { Self::Meta(payload) } } -impl<'a> From<&'a [StonePayloadAttributeBody]> for StoneWritePayload<'a> { - fn from(payload: &'a [StonePayloadAttributeBody]) -> Self { +impl<'a> From<&'a [StonePayloadAttribute]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadAttribute]) -> Self { Self::Attributes(payload) } } -impl<'a> From<&'a [StonePayloadLayoutBody]> for StoneWritePayload<'a> { - fn from(payload: &'a [StonePayloadLayoutBody]) -> Self { +impl<'a> From<&'a [StonePayloadLayout]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadLayout]) -> Self { Self::Layout(payload) } } diff --git a/justfile b/justfile index 0b5712b0..9a140450 100644 --- a/justfile +++ b/justfile @@ -84,3 +84,11 @@ diesel db +ARGS: --config-file {{root-dir}}/moss/src/db/{{db}}/diesel.toml \ --database-url sqlite://{{root-dir}}/moss/src/db/{{db}}/test.db \ {{ARGS}} + +test-ffi: + #!/bin/bash + cargo cbuild -p stone-ffi + gcc -o stone-ffi-test ./crates/stone-ffi/test.c -L./target/x86_64-unknown-linux-gnu/debug/ -lstone -I./target/x86_64-unknown-linux-gnu/debug/ + ln -sf $(pwd)/target/x86_64-unknown-linux-gnu/debug/libstone.so ./target/x86_64-unknown-linux-gnu/debug/libstone.so.0.24 + LD_LIBRARY_PATH=$(pwd)/target/x86_64-unknown-linux-gnu/debug/ ./stone-ffi-test + rm stone-ffi-test diff --git a/moss/src/client/boot.rs b/moss/src/client/boot.rs index b1ad10bc..d3c24e32 100644 --- a/moss/src/client/boot.rs +++ b/moss/src/client/boot.rs @@ -18,7 +18,7 @@ use blsforme::{ use fnmatch::Pattern; use fs_err as fs; use itertools::Itertools; -use stone::{StonePayloadLayoutBody, StonePayloadLayoutEntry}; +use stone::{StonePayloadLayout, StonePayloadLayoutEntry}; use thiserror::{self, Error}; use crate::{db, package::Id, Installation, State}; @@ -51,7 +51,7 @@ pub enum Error { #[derive(Debug)] struct KernelCandidate { path: PathBuf, - _layout: StonePayloadLayoutBody, + _layout: StonePayloadLayout, } impl AsRef for KernelCandidate { @@ -63,10 +63,7 @@ impl AsRef for KernelCandidate { /// From a given set of input paths, produce a set of match pairs /// NOTE: This only works for a *new* blit and doesn't retroactively /// sync old kernels! -fn kernel_files_from_state<'a>( - layouts: &'a [(Id, StonePayloadLayoutBody)], - pattern: &'a Pattern, -) -> Vec { +fn kernel_files_from_state<'a>(layouts: &'a [(Id, StonePayloadLayout)], pattern: &'a Pattern) -> Vec { let mut kernel_entries = vec![]; for (_, path) in layouts.iter() { @@ -97,7 +94,7 @@ fn kernel_files_from_state<'a>( /// Find bootloader assets in the new state fn boot_files_from_new_state<'a>( install: &Installation, - layouts: &'a [(Id, StonePayloadLayoutBody)], + layouts: &'a [(Id, StonePayloadLayout)], pattern: &'a Pattern, ) -> Vec { let mut rets = vec![]; @@ -114,7 +111,7 @@ fn boot_files_from_new_state<'a>( } /// Grab all layouts for the provided state, mapped to package id -fn layouts_for_state(client: &Client, state: &State) -> Result, db::Error> { +fn layouts_for_state(client: &Client, state: &State) -> Result, db::Error> { client.layout_db.query(state.selections.iter().map(|s| &s.package)) } diff --git a/moss/src/client/cache.rs b/moss/src/client/cache.rs index c8d30578..45fd095e 100644 --- a/moss/src/client/cache.rs +++ b/moss/src/client/cache.rs @@ -12,7 +12,7 @@ use std::{ }; use futures_util::StreamExt; -use stone::{StoneDecodedPayload, StonePayloadIndexBody, StoneReadError}; +use stone::{StoneDecodedPayload, StonePayloadIndex, StoneReadError}; use thiserror::Error; use tokio::io::AsyncWriteExt; use url::Url; @@ -257,7 +257,7 @@ impl Download { } /// Returns true if all assets already exist in the installation -fn check_assets_exist(indices: &[&StonePayloadIndexBody], installation: &Installation) -> bool { +fn check_assets_exist(indices: &[&StonePayloadIndex], installation: &Installation) -> bool { indices.iter().all(|index| { let path = asset_path(installation, &format!("{:02x}", index.digest)); path.exists() diff --git a/moss/src/client/mod.rs b/moss/src/client/mod.rs index b6aaa20e..87ef2bf6 100644 --- a/moss/src/client/mod.rs +++ b/moss/src/client/mod.rs @@ -26,7 +26,7 @@ use nix::{ unistd::{close, linkat, mkdir, symlinkat}, }; use postblit::TriggerScope; -use stone::{StoneDecodedPayload, StonePayloadLayoutBody, StonePayloadLayoutEntry}; +use stone::{StoneDecodedPayload, StonePayloadLayout, StonePayloadLayoutEntry}; use thiserror::Error; use tui::{MultiProgress, ProgressBar, ProgressStyle, Styled}; use vfs::tree::{builder::TreeBuilder, BlitFile, Element}; @@ -840,7 +840,7 @@ pub struct PendingFile { pub id: package::Id, /// Corresponding layout entry, describing the inode - pub layout: StonePayloadLayoutBody, + pub layout: StonePayloadLayout, } impl BlitFile for PendingFile { @@ -893,7 +893,7 @@ impl From for PendingFile { fn from(value: String) -> Self { PendingFile { id: Default::default(), - layout: StonePayloadLayoutBody { + layout: StonePayloadLayout { uid: 0, gid: 0, mode: 0o755, diff --git a/moss/src/db/layout/mod.rs b/moss/src/db/layout/mod.rs index a023bccc..455203d0 100644 --- a/moss/src/db/layout/mod.rs +++ b/moss/src/db/layout/mod.rs @@ -6,7 +6,7 @@ use diesel::prelude::*; use diesel::{Connection as _, SqliteConnection}; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use std::collections::BTreeSet; -use stone::{StonePayloadLayoutBody, StonePayloadLayoutEntry}; +use stone::{StonePayloadLayout, StonePayloadLayoutEntry}; use crate::package; @@ -37,7 +37,7 @@ impl Database { pub fn query<'a>( &self, packages: impl IntoIterator, - ) -> Result, Error> { + ) -> Result, Error> { self.conn.exec(|conn| { let packages = packages.into_iter().map(AsRef::::as_ref).collect::>(); @@ -58,7 +58,7 @@ impl Database { }) } - pub fn all(&self) -> Result, Error> { + pub fn all(&self) -> Result, Error> { self.conn.exec(|conn| { model::layout::table .select(model::Layout::as_select()) @@ -83,13 +83,13 @@ impl Database { }) } - pub fn add(&self, package: &package::Id, layout: &StonePayloadLayoutBody) -> Result<(), Error> { + pub fn add(&self, package: &package::Id, layout: &StonePayloadLayout) -> Result<(), Error> { self.batch_add(vec![(package, layout)]) } pub fn batch_add<'a>( &self, - layouts: impl IntoIterator, + layouts: impl IntoIterator, ) -> Result<(), Error> { self.conn.exclusive_tx(|tx| { let mut ids = vec![]; @@ -148,12 +148,12 @@ fn batch_remove_impl(packages: &[&str], tx: &mut SqliteConnection) -> Result<(), Ok(()) } -fn map_layout(result: QueryResult) -> Result<(package::Id, StonePayloadLayoutBody), Error> { +fn map_layout(result: QueryResult) -> Result<(package::Id, StonePayloadLayout), Error> { let row = result?; let entry = decode_entry(row.entry_type, row.entry_value1, row.entry_value2).ok_or(Error::LayoutEntryDecode)?; - let layout = StonePayloadLayoutBody { + let layout = StonePayloadLayout { uid: row.uid as u32, gid: row.gid as u32, mode: row.mode as u32, diff --git a/moss/src/package/meta.rs b/moss/src/package/meta.rs index 95191ffc..6271677f 100644 --- a/moss/src/package/meta.rs +++ b/moss/src/package/meta.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use derive_more::{AsRef, Display, From, Into}; -use stone::{StonePayloadMetaBody, StonePayloadMetaKind, StonePayloadMetaTag}; +use stone::{StonePayloadMeta, StonePayloadMetaKind, StonePayloadMetaTag}; use thiserror::Error; use crate::{dependency, Dependency, Provider}; @@ -62,7 +62,7 @@ pub struct Meta { } impl Meta { - pub fn from_stone_payload(payload: &[StonePayloadMetaBody]) -> Result { + pub fn from_stone_payload(payload: &[StonePayloadMeta]) -> Result { let name = find_meta_string(payload, StonePayloadMetaTag::Name)?; let version_identifier = find_meta_string(payload, StonePayloadMetaTag::Version)?; let source_release = find_meta_u64(payload, StonePayloadMetaTag::Release)?; @@ -112,7 +112,7 @@ impl Meta { }) } - pub fn to_stone_payload(self) -> Vec { + pub fn to_stone_payload(self) -> Vec { vec![ ( StonePayloadMetaTag::Name, @@ -195,7 +195,7 @@ impl Meta { ) }), ) - .map(|(tag, kind)| StonePayloadMetaBody { tag, kind }) + .map(|(tag, kind)| StonePayloadMeta { tag, kind }) .collect() } @@ -208,19 +208,19 @@ impl Meta { } } -fn find_meta_string(meta: &[StonePayloadMetaBody], tag: StonePayloadMetaTag) -> Result { +fn find_meta_string(meta: &[StonePayloadMeta], tag: StonePayloadMetaTag) -> Result { meta.iter() .find_map(|meta| meta_string(meta, tag)) .ok_or(MissingMetaFieldError(tag)) } -fn find_meta_u64(meta: &[StonePayloadMetaBody], tag: StonePayloadMetaTag) -> Result { +fn find_meta_u64(meta: &[StonePayloadMeta], tag: StonePayloadMetaTag) -> Result { meta.iter() .find_map(|meta| meta_u64(meta, tag)) .ok_or(MissingMetaFieldError(tag)) } -fn meta_u64(meta: &StonePayloadMetaBody, tag: StonePayloadMetaTag) -> Option { +fn meta_u64(meta: &StonePayloadMeta, tag: StonePayloadMetaTag) -> Option { if meta.tag == tag { Some(match meta.kind { StonePayloadMetaKind::Int8(i) => i as _, @@ -238,14 +238,14 @@ fn meta_u64(meta: &StonePayloadMetaBody, tag: StonePayloadMetaTag) -> Option Option { +fn meta_string(meta: &StonePayloadMeta, tag: StonePayloadMetaTag) -> Option { match (meta.tag, &meta.kind) { (meta_tag, StonePayloadMetaKind::String(value)) if meta_tag == tag => Some(value.clone()), _ => None, } } -fn meta_dependency(meta: &StonePayloadMetaBody) -> Option { +fn meta_dependency(meta: &StonePayloadMeta) -> Option { if let StonePayloadMetaKind::Dependency(kind, name) = meta.kind.clone() { Some(Dependency { kind: dependency::Kind::from(kind), @@ -256,7 +256,7 @@ fn meta_dependency(meta: &StonePayloadMetaBody) -> Option { } } -fn meta_provider(meta: &StonePayloadMetaBody) -> Option { +fn meta_provider(meta: &StonePayloadMeta) -> Option { match (meta.tag, meta.kind.clone()) { (StonePayloadMetaTag::Provides, StonePayloadMetaKind::Provider(kind, name)) => Some(Provider { kind: dependency::Kind::from(kind), @@ -266,7 +266,7 @@ fn meta_provider(meta: &StonePayloadMetaBody) -> Option { } } -fn meta_conflict(meta: &StonePayloadMetaBody) -> Option { +fn meta_conflict(meta: &StonePayloadMeta) -> Option { match (meta.tag, meta.kind.clone()) { (StonePayloadMetaTag::Conflicts, StonePayloadMetaKind::Provider(kind, name)) => Some(Provider { kind: dependency::Kind::from(kind), From b6121879f8808332096ee3cd965d3f6281a804ab Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Mon, 28 Oct 2024 14:23:35 -0700 Subject: [PATCH 03/12] Close file pointer in test --- crates/stone-ffi/test.c | 1 + justfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/stone-ffi/test.c b/crates/stone-ffi/test.c index bafb6986..d525d863 100644 --- a/crates/stone-ffi/test.c +++ b/crates/stone-ffi/test.c @@ -118,6 +118,7 @@ int main(int argc, char *argv[]) { fptr = fopen(file, "r"); stone_reader_read_file(fileno(fptr), &reader, &version); process_reader(reader, version); + fclose(fptr); printf("\n"); printf("Reading stone header from buffer\n\n"); diff --git a/justfile b/justfile index 9a140450..8fa7eb3a 100644 --- a/justfile +++ b/justfile @@ -90,5 +90,5 @@ test-ffi: cargo cbuild -p stone-ffi gcc -o stone-ffi-test ./crates/stone-ffi/test.c -L./target/x86_64-unknown-linux-gnu/debug/ -lstone -I./target/x86_64-unknown-linux-gnu/debug/ ln -sf $(pwd)/target/x86_64-unknown-linux-gnu/debug/libstone.so ./target/x86_64-unknown-linux-gnu/debug/libstone.so.0.24 - LD_LIBRARY_PATH=$(pwd)/target/x86_64-unknown-linux-gnu/debug/ ./stone-ffi-test + LD_LIBRARY_PATH=$(pwd)/target/x86_64-unknown-linux-gnu/debug/ valgrind ./stone-ffi-test rm stone-ffi-test From b115ee2d6ac93060fadfd168736289d8ff0ec181 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Tue, 29 Oct 2024 13:38:01 -0700 Subject: [PATCH 04/12] Add remaining reader methods --- crates/stone-ffi/src/lib.rs | 65 +++++- crates/stone-ffi/src/payload.rs | 121 +++++------ crates/stone-ffi/src/payload/attribute.rs | 19 ++ crates/stone-ffi/src/payload/index.rs | 17 ++ crates/stone-ffi/src/payload/layout.rs | 81 ++++++++ crates/stone-ffi/src/payload/meta.rs | 101 ++++++++++ crates/stone-ffi/test.c | 232 +++++++++++++++++++--- crates/stone/src/payload/meta.rs | 3 +- justfile | 2 +- 9 files changed, 538 insertions(+), 103 deletions(-) create mode 100644 crates/stone-ffi/src/payload/attribute.rs create mode 100644 crates/stone-ffi/src/payload/index.rs create mode 100644 crates/stone-ffi/src/payload/layout.rs create mode 100644 crates/stone-ffi/src/payload/meta.rs diff --git a/crates/stone-ffi/src/lib.rs b/crates/stone-ffi/src/lib.rs index 7b369180..f43c4c20 100644 --- a/crates/stone-ffi/src/lib.rs +++ b/crates/stone-ffi/src/lib.rs @@ -15,14 +15,18 @@ use std::{ use libc::{c_int, size_t}; use stone::{ StoneDecodedPayload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StoneHeaderVersion, - StonePayloadCompression, StonePayloadHeader, StonePayloadKind, StonePayloadLayoutFileType, StoneReadError, + StonePayloadCompression, StonePayloadHeader, StonePayloadKind, StonePayloadLayoutFileType, + StonePayloadMetaDependency, StonePayloadMetaTag, StoneReadError, }; -pub use self::payload::{StonePayload, StonePayloadLayoutRecord}; +pub use self::payload::{ + StonePayload, StonePayloadAttributeRecord, StonePayloadIndexRecord, StonePayloadLayoutRecord, + StonePayloadMetaRecord, +}; mod payload; -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] #[repr(C)] pub struct StoneString { pub buf: *const u8, @@ -217,6 +221,51 @@ pub unsafe extern "C" fn stone_payload_next_layout_record( }) } +#[no_mangle] +pub unsafe extern "C" fn stone_payload_next_meta_record( + payload: *mut StonePayload, + record: *mut StonePayloadMetaRecord, +) -> c_int { + fallible(|| { + let mut payload = NonNull::new(payload).ok_or("")?; + let mut record = NonNull::new(record).ok_or("")?; + + *record.as_mut() = payload.as_mut().next_meta_record().ok_or("")?; + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_next_index_record( + payload: *mut StonePayload, + record: *mut StonePayloadIndexRecord, +) -> c_int { + fallible(|| { + let mut payload = NonNull::new(payload).ok_or("")?; + let mut record = NonNull::new(record).ok_or("")?; + + *record.as_mut() = payload.as_mut().next_index_record().ok_or("")?; + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_next_attribute_record( + payload: *mut StonePayload, + record: *mut StonePayloadAttributeRecord, +) -> c_int { + fallible(|| { + let mut payload = NonNull::new(payload).ok_or("")?; + let mut record = NonNull::new(record).ok_or("")?; + + *record.as_mut() = payload.as_mut().next_attribute_record().ok_or("")?; + + Ok(()) + }) +} + #[no_mangle] pub unsafe extern "C" fn stone_payload_destroy(payload: *mut StonePayload) { let Some(payload) = NonNull::new(payload) else { @@ -246,6 +295,16 @@ pub unsafe extern "C" fn stone_format_payload_layout_file_type(file_type: StoneP fill_c_string(buf, file_type); } +#[no_mangle] +pub unsafe extern "C" fn stone_format_payload_meta_tag(tag: StonePayloadMetaTag, buf: *mut u8) { + fill_c_string(buf, tag); +} + +#[no_mangle] +pub unsafe extern "C" fn stone_format_payload_meta_dependency(dependency: StonePayloadMetaDependency, buf: *mut u8) { + fill_c_string(buf, dependency); +} + unsafe fn fill_c_string(buf: *mut u8, content: impl ToString) { let Some(buf) = NonNull::new(buf) else { return; diff --git a/crates/stone-ffi/src/payload.rs b/crates/stone-ffi/src/payload.rs index f5058ab5..e2e3f620 100644 --- a/crates/stone-ffi/src/payload.rs +++ b/crates/stone-ffi/src/payload.rs @@ -1,8 +1,14 @@ -use std::mem::ManuallyDrop; +use stone::{StoneDecodedPayload, StonePayloadHeader}; -use stone::{StoneDecodedPayload, StonePayloadHeader, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; +pub use self::attribute::StonePayloadAttributeRecord; +pub use self::index::StonePayloadIndexRecord; +pub use self::layout::StonePayloadLayoutRecord; +pub use self::meta::StonePayloadMetaRecord; -use super::StoneString; +mod attribute; +mod index; +mod layout; +mod meta; pub struct StonePayload { pub decoded: StoneDecodedPayload, @@ -24,42 +30,46 @@ impl StonePayload { let payload = self.decoded.layout()?; let record = payload.body.get(self.next_record)?; - Some(StonePayloadLayoutRecord { - uid: record.uid, - gid: record.gid, - mode: record.mode, - tag: record.tag, - file_type: record.entry.file_type(), - file_payload: match &record.entry { - StonePayloadLayoutEntry::Regular(hash, name) => StonePayloadLayoutFilePayload { - regular: ManuallyDrop::new(StonePayloadLayoutFileRegular { - hash: hash.to_be_bytes(), - name: StoneString::new(name), - }), - }, - StonePayloadLayoutEntry::Symlink(source, target) => StonePayloadLayoutFilePayload { - symlink: ManuallyDrop::new(StonePayloadLayoutFileSymlink { - source: StoneString::new(source), - target: StoneString::new(target), - }), - }, - StonePayloadLayoutEntry::Directory(name) => StonePayloadLayoutFilePayload { - directory: ManuallyDrop::new(StoneString::new(name)), - }, - StonePayloadLayoutEntry::CharacterDevice(name) => StonePayloadLayoutFilePayload { - character_device: ManuallyDrop::new(StoneString::new(name)), - }, - StonePayloadLayoutEntry::BlockDevice(name) => StonePayloadLayoutFilePayload { - block_device: ManuallyDrop::new(StoneString::new(name)), - }, - StonePayloadLayoutEntry::Fifo(name) => StonePayloadLayoutFilePayload { - fifo: ManuallyDrop::new(StoneString::new(name)), - }, - StonePayloadLayoutEntry::Socket(name) => StonePayloadLayoutFilePayload { - socket: ManuallyDrop::new(StoneString::new(name)), - }, - }, - }) + Some(record.into()) + } + + pub fn next_meta_record(&mut self) -> Option { + if self.next_record >= self.header().num_records { + return None; + } + + self.next_record += 1; + + let payload = self.decoded.meta()?; + let record = payload.body.get(self.next_record)?; + + Some(record.into()) + } + + pub fn next_index_record(&mut self) -> Option { + if self.next_record >= self.header().num_records { + return None; + } + + self.next_record += 1; + + let payload = self.decoded.index()?; + let record = payload.body.get(self.next_record)?; + + Some(record.into()) + } + + pub fn next_attribute_record(&mut self) -> Option { + if self.next_record >= self.header().num_records { + return None; + } + + self.next_record += 1; + + let payload = self.decoded.attributes()?; + let record = payload.body.get(self.next_record)?; + + Some(record.into()) } } @@ -71,36 +81,3 @@ impl From for StonePayload { } } } - -#[repr(C)] -pub struct StonePayloadLayoutRecord { - pub uid: u32, - pub gid: u32, - pub mode: u32, - pub tag: u32, - pub file_type: StonePayloadLayoutFileType, - pub file_payload: StonePayloadLayoutFilePayload, -} - -#[repr(C)] -pub union StonePayloadLayoutFilePayload { - regular: ManuallyDrop, - symlink: ManuallyDrop, - directory: ManuallyDrop, - character_device: ManuallyDrop, - block_device: ManuallyDrop, - fifo: ManuallyDrop, - socket: ManuallyDrop, -} - -#[repr(C)] -pub struct StonePayloadLayoutFileRegular { - pub hash: [u8; 16], - pub name: StoneString, -} - -#[repr(C)] -pub struct StonePayloadLayoutFileSymlink { - pub source: StoneString, - pub target: StoneString, -} diff --git a/crates/stone-ffi/src/payload/attribute.rs b/crates/stone-ffi/src/payload/attribute.rs new file mode 100644 index 00000000..9539c541 --- /dev/null +++ b/crates/stone-ffi/src/payload/attribute.rs @@ -0,0 +1,19 @@ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct StonePayloadAttributeRecord { + pub key_size: usize, + pub key_buf: *const u8, + pub value_size: usize, + pub value_buf: *const u8, +} + +impl From<&stone::StonePayloadAttribute> for StonePayloadAttributeRecord { + fn from(record: &stone::StonePayloadAttribute) -> Self { + Self { + key_size: record.key.len(), + key_buf: record.key.as_ptr(), + value_size: record.value.len(), + value_buf: record.value.as_ptr(), + } + } +} diff --git a/crates/stone-ffi/src/payload/index.rs b/crates/stone-ffi/src/payload/index.rs new file mode 100644 index 00000000..9d17c5ca --- /dev/null +++ b/crates/stone-ffi/src/payload/index.rs @@ -0,0 +1,17 @@ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct StonePayloadIndexRecord { + pub start: u64, + pub end: u64, + pub digest: [u8; 16], +} + +impl From<&stone::StonePayloadIndex> for StonePayloadIndexRecord { + fn from(record: &stone::StonePayloadIndex) -> Self { + Self { + start: record.start, + end: record.end, + digest: record.digest.to_be_bytes(), + } + } +} diff --git a/crates/stone-ffi/src/payload/layout.rs b/crates/stone-ffi/src/payload/layout.rs new file mode 100644 index 00000000..8c001355 --- /dev/null +++ b/crates/stone-ffi/src/payload/layout.rs @@ -0,0 +1,81 @@ +use stone::{StonePayloadLayout, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; + +use crate::StoneString; + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct StonePayloadLayoutRecord { + pub uid: u32, + pub gid: u32, + pub mode: u32, + pub tag: u32, + pub file_type: StonePayloadLayoutFileType, + pub file_payload: StonePayloadLayoutFilePayload, +} + +impl From<&StonePayloadLayout> for StonePayloadLayoutRecord { + fn from(record: &StonePayloadLayout) -> Self { + StonePayloadLayoutRecord { + uid: record.uid, + gid: record.gid, + mode: record.mode, + tag: record.tag, + file_type: record.entry.file_type(), + file_payload: match &record.entry { + StonePayloadLayoutEntry::Regular(hash, name) => StonePayloadLayoutFilePayload { + regular: StonePayloadLayoutFileRegular { + hash: hash.to_be_bytes(), + name: StoneString::new(name), + }, + }, + StonePayloadLayoutEntry::Symlink(source, target) => StonePayloadLayoutFilePayload { + symlink: StonePayloadLayoutFileSymlink { + source: StoneString::new(source), + target: StoneString::new(target), + }, + }, + StonePayloadLayoutEntry::Directory(name) => StonePayloadLayoutFilePayload { + directory: StoneString::new(name), + }, + StonePayloadLayoutEntry::CharacterDevice(name) => StonePayloadLayoutFilePayload { + character_device: StoneString::new(name), + }, + StonePayloadLayoutEntry::BlockDevice(name) => StonePayloadLayoutFilePayload { + block_device: StoneString::new(name), + }, + StonePayloadLayoutEntry::Fifo(name) => StonePayloadLayoutFilePayload { + fifo: StoneString::new(name), + }, + StonePayloadLayoutEntry::Socket(name) => StonePayloadLayoutFilePayload { + socket: StoneString::new(name), + }, + }, + } + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub union StonePayloadLayoutFilePayload { + regular: StonePayloadLayoutFileRegular, + symlink: StonePayloadLayoutFileSymlink, + directory: StoneString, + character_device: StoneString, + block_device: StoneString, + fifo: StoneString, + socket: StoneString, +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct StonePayloadLayoutFileRegular { + pub hash: [u8; 16], + pub name: StoneString, +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct StonePayloadLayoutFileSymlink { + pub source: StoneString, + pub target: StoneString, +} diff --git a/crates/stone-ffi/src/payload/meta.rs b/crates/stone-ffi/src/payload/meta.rs new file mode 100644 index 00000000..f9df29c2 --- /dev/null +++ b/crates/stone-ffi/src/payload/meta.rs @@ -0,0 +1,101 @@ +use stone::{StonePayloadMetaDependency, StonePayloadMetaTag}; + +use crate::StoneString; + +#[repr(C)] +pub struct StonePayloadMetaRecord { + pub tag: StonePayloadMetaTag, + pub primitive_type: StonePayloadMetaPrimitiveType, + pub primitive_payload: StonePayloadMetaPrimitivePayload, +} + +impl From<&stone::StonePayloadMeta> for StonePayloadMetaRecord { + fn from(record: &stone::StonePayloadMeta) -> Self { + Self { + tag: record.tag, + primitive_type: match &record.kind { + stone::StonePayloadMetaKind::Int8(_) => StonePayloadMetaPrimitiveType::Int8, + stone::StonePayloadMetaKind::Uint8(_) => StonePayloadMetaPrimitiveType::Uint8, + stone::StonePayloadMetaKind::Int16(_) => StonePayloadMetaPrimitiveType::Int16, + stone::StonePayloadMetaKind::Uint16(_) => StonePayloadMetaPrimitiveType::Uint16, + stone::StonePayloadMetaKind::Int32(_) => StonePayloadMetaPrimitiveType::Int32, + stone::StonePayloadMetaKind::Uint32(_) => StonePayloadMetaPrimitiveType::Uint32, + stone::StonePayloadMetaKind::Int64(_) => StonePayloadMetaPrimitiveType::Int64, + stone::StonePayloadMetaKind::Uint64(_) => StonePayloadMetaPrimitiveType::Uint64, + stone::StonePayloadMetaKind::String(_) => StonePayloadMetaPrimitiveType::String, + stone::StonePayloadMetaKind::Dependency(_, _) => StonePayloadMetaPrimitiveType::Dependency, + stone::StonePayloadMetaKind::Provider(_, _) => StonePayloadMetaPrimitiveType::Provider, + }, + primitive_payload: match &record.kind { + stone::StonePayloadMetaKind::Int8(a) => StonePayloadMetaPrimitivePayload { int8: *a }, + stone::StonePayloadMetaKind::Uint8(a) => StonePayloadMetaPrimitivePayload { uint8: *a }, + stone::StonePayloadMetaKind::Int16(a) => StonePayloadMetaPrimitivePayload { int16: *a }, + stone::StonePayloadMetaKind::Uint16(a) => StonePayloadMetaPrimitivePayload { uint16: *a }, + stone::StonePayloadMetaKind::Int32(a) => StonePayloadMetaPrimitivePayload { int32: *a }, + stone::StonePayloadMetaKind::Uint32(a) => StonePayloadMetaPrimitivePayload { uint32: *a }, + stone::StonePayloadMetaKind::Int64(a) => StonePayloadMetaPrimitivePayload { int64: *a }, + stone::StonePayloadMetaKind::Uint64(a) => StonePayloadMetaPrimitivePayload { uint64: *a }, + stone::StonePayloadMetaKind::String(a) => StonePayloadMetaPrimitivePayload { + string: StoneString::new(a), + }, + stone::StonePayloadMetaKind::Dependency(kind, name) => StonePayloadMetaPrimitivePayload { + dependency: StonePayloadMetaDependencyValue { + kind: *kind, + name: StoneString::new(name), + }, + }, + stone::StonePayloadMetaKind::Provider(kind, name) => StonePayloadMetaPrimitivePayload { + provider: StonePayloadMetaProviderValue { + kind: *kind, + name: StoneString::new(name), + }, + }, + }, + } + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub enum StonePayloadMetaPrimitiveType { + Int8, + Uint8, + Int16, + Uint16, + Int32, + Uint32, + Int64, + Uint64, + String, + Dependency, + Provider, +} + +#[repr(C)] +pub union StonePayloadMetaPrimitivePayload { + int8: i8, + uint8: u8, + int16: i16, + uint16: u16, + int32: i32, + uint32: u32, + int64: i64, + uint64: u64, + string: StoneString, + dependency: StonePayloadMetaDependencyValue, + provider: StonePayloadMetaProviderValue, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct StonePayloadMetaDependencyValue { + pub kind: StonePayloadMetaDependency, + pub name: StoneString, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct StonePayloadMetaProviderValue { + pub kind: StonePayloadMetaDependency, + pub name: StoneString, +} diff --git a/crates/stone-ffi/test.c b/crates/stone-ffi/test.c index d525d863..908bcf8d 100644 --- a/crates/stone-ffi/test.c +++ b/crates/stone-ffi/test.c @@ -12,7 +12,7 @@ static uint8_t HEADER_BUF[] = {0x00, 0x6d, 0x6f, 0x73, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x01}; void print_header_v1(StoneHeaderV1 *header) { - char file_type[100]; + uint8_t file_type[100]; stone_format_header_v1_file_type(header->file_type, file_type); @@ -23,24 +23,31 @@ void print_header_v1(StoneHeaderV1 *header) { } void print_payload_header(StonePayloadHeader *header) { - char compression[100]; - char kind[100]; + uint8_t compression[100]; + uint8_t kind[100]; stone_format_payload_compression(header->compression, compression); stone_format_payload_kind(header->kind, &(kind[0])); printf("StonePayload {\n"); printf(" kind: %s\n", kind); - printf(" plain_size: %d\n", header->plain_size); - printf(" stored_size: %d\n", header->stored_size); + printf(" plain_size: %ld\n", header->plain_size); + printf(" stored_size: %ld\n", header->stored_size); printf(" compression: %s\n", compression); - printf(" num_records: %d\n", header->num_records); + printf(" num_records: %ld\n", header->num_records); printf(" version: %d\n", header->version); printf("}\n"); } +void format_digest(uint8_t digest[16], char (*formatted)[32]) { + for (size_t i = 0; i < 16; i++) { + sprintf((char *)formatted + i * 2, "%.02x", digest[i]); + } +} + void print_payload_layout_record(StonePayloadLayoutRecord *record) { - char file_type[100]; + uint8_t file_type[100]; + char digest[32]; stone_format_payload_layout_file_type(record->file_type, file_type); @@ -53,59 +60,230 @@ void print_payload_layout_record(StonePayloadLayoutRecord *record) { switch (record->file_type) { case STONE_PAYLOAD_LAYOUT_FILE_TYPE_REGULAR: { - printf(" hash: "); - for (size_t i = 0; i < 16; i++) { - printf("%.02x", record->file_payload.regular.hash[i]); - } - printf("\n name: %.*s\n", record->file_payload.regular.name.size, + format_digest(record->file_payload.regular.hash, &digest); + printf(" hash: %.32s\n", digest); + printf(" name: %.*s\n", (int)record->file_payload.regular.name.size, record->file_payload.regular.name.buf); break; } case STONE_PAYLOAD_LAYOUT_FILE_TYPE_SYMLINK: { - printf(" source: %.*s\n", record->file_payload.symlink.source.size, + printf(" source: %.*s\n", (int)record->file_payload.symlink.source.size, record->file_payload.symlink.source.buf); - printf(" target: %.*s\n", record->file_payload.symlink.target.size, + printf(" target: %.*s\n", (int)record->file_payload.symlink.target.size, record->file_payload.symlink.target.buf); break; } - default: } printf("}\n"); } +void print_payload_meta_record(StonePayloadMetaRecord *record) { + uint8_t tag[100]; + + stone_format_payload_meta_tag(record->tag, tag); + + printf("StonePayloadMetaRecord {\n"); + printf(" tag: %s\n", tag); + + switch (record->primitive_type) { + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT8: { + printf(" int8: %d\n", record->primitive_payload.int8); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT8: { + printf(" uint8: %d\n", record->primitive_payload.uint8); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT16: { + printf(" int16: %d\n", record->primitive_payload.int16); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT16: { + printf(" uint16: %d\n", record->primitive_payload.uint16); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT32: { + printf(" int32: %d\n", record->primitive_payload.int32); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT32: { + printf(" uint32: %d\n", record->primitive_payload.uint32); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT64: { + printf(" int64: %ld\n", record->primitive_payload.int64); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT64: { + printf(" uint64: %ld\n", record->primitive_payload.uint64); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_STRING: { + printf(" string: %.*s\n", (int)record->primitive_payload.string.size, + record->primitive_payload.string.buf); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_DEPENDENCY: { + uint8_t dependency[100]; + + stone_format_payload_meta_dependency( + record->primitive_payload.dependency.kind, dependency); + + printf(" dependency: %s(%.*s)\n", dependency, + (int)record->primitive_payload.dependency.name.size, + record->primitive_payload.dependency.name.buf); + break; + } + case STONE_PAYLOAD_META_PRIMITIVE_TYPE_PROVIDER: { + uint8_t provider[100]; + + stone_format_payload_meta_dependency( + record->primitive_payload.provider.kind, provider); + + printf(" provider: %s(%.*s)\n", provider, + (int)record->primitive_payload.provider.name.size, + record->primitive_payload.provider.name.buf); + break; + } + } + + printf("}\n"); +} + +void print_payload_index_record(StonePayloadIndexRecord *record) { + char digest[32]; + + format_digest(record->digest, &digest); + + printf("StonePayloadIndexRecord {\n"); + printf(" start: %ld\n", record->start); + printf(" end: %ld\n", record->end); + printf(" digest: %.32s\n", digest); + printf("}\n"); +} + +void print_payload_attribute_record(StonePayloadAttributeRecord *record) { + printf("StonePayloadAttributeRecord {\n"); + printf(" key_size: %ld\n", record->key_size); + printf(" value_size: %ld\n", record->value_size); + printf("}\n"); +} + +void process_records(StonePayload *payload, StonePayloadHeader *payload_header, + void **records, int *num_records, int record_size, + int (*next_record)(StonePayload *, void *), + void (*print_record)(void *)) { + + void *record; + int i = 0; + + *records = + realloc(*records, sizeof(StonePayloadLayoutRecord) * + (*num_records + payload_header->num_records)); + if (records == NULL) { + exit(1); + } + + while (next_record(payload, record = *records + + (*num_records + i) * record_size) >= 0) { + print_record(record); + i++; + } + + *num_records += payload_header->num_records; +} + void process_reader(StoneReader *reader, StoneHeaderVersion version) { - StoneHeaderV1 header; - StonePayload *payload; + StoneHeaderV1 header = {0}; + StonePayload **payloads = NULL; + StonePayloadLayoutRecord *layouts = NULL; + StonePayloadMetaRecord *metas = NULL; + StonePayloadIndexRecord *indexes = NULL; + StonePayloadAttributeRecord *attributes = NULL; + int num_layouts = 0, num_metas = 0, num_indexes = 0, num_attributes = 0, + current_payload = 0; assert(version == STONE_HEADER_VERSION_V1); stone_reader_header_v1(reader, &header); print_header_v1(&header); - while (stone_reader_next_payload(reader, &payload) >= 0) { - StonePayloadHeader payload_header; + payloads = calloc(header.num_payloads, sizeof(StonePayload *)); + if (payloads == NULL) { + exit(1); + } + + while (stone_reader_next_payload(reader, &payloads[current_payload]) >= 0) { + StonePayload *payload = payloads[current_payload]; + StonePayloadHeader payload_header = {0}; stone_payload_header(payload, &payload_header); print_payload_header(&payload_header); switch (payload_header.kind) { case STONE_PAYLOAD_KIND_LAYOUT: { - StonePayloadLayoutRecord record; + process_records(payload, &payload_header, (void *)&layouts, &num_layouts, + sizeof(StonePayloadLayoutRecord), + (void *)stone_payload_next_layout_record, + (void *)print_payload_layout_record); + break; + } + case STONE_PAYLOAD_KIND_META: { + process_records(payload, &payload_header, (void *)&metas, &num_metas, + sizeof(StonePayloadMetaRecord), + (void *)stone_payload_next_meta_record, + (void *)print_payload_meta_record); + break; + } + case STONE_PAYLOAD_KIND_INDEX: { + process_records(payload, &payload_header, (void *)&indexes, &num_indexes, + sizeof(StonePayloadIndexRecord), + (void *)stone_payload_next_index_record, + (void *)print_payload_index_record); + break; + } + case STONE_PAYLOAD_KIND_ATTRIBUTES: { + process_records(payload, &payload_header, (void *)&attributes, + &num_attributes, sizeof(StonePayloadAttributeRecord), + (void *)stone_payload_next_attribute_record, + (void *)print_payload_attribute_record); + break; + } + case STONE_PAYLOAD_KIND_CONTENT: { + void *data = malloc(payload_header.plain_size); - while (stone_payload_next_layout_record(payload, &record) >= 0) { - print_payload_layout_record(&record); + if (data == NULL) { + exit(1); } - break; + stone_reader_unpack_content_payload(reader, payload, data); + + free(data); } - default: } - stone_payload_destroy(payload); + current_payload += 1; } - stone_reader_destroy(reader); + if (layouts != NULL) { + free(layouts); + } + if (metas != NULL) { + free(metas); + } + if (indexes != NULL) { + free(indexes); + } + if (attributes != NULL) { + free(attributes); + } + for (int i = 0; i < header.num_payloads; i++) { + stone_payload_destroy(payloads[i]); + } + if (payloads != NULL) { + free(payloads); + } } int main(int argc, char *argv[]) { @@ -118,12 +296,14 @@ int main(int argc, char *argv[]) { fptr = fopen(file, "r"); stone_reader_read_file(fileno(fptr), &reader, &version); process_reader(reader, version); + stone_reader_destroy(reader); fclose(fptr); printf("\n"); printf("Reading stone header from buffer\n\n"); stone_reader_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); process_reader(reader, version); + stone_reader_destroy(reader); return 0; } diff --git a/crates/stone/src/payload/meta.rs b/crates/stone/src/payload/meta.rs index 20845d06..8b3cd5ef 100644 --- a/crates/stone/src/payload/meta.rs +++ b/crates/stone/src/payload/meta.rs @@ -89,8 +89,9 @@ impl StonePayloadMetaKind { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] +#[strum(serialize_all = "kebab-case")] #[repr(u16)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StonePayloadMetaTag { // Name of the package Name = 1, diff --git a/justfile b/justfile index 8fa7eb3a..971721b7 100644 --- a/justfile +++ b/justfile @@ -90,5 +90,5 @@ test-ffi: cargo cbuild -p stone-ffi gcc -o stone-ffi-test ./crates/stone-ffi/test.c -L./target/x86_64-unknown-linux-gnu/debug/ -lstone -I./target/x86_64-unknown-linux-gnu/debug/ ln -sf $(pwd)/target/x86_64-unknown-linux-gnu/debug/libstone.so ./target/x86_64-unknown-linux-gnu/debug/libstone.so.0.24 - LD_LIBRARY_PATH=$(pwd)/target/x86_64-unknown-linux-gnu/debug/ valgrind ./stone-ffi-test + LD_LIBRARY_PATH=$(pwd)/target/x86_64-unknown-linux-gnu/debug/ valgrind --track-origins=yes ./stone-ffi-test rm stone-ffi-test From 4c17712b5a091aaa4defac35c281f303e00bdab0 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Tue, 29 Oct 2024 13:45:45 -0700 Subject: [PATCH 05/12] Rename all payload types to *Record --- boulder/src/package/collect.rs | 28 ++--- boulder/src/package/emit/manifest/binary.rs | 8 +- boulder/src/package/emit/manifest/json.rs | 2 +- crates/stone-ffi/src/payload/attribute.rs | 4 +- crates/stone-ffi/src/payload/index.rs | 4 +- crates/stone-ffi/src/payload/layout.rs | 24 ++--- crates/stone-ffi/src/payload/meta.rs | 52 +++++----- crates/stone/src/lib.rs | 8 +- crates/stone/src/payload/attribute.rs | 4 +- crates/stone/src/payload/index.rs | 4 +- crates/stone/src/payload/layout.rs | 68 ++++++------ crates/stone/src/payload/meta.rs | 106 +++++++++---------- crates/stone/src/payload/mod.rs | 10 +- crates/stone/src/read/mod.rs | 26 ++--- crates/stone/src/write.rs | 38 +++---- moss/src/cli/extract.rs | 10 +- moss/src/cli/info.rs | 8 +- moss/src/cli/inspect.rs | 36 ++++--- moss/src/client/boot.rs | 21 ++-- moss/src/client/cache.rs | 4 +- moss/src/client/mod.rs | 62 +++++------ moss/src/client/verify.rs | 4 +- moss/src/db/layout/mod.rs | 50 ++++----- moss/src/package/meta.rs | 108 +++++++++++--------- 24 files changed, 357 insertions(+), 332 deletions(-) diff --git a/boulder/src/package/collect.rs b/boulder/src/package/collect.rs index 6f38d89d..96dc0b48 100644 --- a/boulder/src/package/collect.rs +++ b/boulder/src/package/collect.rs @@ -12,7 +12,7 @@ use std::{ use fs_err as fs; use glob::Pattern; use nix::libc::{S_IFDIR, S_IRGRP, S_IROTH, S_IRWXU, S_IXGRP, S_IXOTH}; -use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayout, StonePayloadLayoutEntry}; +use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutFile, StonePayloadLayoutRecord}; use thiserror::Error; #[derive(Debug, Clone, Eq, PartialEq)] @@ -126,7 +126,7 @@ impl Collector { pub struct PathInfo { pub path: PathBuf, pub target_path: PathBuf, - pub layout: StonePayloadLayout, + pub layout: StonePayloadLayoutRecord, pub size: u64, pub package: String, } @@ -158,11 +158,11 @@ impl PathInfo { } pub fn is_file(&self) -> bool { - matches!(self.layout.entry, StonePayloadLayoutEntry::Regular(_, _)) + matches!(self.layout.file, StonePayloadLayoutFile::Regular(_, _)) } pub fn file_hash(&self) -> Option { - if let StonePayloadLayoutEntry::Regular(hash, _) = &self.layout.entry { + if let StonePayloadLayoutFile::Regular(hash, _) = &self.layout.file { Some(*hash) } else { None @@ -188,7 +188,7 @@ fn layout_from_metadata( target_path: &Path, metadata: &Metadata, hasher: &mut StoneDigestWriterHasher, -) -> Result { +) -> Result { // Strip /usr let target = target_path .strip_prefix("/usr") @@ -198,25 +198,25 @@ fn layout_from_metadata( let file_type = metadata.file_type(); - Ok(StonePayloadLayout { + Ok(StonePayloadLayoutRecord { uid: metadata.uid(), gid: metadata.gid(), mode: metadata.mode(), tag: 0, - entry: if file_type.is_symlink() { + file: if file_type.is_symlink() { let source = fs::read_link(path)?; - StonePayloadLayoutEntry::Symlink(source.to_string_lossy().to_string(), target) + StonePayloadLayoutFile::Symlink(source.to_string_lossy().to_string(), target) } else if file_type.is_dir() { - StonePayloadLayoutEntry::Directory(target) + StonePayloadLayoutFile::Directory(target) } else if file_type.is_char_device() { - StonePayloadLayoutEntry::CharacterDevice(target) + StonePayloadLayoutFile::CharacterDevice(target) } else if file_type.is_block_device() { - StonePayloadLayoutEntry::BlockDevice(target) + StonePayloadLayoutFile::BlockDevice(target) } else if file_type.is_fifo() { - StonePayloadLayoutEntry::Fifo(target) + StonePayloadLayoutFile::Fifo(target) } else if file_type.is_socket() { - StonePayloadLayoutEntry::Socket(target) + StonePayloadLayoutFile::Socket(target) } else { hasher.reset(); @@ -229,7 +229,7 @@ fn layout_from_metadata( let hash = hasher.digest128(); - StonePayloadLayoutEntry::Regular(hash, target) + StonePayloadLayoutFile::Regular(hash, target) }, }) } diff --git a/boulder/src/package/emit/manifest/binary.rs b/boulder/src/package/emit/manifest/binary.rs index 461f01ca..a3b59531 100644 --- a/boulder/src/package/emit/manifest/binary.rs +++ b/boulder/src/package/emit/manifest/binary.rs @@ -6,7 +6,9 @@ use std::{collections::BTreeSet, path::Path}; use fs_err::File; use moss::Dependency; -use stone::{StoneHeaderV1FileType, StonePayloadMeta, StonePayloadMetaKind, StonePayloadMetaTag, StoneWriter}; +use stone::{ + StoneHeaderV1FileType, StonePayloadMetaPrimitive, StonePayloadMetaRecord, StonePayloadMetaTag, StoneWriter, +}; use super::Error; use crate::package::emit::Package; @@ -26,9 +28,9 @@ pub fn write(path: &Path, packages: &BTreeSet<&Package<'_>>, build_deps: &BTreeS // Add build deps for name in build_deps { if let Ok(dep) = Dependency::from_name(name) { - payload.push(StonePayloadMeta { + payload.push(StonePayloadMetaRecord { tag: StonePayloadMetaTag::BuildDepends, - kind: StonePayloadMetaKind::Dependency(dep.kind.into(), dep.name), + primitive: StonePayloadMetaPrimitive::Dependency(dep.kind.into(), dep.name), }); } } diff --git a/boulder/src/package/emit/manifest/json.rs b/boulder/src/package/emit/manifest/json.rs index 9e652544..4f1c2430 100644 --- a/boulder/src/package/emit/manifest/json.rs +++ b/boulder/src/package/emit/manifest/json.rs @@ -42,7 +42,7 @@ pub fn write( .analysis .paths .iter() - .map(|p| format!("/usr/{}", p.layout.entry.target())) + .map(|p| format!("/usr/{}", p.layout.file.target())) .sorted() .collect(); diff --git a/crates/stone-ffi/src/payload/attribute.rs b/crates/stone-ffi/src/payload/attribute.rs index 9539c541..1c0664a0 100644 --- a/crates/stone-ffi/src/payload/attribute.rs +++ b/crates/stone-ffi/src/payload/attribute.rs @@ -7,8 +7,8 @@ pub struct StonePayloadAttributeRecord { pub value_buf: *const u8, } -impl From<&stone::StonePayloadAttribute> for StonePayloadAttributeRecord { - fn from(record: &stone::StonePayloadAttribute) -> Self { +impl From<&stone::StonePayloadAttributeRecord> for StonePayloadAttributeRecord { + fn from(record: &stone::StonePayloadAttributeRecord) -> Self { Self { key_size: record.key.len(), key_buf: record.key.as_ptr(), diff --git a/crates/stone-ffi/src/payload/index.rs b/crates/stone-ffi/src/payload/index.rs index 9d17c5ca..e8678188 100644 --- a/crates/stone-ffi/src/payload/index.rs +++ b/crates/stone-ffi/src/payload/index.rs @@ -6,8 +6,8 @@ pub struct StonePayloadIndexRecord { pub digest: [u8; 16], } -impl From<&stone::StonePayloadIndex> for StonePayloadIndexRecord { - fn from(record: &stone::StonePayloadIndex) -> Self { +impl From<&stone::StonePayloadIndexRecord> for StonePayloadIndexRecord { + fn from(record: &stone::StonePayloadIndexRecord) -> Self { Self { start: record.start, end: record.end, diff --git a/crates/stone-ffi/src/payload/layout.rs b/crates/stone-ffi/src/payload/layout.rs index 8c001355..bbe6ecbd 100644 --- a/crates/stone-ffi/src/payload/layout.rs +++ b/crates/stone-ffi/src/payload/layout.rs @@ -1,4 +1,4 @@ -use stone::{StonePayloadLayout, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; +use stone::{StonePayloadLayoutFile, StonePayloadLayoutFileType}; use crate::StoneString; @@ -13,40 +13,40 @@ pub struct StonePayloadLayoutRecord { pub file_payload: StonePayloadLayoutFilePayload, } -impl From<&StonePayloadLayout> for StonePayloadLayoutRecord { - fn from(record: &StonePayloadLayout) -> Self { +impl From<&stone::StonePayloadLayoutRecord> for StonePayloadLayoutRecord { + fn from(record: &stone::StonePayloadLayoutRecord) -> Self { StonePayloadLayoutRecord { uid: record.uid, gid: record.gid, mode: record.mode, tag: record.tag, - file_type: record.entry.file_type(), - file_payload: match &record.entry { - StonePayloadLayoutEntry::Regular(hash, name) => StonePayloadLayoutFilePayload { + file_type: record.file.file_type(), + file_payload: match &record.file { + StonePayloadLayoutFile::Regular(hash, name) => StonePayloadLayoutFilePayload { regular: StonePayloadLayoutFileRegular { hash: hash.to_be_bytes(), name: StoneString::new(name), }, }, - StonePayloadLayoutEntry::Symlink(source, target) => StonePayloadLayoutFilePayload { + StonePayloadLayoutFile::Symlink(source, target) => StonePayloadLayoutFilePayload { symlink: StonePayloadLayoutFileSymlink { source: StoneString::new(source), target: StoneString::new(target), }, }, - StonePayloadLayoutEntry::Directory(name) => StonePayloadLayoutFilePayload { + StonePayloadLayoutFile::Directory(name) => StonePayloadLayoutFilePayload { directory: StoneString::new(name), }, - StonePayloadLayoutEntry::CharacterDevice(name) => StonePayloadLayoutFilePayload { + StonePayloadLayoutFile::CharacterDevice(name) => StonePayloadLayoutFilePayload { character_device: StoneString::new(name), }, - StonePayloadLayoutEntry::BlockDevice(name) => StonePayloadLayoutFilePayload { + StonePayloadLayoutFile::BlockDevice(name) => StonePayloadLayoutFilePayload { block_device: StoneString::new(name), }, - StonePayloadLayoutEntry::Fifo(name) => StonePayloadLayoutFilePayload { + StonePayloadLayoutFile::Fifo(name) => StonePayloadLayoutFilePayload { fifo: StoneString::new(name), }, - StonePayloadLayoutEntry::Socket(name) => StonePayloadLayoutFilePayload { + StonePayloadLayoutFile::Socket(name) => StonePayloadLayoutFilePayload { socket: StoneString::new(name), }, }, diff --git a/crates/stone-ffi/src/payload/meta.rs b/crates/stone-ffi/src/payload/meta.rs index f9df29c2..6c5266ea 100644 --- a/crates/stone-ffi/src/payload/meta.rs +++ b/crates/stone-ffi/src/payload/meta.rs @@ -9,42 +9,42 @@ pub struct StonePayloadMetaRecord { pub primitive_payload: StonePayloadMetaPrimitivePayload, } -impl From<&stone::StonePayloadMeta> for StonePayloadMetaRecord { - fn from(record: &stone::StonePayloadMeta) -> Self { +impl From<&stone::StonePayloadMetaRecord> for StonePayloadMetaRecord { + fn from(record: &stone::StonePayloadMetaRecord) -> Self { Self { tag: record.tag, - primitive_type: match &record.kind { - stone::StonePayloadMetaKind::Int8(_) => StonePayloadMetaPrimitiveType::Int8, - stone::StonePayloadMetaKind::Uint8(_) => StonePayloadMetaPrimitiveType::Uint8, - stone::StonePayloadMetaKind::Int16(_) => StonePayloadMetaPrimitiveType::Int16, - stone::StonePayloadMetaKind::Uint16(_) => StonePayloadMetaPrimitiveType::Uint16, - stone::StonePayloadMetaKind::Int32(_) => StonePayloadMetaPrimitiveType::Int32, - stone::StonePayloadMetaKind::Uint32(_) => StonePayloadMetaPrimitiveType::Uint32, - stone::StonePayloadMetaKind::Int64(_) => StonePayloadMetaPrimitiveType::Int64, - stone::StonePayloadMetaKind::Uint64(_) => StonePayloadMetaPrimitiveType::Uint64, - stone::StonePayloadMetaKind::String(_) => StonePayloadMetaPrimitiveType::String, - stone::StonePayloadMetaKind::Dependency(_, _) => StonePayloadMetaPrimitiveType::Dependency, - stone::StonePayloadMetaKind::Provider(_, _) => StonePayloadMetaPrimitiveType::Provider, + primitive_type: match &record.primitive { + stone::StonePayloadMetaPrimitive::Int8(_) => StonePayloadMetaPrimitiveType::Int8, + stone::StonePayloadMetaPrimitive::Uint8(_) => StonePayloadMetaPrimitiveType::Uint8, + stone::StonePayloadMetaPrimitive::Int16(_) => StonePayloadMetaPrimitiveType::Int16, + stone::StonePayloadMetaPrimitive::Uint16(_) => StonePayloadMetaPrimitiveType::Uint16, + stone::StonePayloadMetaPrimitive::Int32(_) => StonePayloadMetaPrimitiveType::Int32, + stone::StonePayloadMetaPrimitive::Uint32(_) => StonePayloadMetaPrimitiveType::Uint32, + stone::StonePayloadMetaPrimitive::Int64(_) => StonePayloadMetaPrimitiveType::Int64, + stone::StonePayloadMetaPrimitive::Uint64(_) => StonePayloadMetaPrimitiveType::Uint64, + stone::StonePayloadMetaPrimitive::String(_) => StonePayloadMetaPrimitiveType::String, + stone::StonePayloadMetaPrimitive::Dependency(_, _) => StonePayloadMetaPrimitiveType::Dependency, + stone::StonePayloadMetaPrimitive::Provider(_, _) => StonePayloadMetaPrimitiveType::Provider, }, - primitive_payload: match &record.kind { - stone::StonePayloadMetaKind::Int8(a) => StonePayloadMetaPrimitivePayload { int8: *a }, - stone::StonePayloadMetaKind::Uint8(a) => StonePayloadMetaPrimitivePayload { uint8: *a }, - stone::StonePayloadMetaKind::Int16(a) => StonePayloadMetaPrimitivePayload { int16: *a }, - stone::StonePayloadMetaKind::Uint16(a) => StonePayloadMetaPrimitivePayload { uint16: *a }, - stone::StonePayloadMetaKind::Int32(a) => StonePayloadMetaPrimitivePayload { int32: *a }, - stone::StonePayloadMetaKind::Uint32(a) => StonePayloadMetaPrimitivePayload { uint32: *a }, - stone::StonePayloadMetaKind::Int64(a) => StonePayloadMetaPrimitivePayload { int64: *a }, - stone::StonePayloadMetaKind::Uint64(a) => StonePayloadMetaPrimitivePayload { uint64: *a }, - stone::StonePayloadMetaKind::String(a) => StonePayloadMetaPrimitivePayload { + primitive_payload: match &record.primitive { + stone::StonePayloadMetaPrimitive::Int8(a) => StonePayloadMetaPrimitivePayload { int8: *a }, + stone::StonePayloadMetaPrimitive::Uint8(a) => StonePayloadMetaPrimitivePayload { uint8: *a }, + stone::StonePayloadMetaPrimitive::Int16(a) => StonePayloadMetaPrimitivePayload { int16: *a }, + stone::StonePayloadMetaPrimitive::Uint16(a) => StonePayloadMetaPrimitivePayload { uint16: *a }, + stone::StonePayloadMetaPrimitive::Int32(a) => StonePayloadMetaPrimitivePayload { int32: *a }, + stone::StonePayloadMetaPrimitive::Uint32(a) => StonePayloadMetaPrimitivePayload { uint32: *a }, + stone::StonePayloadMetaPrimitive::Int64(a) => StonePayloadMetaPrimitivePayload { int64: *a }, + stone::StonePayloadMetaPrimitive::Uint64(a) => StonePayloadMetaPrimitivePayload { uint64: *a }, + stone::StonePayloadMetaPrimitive::String(a) => StonePayloadMetaPrimitivePayload { string: StoneString::new(a), }, - stone::StonePayloadMetaKind::Dependency(kind, name) => StonePayloadMetaPrimitivePayload { + stone::StonePayloadMetaPrimitive::Dependency(kind, name) => StonePayloadMetaPrimitivePayload { dependency: StonePayloadMetaDependencyValue { kind: *kind, name: StoneString::new(name), }, }, - stone::StonePayloadMetaKind::Provider(kind, name) => StonePayloadMetaPrimitivePayload { + stone::StonePayloadMetaPrimitive::Provider(kind, name) => StonePayloadMetaPrimitivePayload { provider: StonePayloadMetaProviderValue { kind: *kind, name: StoneString::new(name), diff --git a/crates/stone/src/lib.rs b/crates/stone/src/lib.rs index 233dbb3b..32cb10d6 100644 --- a/crates/stone/src/lib.rs +++ b/crates/stone/src/lib.rs @@ -13,10 +13,10 @@ pub use self::header::{ StoneHeaderV1FileType, StoneHeaderVersion, STONE_HEADER_MAGIC, }; pub use self::payload::{ - StonePayload, StonePayloadAttribute, StonePayloadCompression, StonePayloadContent, StonePayloadDecodeError, - StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndex, StonePayloadKind, StonePayloadLayout, - StonePayloadLayoutEntry, StonePayloadLayoutFileType, StonePayloadMeta, StonePayloadMetaDependency, - StonePayloadMetaKind, StonePayloadMetaTag, + StonePayload, StonePayloadAttributeRecord, StonePayloadCompression, StonePayloadContent, StonePayloadDecodeError, + StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndexRecord, StonePayloadKind, StonePayloadLayoutFile, + StonePayloadLayoutFileType, StonePayloadLayoutRecord, StonePayloadMetaDependency, StonePayloadMetaPrimitive, + StonePayloadMetaRecord, StonePayloadMetaTag, }; pub use self::read::{read, read_bytes, StoneDecodedPayload, StoneReadError, StoneReader}; pub use self::write::{ diff --git a/crates/stone/src/payload/attribute.rs b/crates/stone/src/payload/attribute.rs index 8091ed41..60632f0a 100644 --- a/crates/stone/src/payload/attribute.rs +++ b/crates/stone/src/payload/attribute.rs @@ -8,12 +8,12 @@ use super::{Record, StonePayloadDecodeError, StonePayloadEncodeError}; use crate::ext::{ReadExt, WriteExt}; #[derive(Debug, Clone)] -pub struct StonePayloadAttribute { +pub struct StonePayloadAttributeRecord { pub key: Vec, pub value: Vec, } -impl Record for StonePayloadAttribute { +impl Record for StonePayloadAttributeRecord { fn decode(mut reader: R) -> Result { let key_length = reader.read_u64()?; let value_length = reader.read_u64()?; diff --git a/crates/stone/src/payload/index.rs b/crates/stone/src/payload/index.rs index 52d78aad..d8670323 100644 --- a/crates/stone/src/payload/index.rs +++ b/crates/stone/src/payload/index.rs @@ -13,7 +13,7 @@ use crate::ext::{ReadExt, WriteExt}; /// This is used to split the file into the content store on disk before promoting /// to a transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct StonePayloadIndex { +pub struct StonePayloadIndexRecord { /// Start pf the entry within the ContentPayload pub start: u64, @@ -24,7 +24,7 @@ pub struct StonePayloadIndex { pub digest: u128, } -impl Record for StonePayloadIndex { +impl Record for StonePayloadIndexRecord { fn decode(mut reader: R) -> Result { let start = reader.read_u64()?; let end = reader.read_u64()?; diff --git a/crates/stone/src/payload/layout.rs b/crates/stone/src/payload/layout.rs index dfc8f819..9326ad1e 100644 --- a/crates/stone/src/payload/layout.rs +++ b/crates/stone/src/payload/layout.rs @@ -36,7 +36,7 @@ pub enum StonePayloadLayoutFileType { } #[derive(Debug, Clone, PartialEq, Eq)] -pub enum StonePayloadLayoutEntry { +pub enum StonePayloadLayoutFile { Regular(u128, String), Symlink(String, String), Directory(String), @@ -48,54 +48,54 @@ pub enum StonePayloadLayoutEntry { Socket(String), } -impl StonePayloadLayoutEntry { +impl StonePayloadLayoutFile { fn source(&self) -> Vec { match self { - StonePayloadLayoutEntry::Regular(hash, _) => hash.to_be_bytes().to_vec(), - StonePayloadLayoutEntry::Symlink(source, _) => source.as_bytes().to_vec(), - StonePayloadLayoutEntry::Directory(_) => vec![], - StonePayloadLayoutEntry::CharacterDevice(_) => vec![], - StonePayloadLayoutEntry::BlockDevice(_) => vec![], - StonePayloadLayoutEntry::Fifo(_) => vec![], - StonePayloadLayoutEntry::Socket(_) => vec![], + StonePayloadLayoutFile::Regular(hash, _) => hash.to_be_bytes().to_vec(), + StonePayloadLayoutFile::Symlink(source, _) => source.as_bytes().to_vec(), + StonePayloadLayoutFile::Directory(_) => vec![], + StonePayloadLayoutFile::CharacterDevice(_) => vec![], + StonePayloadLayoutFile::BlockDevice(_) => vec![], + StonePayloadLayoutFile::Fifo(_) => vec![], + StonePayloadLayoutFile::Socket(_) => vec![], } } pub fn target(&self) -> &str { match self { - StonePayloadLayoutEntry::Regular(_, target) => target, - StonePayloadLayoutEntry::Symlink(_, target) => target, - StonePayloadLayoutEntry::Directory(target) => target, - StonePayloadLayoutEntry::CharacterDevice(target) => target, - StonePayloadLayoutEntry::BlockDevice(target) => target, - StonePayloadLayoutEntry::Fifo(target) => target, - StonePayloadLayoutEntry::Socket(target) => target, + StonePayloadLayoutFile::Regular(_, target) => target, + StonePayloadLayoutFile::Symlink(_, target) => target, + StonePayloadLayoutFile::Directory(target) => target, + StonePayloadLayoutFile::CharacterDevice(target) => target, + StonePayloadLayoutFile::BlockDevice(target) => target, + StonePayloadLayoutFile::Fifo(target) => target, + StonePayloadLayoutFile::Socket(target) => target, } } pub fn file_type(&self) -> StonePayloadLayoutFileType { match self { - StonePayloadLayoutEntry::Regular(..) => StonePayloadLayoutFileType::Regular, - StonePayloadLayoutEntry::Symlink(..) => StonePayloadLayoutFileType::Symlink, - StonePayloadLayoutEntry::Directory(_) => StonePayloadLayoutFileType::Directory, - StonePayloadLayoutEntry::CharacterDevice(_) => StonePayloadLayoutFileType::CharacterDevice, - StonePayloadLayoutEntry::BlockDevice(_) => StonePayloadLayoutFileType::BlockDevice, - StonePayloadLayoutEntry::Fifo(_) => StonePayloadLayoutFileType::Fifo, - StonePayloadLayoutEntry::Socket(_) => StonePayloadLayoutFileType::Socket, + StonePayloadLayoutFile::Regular(..) => StonePayloadLayoutFileType::Regular, + StonePayloadLayoutFile::Symlink(..) => StonePayloadLayoutFileType::Symlink, + StonePayloadLayoutFile::Directory(_) => StonePayloadLayoutFileType::Directory, + StonePayloadLayoutFile::CharacterDevice(_) => StonePayloadLayoutFileType::CharacterDevice, + StonePayloadLayoutFile::BlockDevice(_) => StonePayloadLayoutFileType::BlockDevice, + StonePayloadLayoutFile::Fifo(_) => StonePayloadLayoutFileType::Fifo, + StonePayloadLayoutFile::Socket(_) => StonePayloadLayoutFileType::Socket, } } } #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StonePayloadLayout { +pub struct StonePayloadLayoutRecord { pub uid: u32, pub gid: u32, pub mode: u32, pub tag: u32, - pub entry: StonePayloadLayoutEntry, + pub file: StonePayloadLayoutFile, } -impl Record for StonePayloadLayout { +impl Record for StonePayloadLayoutRecord { fn decode(mut reader: R) -> Result { let uid = reader.read_u32()?; let gid = reader.read_u32()?; @@ -125,14 +125,14 @@ impl Record for StonePayloadLayout { StonePayloadLayoutFileType::Regular => { let source = reader.read_vec(source_length as usize)?; let hash = u128::from_be_bytes(source.try_into().unwrap()); - StonePayloadLayoutEntry::Regular(hash, sanitize(reader.read_string(target_length as u64)?)) + StonePayloadLayoutFile::Regular(hash, sanitize(reader.read_string(target_length as u64)?)) } - StonePayloadLayoutFileType::Symlink => StonePayloadLayoutEntry::Symlink( + StonePayloadLayoutFileType::Symlink => StonePayloadLayoutFile::Symlink( sanitize(reader.read_string(source_length as u64)?), sanitize(reader.read_string(target_length as u64)?), ), StonePayloadLayoutFileType::Directory => { - StonePayloadLayoutEntry::Directory(sanitize(reader.read_string(target_length as u64)?)) + StonePayloadLayoutFile::Directory(sanitize(reader.read_string(target_length as u64)?)) } _ => { if source_length > 0 { @@ -147,7 +147,7 @@ impl Record for StonePayloadLayout { gid, mode, tag, - entry, + file: entry, }) } @@ -157,12 +157,12 @@ impl Record for StonePayloadLayout { writer.write_u32(self.mode)?; writer.write_u32(self.tag)?; - let source = self.entry.source(); - let target = self.entry.target(); + let source = self.file.source(); + let target = self.file.target(); writer.write_u16(source.len() as u16)?; writer.write_u16(target.len() as u16)?; - writer.write_u8(self.entry.file_type() as u8)?; + writer.write_u8(self.file.file_type() as u8)?; writer.write_array([0; 11])?; writer.write_all(&source)?; writer.write_all(target.as_bytes())?; @@ -171,6 +171,6 @@ impl Record for StonePayloadLayout { } fn size(&self) -> usize { - 4 + 4 + 4 + 4 + 2 + 2 + 1 + 11 + self.entry.source().len() + self.entry.target().len() + 4 + 4 + 4 + 4 + 2 + 2 + 1 + 11 + self.file.source().len() + self.file.target().len() } } diff --git a/crates/stone/src/payload/meta.rs b/crates/stone/src/payload/meta.rs index 8b3cd5ef..c53802a9 100644 --- a/crates/stone/src/payload/meta.rs +++ b/crates/stone/src/payload/meta.rs @@ -12,9 +12,9 @@ use crate::ext::{ReadExt, WriteExt}; /// These record all metadata for every .stone packages and provide /// no content #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StonePayloadMeta { +pub struct StonePayloadMetaRecord { pub tag: StonePayloadMetaTag, - pub kind: StonePayloadMetaKind, + pub primitive: StonePayloadMetaPrimitive, } #[repr(u8)] @@ -54,7 +54,7 @@ pub enum StonePayloadMetaDependency { #[repr(u8)] #[derive(Debug, Clone, PartialEq, Eq)] -pub enum StonePayloadMetaKind { +pub enum StonePayloadMetaPrimitive { Int8(i8), Uint8(u8), Int16(i16), @@ -68,23 +68,23 @@ pub enum StonePayloadMetaKind { Provider(StonePayloadMetaDependency, String), } -impl StonePayloadMetaKind { +impl StonePayloadMetaPrimitive { fn size(&self) -> usize { match self { - StonePayloadMetaKind::Int8(_) => size_of::(), - StonePayloadMetaKind::Uint8(_) => size_of::(), - StonePayloadMetaKind::Int16(_) => size_of::(), - StonePayloadMetaKind::Uint16(_) => size_of::(), - StonePayloadMetaKind::Int32(_) => size_of::(), - StonePayloadMetaKind::Uint32(_) => size_of::(), - StonePayloadMetaKind::Int64(_) => size_of::(), - StonePayloadMetaKind::Uint64(_) => size_of::(), + StonePayloadMetaPrimitive::Int8(_) => size_of::(), + StonePayloadMetaPrimitive::Uint8(_) => size_of::(), + StonePayloadMetaPrimitive::Int16(_) => size_of::(), + StonePayloadMetaPrimitive::Uint16(_) => size_of::(), + StonePayloadMetaPrimitive::Int32(_) => size_of::(), + StonePayloadMetaPrimitive::Uint32(_) => size_of::(), + StonePayloadMetaPrimitive::Int64(_) => size_of::(), + StonePayloadMetaPrimitive::Uint64(_) => size_of::(), // nul terminator - StonePayloadMetaKind::String(s) => s.len() + 1, + StonePayloadMetaPrimitive::String(s) => s.len() + 1, // Plus dep size & nul terminator - StonePayloadMetaKind::Dependency(_, s) => s.len() + 2, + StonePayloadMetaPrimitive::Dependency(_, s) => s.len() + 2, // Plus dep size & nul terminator - StonePayloadMetaKind::Provider(_, s) => s.len() + 2, + StonePayloadMetaPrimitive::Provider(_, s) => s.len() + 2, } } } @@ -152,7 +152,7 @@ fn decode_dependency(i: u8) -> Result(mut reader: R) -> Result { let length = reader.read_u32()?; @@ -187,21 +187,21 @@ impl Record for StonePayloadMeta { let sanitize = |s: String| s.trim_end_matches('\0').to_owned(); let kind = match kind { - 1 => StonePayloadMetaKind::Int8(reader.read_u8()? as i8), - 2 => StonePayloadMetaKind::Uint8(reader.read_u8()?), - 3 => StonePayloadMetaKind::Int16(reader.read_u16()? as i16), - 4 => StonePayloadMetaKind::Uint16(reader.read_u16()?), - 5 => StonePayloadMetaKind::Int32(reader.read_u32()? as i32), - 6 => StonePayloadMetaKind::Uint32(reader.read_u32()?), - 7 => StonePayloadMetaKind::Int64(reader.read_u64()? as i64), - 8 => StonePayloadMetaKind::Uint64(reader.read_u64()?), - 9 => StonePayloadMetaKind::String(sanitize(reader.read_string(length as u64)?)), - 10 => StonePayloadMetaKind::Dependency( + 1 => StonePayloadMetaPrimitive::Int8(reader.read_u8()? as i8), + 2 => StonePayloadMetaPrimitive::Uint8(reader.read_u8()?), + 3 => StonePayloadMetaPrimitive::Int16(reader.read_u16()? as i16), + 4 => StonePayloadMetaPrimitive::Uint16(reader.read_u16()?), + 5 => StonePayloadMetaPrimitive::Int32(reader.read_u32()? as i32), + 6 => StonePayloadMetaPrimitive::Uint32(reader.read_u32()?), + 7 => StonePayloadMetaPrimitive::Int64(reader.read_u64()? as i64), + 8 => StonePayloadMetaPrimitive::Uint64(reader.read_u64()?), + 9 => StonePayloadMetaPrimitive::String(sanitize(reader.read_string(length as u64)?)), + 10 => StonePayloadMetaPrimitive::Dependency( // DependencyKind u8 subtracted from length decode_dependency(reader.read_u8()?)?, sanitize(reader.read_string(length as u64 - 1)?), ), - 11 => StonePayloadMetaKind::Provider( + 11 => StonePayloadMetaPrimitive::Provider( // DependencyKind u8 subtracted from length decode_dependency(reader.read_u8()?)?, sanitize(reader.read_string(length as u64 - 1)?), @@ -209,44 +209,44 @@ impl Record for StonePayloadMeta { k => return Err(StonePayloadDecodeError::UnknownMetaKind(k)), }; - Ok(Self { tag, kind }) + Ok(Self { tag, primitive: kind }) } fn encode(&self, writer: &mut W) -> Result<(), StonePayloadEncodeError> { - let kind = match self.kind { - StonePayloadMetaKind::Int8(_) => 1, - StonePayloadMetaKind::Uint8(_) => 2, - StonePayloadMetaKind::Int16(_) => 3, - StonePayloadMetaKind::Uint16(_) => 4, - StonePayloadMetaKind::Int32(_) => 5, - StonePayloadMetaKind::Uint32(_) => 6, - StonePayloadMetaKind::Int64(_) => 7, - StonePayloadMetaKind::Uint64(_) => 8, - StonePayloadMetaKind::String(_) => 9, - StonePayloadMetaKind::Dependency(_, _) => 10, - StonePayloadMetaKind::Provider(_, _) => 11, + let kind = match self.primitive { + StonePayloadMetaPrimitive::Int8(_) => 1, + StonePayloadMetaPrimitive::Uint8(_) => 2, + StonePayloadMetaPrimitive::Int16(_) => 3, + StonePayloadMetaPrimitive::Uint16(_) => 4, + StonePayloadMetaPrimitive::Int32(_) => 5, + StonePayloadMetaPrimitive::Uint32(_) => 6, + StonePayloadMetaPrimitive::Int64(_) => 7, + StonePayloadMetaPrimitive::Uint64(_) => 8, + StonePayloadMetaPrimitive::String(_) => 9, + StonePayloadMetaPrimitive::Dependency(_, _) => 10, + StonePayloadMetaPrimitive::Provider(_, _) => 11, }; - writer.write_u32(self.kind.size() as u32)?; + writer.write_u32(self.primitive.size() as u32)?; writer.write_u16(self.tag as u16)?; writer.write_u8(kind)?; // Padding writer.write_array::<1>([0])?; - match &self.kind { - StonePayloadMetaKind::Int8(i) => writer.write_u8(*i as u8)?, - StonePayloadMetaKind::Uint8(i) => writer.write_u8(*i)?, - StonePayloadMetaKind::Int16(i) => writer.write_u16(*i as u16)?, - StonePayloadMetaKind::Uint16(i) => writer.write_u16(*i)?, - StonePayloadMetaKind::Int32(i) => writer.write_u32(*i as u32)?, - StonePayloadMetaKind::Uint32(i) => writer.write_u32(*i)?, - StonePayloadMetaKind::Int64(i) => writer.write_u64(*i as u64)?, - StonePayloadMetaKind::Uint64(i) => writer.write_u64(*i)?, - StonePayloadMetaKind::String(s) => { + match &self.primitive { + StonePayloadMetaPrimitive::Int8(i) => writer.write_u8(*i as u8)?, + StonePayloadMetaPrimitive::Uint8(i) => writer.write_u8(*i)?, + StonePayloadMetaPrimitive::Int16(i) => writer.write_u16(*i as u16)?, + StonePayloadMetaPrimitive::Uint16(i) => writer.write_u16(*i)?, + StonePayloadMetaPrimitive::Int32(i) => writer.write_u32(*i as u32)?, + StonePayloadMetaPrimitive::Uint32(i) => writer.write_u32(*i)?, + StonePayloadMetaPrimitive::Int64(i) => writer.write_u64(*i as u64)?, + StonePayloadMetaPrimitive::Uint64(i) => writer.write_u64(*i)?, + StonePayloadMetaPrimitive::String(s) => { writer.write_all(s.as_bytes())?; writer.write_u8(b'\0')?; } - StonePayloadMetaKind::Dependency(dep, s) | StonePayloadMetaKind::Provider(dep, s) => { + StonePayloadMetaPrimitive::Dependency(dep, s) | StonePayloadMetaPrimitive::Provider(dep, s) => { writer.write_u8(*dep as u8)?; writer.write_all(s.as_bytes())?; writer.write_u8(b'\0')?; @@ -257,6 +257,6 @@ impl Record for StonePayloadMeta { } fn size(&self) -> usize { - 4 + 2 + 1 + 1 + self.kind.size() + 4 + 2 + 1 + 1 + self.primitive.size() } } diff --git a/crates/stone/src/payload/mod.rs b/crates/stone/src/payload/mod.rs index d6a519d4..367a7b57 100644 --- a/crates/stone/src/payload/mod.rs +++ b/crates/stone/src/payload/mod.rs @@ -14,11 +14,13 @@ use thiserror::Error; use crate::ext::{ReadExt, WriteExt}; -pub use self::attribute::StonePayloadAttribute; +pub use self::attribute::StonePayloadAttributeRecord; pub use self::content::StonePayloadContent; -pub use self::index::StonePayloadIndex; -pub use self::layout::{StonePayloadLayout, StonePayloadLayoutEntry, StonePayloadLayoutFileType}; -pub use self::meta::{StonePayloadMeta, StonePayloadMetaDependency, StonePayloadMetaKind, StonePayloadMetaTag}; +pub use self::index::StonePayloadIndexRecord; +pub use self::layout::{StonePayloadLayoutFile, StonePayloadLayoutFileType, StonePayloadLayoutRecord}; +pub use self::meta::{ + StonePayloadMetaDependency, StonePayloadMetaPrimitive, StonePayloadMetaRecord, StonePayloadMetaTag, +}; #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)] #[strum(serialize_all = "kebab-case")] diff --git a/crates/stone/src/read/mod.rs b/crates/stone/src/read/mod.rs index 5695415d..dec320c1 100644 --- a/crates/stone/src/read/mod.rs +++ b/crates/stone/src/read/mod.rs @@ -7,9 +7,9 @@ use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; use thiserror::Error; use crate::{ - payload, StoneHeader, StoneHeaderDecodeError, StonePayload, StonePayloadAttribute, StonePayloadCompression, - StonePayloadContent, StonePayloadDecodeError, StonePayloadHeader, StonePayloadIndex, StonePayloadKind, - StonePayloadLayout, StonePayloadMeta, + payload, StoneHeader, StoneHeaderDecodeError, StonePayload, StonePayloadAttributeRecord, StonePayloadCompression, + StonePayloadContent, StonePayloadDecodeError, StonePayloadHeader, StonePayloadIndexRecord, StonePayloadKind, + StonePayloadLayoutRecord, StonePayloadMetaRecord, }; use self::zstd::Zstd; @@ -127,10 +127,10 @@ impl Read for PayloadReader { #[derive(Debug)] pub enum StoneDecodedPayload { - Meta(StonePayload>), - Attributes(StonePayload>), - Layout(StonePayload>), - Index(StonePayload>), + Meta(StonePayload>), + Attributes(StonePayload>), + Layout(StonePayload>), + Index(StonePayload>), Content(StonePayload), } @@ -207,7 +207,7 @@ impl StoneDecodedPayload { } } - pub fn meta(&self) -> Option<&StonePayload>> { + pub fn meta(&self) -> Option<&StonePayload>> { if let Self::Meta(meta) = self { Some(meta) } else { @@ -215,7 +215,7 @@ impl StoneDecodedPayload { } } - pub fn attributes(&self) -> Option<&StonePayload>> { + pub fn attributes(&self) -> Option<&StonePayload>> { if let Self::Attributes(attributes) = self { Some(attributes) } else { @@ -223,7 +223,7 @@ impl StoneDecodedPayload { } } - pub fn layout(&self) -> Option<&StonePayload>> { + pub fn layout(&self) -> Option<&StonePayload>> { if let Self::Layout(layouts) = self { Some(layouts) } else { @@ -231,7 +231,7 @@ impl StoneDecodedPayload { } } - pub fn index(&self) -> Option<&StonePayload>> { + pub fn index(&self) -> Option<&StonePayload>> { if let Self::Index(indices) = self { Some(indices) } else { @@ -277,7 +277,7 @@ pub enum StoneReadError { mod test { use xxhash_rust::xxh3::xxh3_128; - use crate::{StoneHeaderVersion, StonePayloadLayoutEntry}; + use crate::{StoneHeaderVersion, StonePayloadLayoutFile}; use super::*; @@ -326,7 +326,7 @@ mod test { .filter_map(StoneDecodedPayload::layout) .flat_map(|p| &p.body) .find(|layout| { - if let StonePayloadLayoutEntry::Regular(digest, _) = &layout.entry { + if let StonePayloadLayoutFile::Regular(digest, _) = &layout.file { return *digest == index.digest; } false diff --git a/crates/stone/src/write.rs b/crates/stone/src/write.rs index 79d5fe96..a7d0ef0d 100644 --- a/crates/stone/src/write.rs +++ b/crates/stone/src/write.rs @@ -6,9 +6,9 @@ use std::io::{self, Read, Seek, SeekFrom, Write}; use thiserror::Error; use crate::{ - payload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StonePayloadAttribute, StonePayloadCompression, - StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndex, StonePayloadKind, StonePayloadLayout, - StonePayloadMeta, + payload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StonePayloadAttributeRecord, StonePayloadCompression, + StonePayloadEncodeError, StonePayloadHeader, StonePayloadIndexRecord, StonePayloadKind, StonePayloadLayoutRecord, + StonePayloadMetaRecord, }; pub use self::digest::{StoneDigestWriter, StoneDigestWriterHasher}; @@ -129,7 +129,9 @@ where let end = self.content.plain_size; // Add index data - self.content.indices.push(StonePayloadIndex { start, end, digest }); + self.content + .indices + .push(StonePayloadIndexRecord { start, end, digest }); Ok(()) } @@ -164,7 +166,7 @@ pub struct StoneContentWriter { buffer: B, plain_size: u64, stored_size: u64, - indices: Vec, + indices: Vec, /// Used to generate un-compressed digest of file /// contents used for [`Index`] index_hasher: StoneDigestWriterHasher, @@ -180,9 +182,9 @@ struct EncodedPayload { } pub enum StoneWritePayload<'a> { - Meta(&'a [StonePayloadMeta]), - Attributes(&'a [StonePayloadAttribute]), - Layout(&'a [StonePayloadLayout]), + Meta(&'a [StonePayloadMetaRecord]), + Attributes(&'a [StonePayloadAttributeRecord]), + Layout(&'a [StonePayloadLayoutRecord]), } impl<'a> From> for InnerPayload<'a> { @@ -199,10 +201,10 @@ impl<'a> From> for InnerPayload<'a> { /// doesn't support passing in `Index` payloads /// since it's a side-effect of [`Writer::add_content`] enum InnerPayload<'a> { - Meta(&'a [StonePayloadMeta]), - Attributes(&'a [StonePayloadAttribute]), - Layout(&'a [StonePayloadLayout]), - Index(&'a [StonePayloadIndex]), + Meta(&'a [StonePayloadMetaRecord]), + Attributes(&'a [StonePayloadAttributeRecord]), + Layout(&'a [StonePayloadLayoutRecord]), + Index(&'a [StonePayloadIndexRecord]), } impl InnerPayload<'_> { @@ -244,20 +246,20 @@ impl InnerPayload<'_> { } } -impl<'a> From<&'a [StonePayloadMeta]> for StoneWritePayload<'a> { - fn from(payload: &'a [StonePayloadMeta]) -> Self { +impl<'a> From<&'a [StonePayloadMetaRecord]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadMetaRecord]) -> Self { Self::Meta(payload) } } -impl<'a> From<&'a [StonePayloadAttribute]> for StoneWritePayload<'a> { - fn from(payload: &'a [StonePayloadAttribute]) -> Self { +impl<'a> From<&'a [StonePayloadAttributeRecord]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadAttributeRecord]) -> Self { Self::Attributes(payload) } } -impl<'a> From<&'a [StonePayloadLayout]> for StoneWritePayload<'a> { - fn from(payload: &'a [StonePayloadLayout]) -> Self { +impl<'a> From<&'a [StonePayloadLayoutRecord]> for StoneWritePayload<'a> { + fn from(payload: &'a [StonePayloadLayoutRecord]) -> Self { Self::Layout(payload) } } diff --git a/moss/src/cli/extract.rs b/moss/src/cli/extract.rs index 7213c8d9..c7c86bd2 100644 --- a/moss/src/cli/extract.rs +++ b/moss/src/cli/extract.rs @@ -11,7 +11,7 @@ use std::{ use clap::{arg, ArgMatches, Command}; use fs_err::{self as fs, File}; use moss::package::{self, MissingMetaFieldError}; -use stone::{StoneDecodedPayload, StonePayloadLayoutEntry, StoneReadError}; +use stone::{StoneDecodedPayload, StonePayloadLayoutFile, StoneReadError}; use thiserror::{self, Error}; use tui::{ProgressBar, ProgressStyle}; @@ -97,8 +97,8 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { if let Some(layouts) = layouts { for layout in &layouts.body { - match &layout.entry { - StonePayloadLayoutEntry::Regular(id, target) => { + match &layout.file { + StonePayloadLayoutFile::Regular(id, target) => { let store_path = content_store.join(format!("{id:02x}")); let target_disk = extraction_root.join("usr").join(target); @@ -110,7 +110,7 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { // link from CA store fs::hard_link(store_path, target_disk)?; } - StonePayloadLayoutEntry::Symlink(source, target) => { + StonePayloadLayoutFile::Symlink(source, target) => { let target_disk = extraction_root.join("usr").join(target); let directory_target = target_disk.parent().unwrap(); @@ -120,7 +120,7 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { // join the link path to the directory target for relative joinery symlink(source, target_disk)?; } - StonePayloadLayoutEntry::Directory(target) => { + StonePayloadLayoutFile::Directory(target) => { let target_disk = extraction_root.join("usr").join(target); // TODO: Fix perms! fs::create_dir_all(target_disk)?; diff --git a/moss/src/cli/info.rs b/moss/src/cli/info.rs index 70cc2252..68c70ec6 100644 --- a/moss/src/cli/info.rs +++ b/moss/src/cli/info.rs @@ -10,7 +10,7 @@ use moss::{ package::Flags, Installation, Package, Provider, }; -use stone::StonePayloadLayoutEntry; +use stone::StonePayloadLayoutFile; use thiserror::Error; use tui::{Styled, TermSize}; use vfs::tree::BlitFile; @@ -178,9 +178,9 @@ fn print_files(vfs: vfs::Tree) { } let path = file.path(); - let meta = match &file.layout.entry { - StonePayloadLayoutEntry::Regular(hash, _) => Some(format!(" ({hash:2x})")), - StonePayloadLayoutEntry::Symlink(source, _) => Some(format!(" -> {source}")), + let meta = match &file.layout.file { + StonePayloadLayoutFile::Regular(hash, _) => Some(format!(" ({hash:2x})")), + StonePayloadLayoutFile::Symlink(source, _) => Some(format!(" -> {source}")), _ => None, }; diff --git a/moss/src/cli/inspect.rs b/moss/src/cli/inspect.rs index a206ca86..d9690176 100644 --- a/moss/src/cli/inspect.rs +++ b/moss/src/cli/inspect.rs @@ -5,7 +5,9 @@ use clap::{arg, ArgMatches, Command}; use fs_err::File; use std::path::PathBuf; -use stone::{StoneDecodedPayload, StonePayloadLayoutEntry, StonePayloadMetaKind, StonePayloadMetaTag, StoneReadError}; +use stone::{ + StoneDecodedPayload, StonePayloadLayoutFile, StonePayloadMetaPrimitive, StonePayloadMetaTag, StoneReadError, +}; use thiserror::Error; const COLUMN_WIDTH: usize = 20; @@ -55,23 +57,27 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { for record in meta.body { let name = format!("{:?}", record.tag); - match &record.kind { - StonePayloadMetaKind::Provider(k, p) if record.tag == StonePayloadMetaTag::Provides => { - provs.push(format!("{k}({p})")); + match &record.primitive { + StonePayloadMetaPrimitive::Provider(k, p) + if record.tag == StonePayloadMetaTag::Provides => + { + provs.push(format!("{k}({p})")) } - StonePayloadMetaKind::Provider(k, p) if record.tag == StonePayloadMetaTag::Conflicts => { - cnfls.push(format!("{k}({p})")); + StonePayloadMetaPrimitive::Provider(k, p) + if record.tag == StonePayloadMetaTag::Conflicts => + { + cnfls.push(format!("{k}({p})")) } - StonePayloadMetaKind::Dependency(k, d) => { - deps.push(format!("{k}({d})")); + StonePayloadMetaPrimitive::Dependency(k, d) => { + deps.push(format!("{}({})", k, d)); } - StonePayloadMetaKind::String(s) => { + StonePayloadMetaPrimitive::String(s) => { println!("{name:COLUMN_WIDTH$} : {s}"); } - StonePayloadMetaKind::Int64(i) => { + StonePayloadMetaPrimitive::Int64(i) => { println!("{name:COLUMN_WIDTH$} : {i}"); } - StonePayloadMetaKind::Uint64(i) => { + StonePayloadMetaPrimitive::Uint64(i) => { println!("{name:COLUMN_WIDTH$} : {i}"); } _ => { @@ -105,14 +111,14 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { if !layouts.is_empty() { println!("\n{:COLUMN_WIDTH$} :", "Layout entries"); for layout in layouts { - match layout.entry { - StonePayloadLayoutEntry::Regular(hash, target) => { + match layout.file { + StonePayloadLayoutFile::Regular(hash, target) => { println!(" - /usr/{target} - [Regular] {hash:032x}"); } - StonePayloadLayoutEntry::Directory(target) => { + StonePayloadLayoutFile::Directory(target) => { println!(" - /usr/{target} [Directory]"); } - StonePayloadLayoutEntry::Symlink(source, target) => { + StonePayloadLayoutFile::Symlink(source, target) => { println!(" - /usr/{target} -> {source} [Symlink]"); } _ => unreachable!(), diff --git a/moss/src/client/boot.rs b/moss/src/client/boot.rs index d3c24e32..f4411efe 100644 --- a/moss/src/client/boot.rs +++ b/moss/src/client/boot.rs @@ -18,7 +18,7 @@ use blsforme::{ use fnmatch::Pattern; use fs_err as fs; use itertools::Itertools; -use stone::{StonePayloadLayout, StonePayloadLayoutEntry}; +use stone::{StonePayloadLayoutFile, StonePayloadLayoutRecord}; use thiserror::{self, Error}; use crate::{db, package::Id, Installation, State}; @@ -51,7 +51,7 @@ pub enum Error { #[derive(Debug)] struct KernelCandidate { path: PathBuf, - _layout: StonePayloadLayout, + _layout: StonePayloadLayoutRecord, } impl AsRef for KernelCandidate { @@ -63,12 +63,15 @@ impl AsRef for KernelCandidate { /// From a given set of input paths, produce a set of match pairs /// NOTE: This only works for a *new* blit and doesn't retroactively /// sync old kernels! -fn kernel_files_from_state<'a>(layouts: &'a [(Id, StonePayloadLayout)], pattern: &'a Pattern) -> Vec { +fn kernel_files_from_state<'a>( + layouts: &'a [(Id, StonePayloadLayoutRecord)], + pattern: &'a Pattern, +) -> Vec { let mut kernel_entries = vec![]; for (_, path) in layouts.iter() { - match &path.entry { - StonePayloadLayoutEntry::Regular(_, target) => { + match &path.file { + StonePayloadLayoutFile::Regular(_, target) => { if pattern.match_path(target).is_some() { kernel_entries.push(KernelCandidate { path: PathBuf::from("usr").join(target), @@ -76,7 +79,7 @@ fn kernel_files_from_state<'a>(layouts: &'a [(Id, StonePayloadLayout)], pattern: }); } } - StonePayloadLayoutEntry::Symlink(_, target) => { + StonePayloadLayoutFile::Symlink(_, target) => { if pattern.match_path(target).is_some() { kernel_entries.push(KernelCandidate { path: PathBuf::from("usr").join(target), @@ -94,13 +97,13 @@ fn kernel_files_from_state<'a>(layouts: &'a [(Id, StonePayloadLayout)], pattern: /// Find bootloader assets in the new state fn boot_files_from_new_state<'a>( install: &Installation, - layouts: &'a [(Id, StonePayloadLayout)], + layouts: &'a [(Id, StonePayloadLayoutRecord)], pattern: &'a Pattern, ) -> Vec { let mut rets = vec![]; for (_, path) in layouts.iter() { - if let StonePayloadLayoutEntry::Regular(_, target) = &path.entry { + if let StonePayloadLayoutFile::Regular(_, target) = &path.file { if pattern.match_path(target).is_some() { rets.push(install.root.join("usr").join(target)); } @@ -111,7 +114,7 @@ fn boot_files_from_new_state<'a>( } /// Grab all layouts for the provided state, mapped to package id -fn layouts_for_state(client: &Client, state: &State) -> Result, db::Error> { +fn layouts_for_state(client: &Client, state: &State) -> Result, db::Error> { client.layout_db.query(state.selections.iter().map(|s| &s.package)) } diff --git a/moss/src/client/cache.rs b/moss/src/client/cache.rs index 45fd095e..d831cab9 100644 --- a/moss/src/client/cache.rs +++ b/moss/src/client/cache.rs @@ -12,7 +12,7 @@ use std::{ }; use futures_util::StreamExt; -use stone::{StoneDecodedPayload, StonePayloadIndex, StoneReadError}; +use stone::{StoneDecodedPayload, StonePayloadIndexRecord, StoneReadError}; use thiserror::Error; use tokio::io::AsyncWriteExt; use url::Url; @@ -257,7 +257,7 @@ impl Download { } /// Returns true if all assets already exist in the installation -fn check_assets_exist(indices: &[&StonePayloadIndex], installation: &Installation) -> bool { +fn check_assets_exist(indices: &[&StonePayloadIndexRecord], installation: &Installation) -> bool { indices.iter().all(|index| { let path = asset_path(installation, &format!("{:02x}", index.digest)); path.exists() diff --git a/moss/src/client/mod.rs b/moss/src/client/mod.rs index 87ef2bf6..01c8deb3 100644 --- a/moss/src/client/mod.rs +++ b/moss/src/client/mod.rs @@ -26,7 +26,7 @@ use nix::{ unistd::{close, linkat, mkdir, symlinkat}, }; use postblit::TriggerScope; -use stone::{StoneDecodedPayload, StonePayloadLayout, StonePayloadLayoutEntry}; +use stone::{StoneDecodedPayload, StonePayloadLayoutFile, StonePayloadLayoutRecord}; use thiserror::Error; use tui::{MultiProgress, ProgressBar, ProgressStyle, Styled}; use vfs::tree::{builder::TreeBuilder, BlitFile, Element}; @@ -696,8 +696,8 @@ impl Client { /// * `subpath` - the base name of the new inode /// * `item` - New inode being recorded fn blit_element_item(&self, parent: RawFd, cache: RawFd, subpath: &str, item: &PendingFile) -> Result<(), Error> { - match &item.layout.entry { - StonePayloadLayoutEntry::Regular(id, _) => { + match &item.layout.file { + StonePayloadLayoutFile::Regular(id, _) => { let hash = format!("{id:02x}"); let directory = if hash.len() >= 10 { PathBuf::from(&hash[..2]).join(&hash[2..4]).join(&hash[4..6]) @@ -740,18 +740,18 @@ impl Client { } } } - StonePayloadLayoutEntry::Symlink(source, _) => { + StonePayloadLayoutFile::Symlink(source, _) => { symlinkat(source.as_str(), Some(parent), subpath)?; } - StonePayloadLayoutEntry::Directory(_) => { + StonePayloadLayoutFile::Directory(_) => { mkdirat(parent, subpath, Mode::from_bits_truncate(item.layout.mode))?; } // unimplemented - StonePayloadLayoutEntry::CharacterDevice(_) => todo!(), - StonePayloadLayoutEntry::BlockDevice(_) => todo!(), - StonePayloadLayoutEntry::Fifo(_) => todo!(), - StonePayloadLayoutEntry::Socket(_) => todo!(), + StonePayloadLayoutFile::CharacterDevice(_) => todo!(), + StonePayloadLayoutFile::BlockDevice(_) => todo!(), + StonePayloadLayoutFile::Fifo(_) => todo!(), + StonePayloadLayoutFile::Socket(_) => todo!(), }; Ok(()) @@ -840,15 +840,15 @@ pub struct PendingFile { pub id: package::Id, /// Corresponding layout entry, describing the inode - pub layout: StonePayloadLayout, + pub layout: StonePayloadLayoutRecord, } impl BlitFile for PendingFile { /// Match internal kind to minimalist vfs kind fn kind(&self) -> vfs::tree::Kind { - match &self.layout.entry { - StonePayloadLayoutEntry::Symlink(source, _) => vfs::tree::Kind::Symlink(source.clone()), - StonePayloadLayoutEntry::Directory(_) => vfs::tree::Kind::Directory, + match &self.layout.file { + StonePayloadLayoutFile::Symlink(source, _) => vfs::tree::Kind::Symlink(source.clone()), + StonePayloadLayoutFile::Directory(_) => vfs::tree::Kind::Directory, _ => vfs::tree::Kind::Regular, } } @@ -860,14 +860,14 @@ impl BlitFile for PendingFile { /// Resolve the target path, including the missing `/usr` prefix fn path(&self) -> String { - let result = match &self.layout.entry { - StonePayloadLayoutEntry::Regular(_, target) => target.clone(), - StonePayloadLayoutEntry::Symlink(_, target) => target.clone(), - StonePayloadLayoutEntry::Directory(target) => target.clone(), - StonePayloadLayoutEntry::CharacterDevice(target) => target.clone(), - StonePayloadLayoutEntry::BlockDevice(target) => target.clone(), - StonePayloadLayoutEntry::Fifo(target) => target.clone(), - StonePayloadLayoutEntry::Socket(target) => target.clone(), + let result = match &self.layout.file { + StonePayloadLayoutFile::Regular(_, target) => target.clone(), + StonePayloadLayoutFile::Symlink(_, target) => target.clone(), + StonePayloadLayoutFile::Directory(target) => target.clone(), + StonePayloadLayoutFile::CharacterDevice(target) => target.clone(), + StonePayloadLayoutFile::BlockDevice(target) => target.clone(), + StonePayloadLayoutFile::Fifo(target) => target.clone(), + StonePayloadLayoutFile::Socket(target) => target.clone(), }; vfs::path::join("/usr", &result) @@ -876,14 +876,14 @@ impl BlitFile for PendingFile { /// Clone the node to a reparented path, for symlink resolution fn cloned_to(&self, path: String) -> Self { let mut new = self.clone(); - new.layout.entry = match &self.layout.entry { - StonePayloadLayoutEntry::Regular(source, _) => StonePayloadLayoutEntry::Regular(*source, path), - StonePayloadLayoutEntry::Symlink(source, _) => StonePayloadLayoutEntry::Symlink(source.clone(), path), - StonePayloadLayoutEntry::Directory(_) => StonePayloadLayoutEntry::Directory(path), - StonePayloadLayoutEntry::CharacterDevice(_) => StonePayloadLayoutEntry::CharacterDevice(path), - StonePayloadLayoutEntry::BlockDevice(_) => StonePayloadLayoutEntry::BlockDevice(path), - StonePayloadLayoutEntry::Fifo(_) => StonePayloadLayoutEntry::Fifo(path), - StonePayloadLayoutEntry::Socket(_) => StonePayloadLayoutEntry::Socket(path), + new.layout.file = match &self.layout.file { + StonePayloadLayoutFile::Regular(source, _) => StonePayloadLayoutFile::Regular(*source, path), + StonePayloadLayoutFile::Symlink(source, _) => StonePayloadLayoutFile::Symlink(source.clone(), path), + StonePayloadLayoutFile::Directory(_) => StonePayloadLayoutFile::Directory(path), + StonePayloadLayoutFile::CharacterDevice(_) => StonePayloadLayoutFile::CharacterDevice(path), + StonePayloadLayoutFile::BlockDevice(_) => StonePayloadLayoutFile::BlockDevice(path), + StonePayloadLayoutFile::Fifo(_) => StonePayloadLayoutFile::Fifo(path), + StonePayloadLayoutFile::Socket(_) => StonePayloadLayoutFile::Socket(path), }; new } @@ -893,12 +893,12 @@ impl From for PendingFile { fn from(value: String) -> Self { PendingFile { id: Default::default(), - layout: StonePayloadLayout { + layout: StonePayloadLayoutRecord { uid: 0, gid: 0, mode: 0o755, tag: 0, - entry: StonePayloadLayoutEntry::Directory(value), + file: StonePayloadLayoutFile::Directory(value), }, } } diff --git a/moss/src/client/verify.rs b/moss/src/client/verify.rs index f2f8c6bc..f404716f 100644 --- a/moss/src/client/verify.rs +++ b/moss/src/client/verify.rs @@ -7,7 +7,7 @@ use std::{collections::BTreeSet, fmt, io, path::PathBuf}; use itertools::Itertools; use fs_err as fs; -use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutEntry}; +use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutFile}; use tui::{ dialoguer::{theme::ColorfulTheme, Confirm}, ProgressBar, ProgressStyle, Styled, @@ -29,7 +29,7 @@ pub fn verify(client: &Client, yes: bool, verbose: bool) -> Result<(), client::E let unique_assets = layouts .into_iter() .filter_map(|(package, layout)| { - if let StonePayloadLayoutEntry::Regular(hash, file) = layout.entry { + if let StonePayloadLayoutFile::Regular(hash, file) = layout.file { Some((format!("{hash:02x}"), (package, file))) } else { None diff --git a/moss/src/db/layout/mod.rs b/moss/src/db/layout/mod.rs index 455203d0..f19daceb 100644 --- a/moss/src/db/layout/mod.rs +++ b/moss/src/db/layout/mod.rs @@ -6,7 +6,7 @@ use diesel::prelude::*; use diesel::{Connection as _, SqliteConnection}; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use std::collections::BTreeSet; -use stone::{StonePayloadLayout, StonePayloadLayoutEntry}; +use stone::{StonePayloadLayoutFile, StonePayloadLayoutRecord}; use crate::package; @@ -37,7 +37,7 @@ impl Database { pub fn query<'a>( &self, packages: impl IntoIterator, - ) -> Result, Error> { + ) -> Result, Error> { self.conn.exec(|conn| { let packages = packages.into_iter().map(AsRef::::as_ref).collect::>(); @@ -58,7 +58,7 @@ impl Database { }) } - pub fn all(&self) -> Result, Error> { + pub fn all(&self) -> Result, Error> { self.conn.exec(|conn| { model::layout::table .select(model::Layout::as_select()) @@ -83,13 +83,13 @@ impl Database { }) } - pub fn add(&self, package: &package::Id, layout: &StonePayloadLayout) -> Result<(), Error> { + pub fn add(&self, package: &package::Id, layout: &StonePayloadLayoutRecord) -> Result<(), Error> { self.batch_add(vec![(package, layout)]) } pub fn batch_add<'a>( &self, - layouts: impl IntoIterator, + layouts: impl IntoIterator, ) -> Result<(), Error> { self.conn.exclusive_tx(|tx| { let mut ids = vec![]; @@ -99,7 +99,7 @@ impl Database { .map(|(package_id, layout)| { ids.push(package_id.as_ref()); - let (entry_type, entry_value1, entry_value2) = encode_entry(layout.entry.clone()); + let (entry_type, entry_value1, entry_value2) = encode_entry(layout.file.clone()); model::NewLayout { package_id: package_id.to_string(), @@ -148,17 +148,17 @@ fn batch_remove_impl(packages: &[&str], tx: &mut SqliteConnection) -> Result<(), Ok(()) } -fn map_layout(result: QueryResult) -> Result<(package::Id, StonePayloadLayout), Error> { +fn map_layout(result: QueryResult) -> Result<(package::Id, StonePayloadLayoutRecord), Error> { let row = result?; let entry = decode_entry(row.entry_type, row.entry_value1, row.entry_value2).ok_or(Error::LayoutEntryDecode)?; - let layout = StonePayloadLayout { + let layout = StonePayloadLayoutRecord { uid: row.uid as u32, gid: row.gid as u32, mode: row.mode as u32, tag: row.tag as u32, - entry, + file: entry, }; Ok((row.package_id, layout)) @@ -168,33 +168,33 @@ fn decode_entry( entry_type: String, entry_value1: Option, entry_value2: Option, -) -> Option { +) -> Option { match entry_type.as_str() { "regular" => { let hash = entry_value1?.parse::().ok()?; let name = entry_value2?; - Some(StonePayloadLayoutEntry::Regular(hash, name)) + Some(StonePayloadLayoutFile::Regular(hash, name)) } - "symlink" => Some(StonePayloadLayoutEntry::Symlink(entry_value1?, entry_value2?)), - "directory" => Some(StonePayloadLayoutEntry::Directory(entry_value1?)), - "character-device" => Some(StonePayloadLayoutEntry::CharacterDevice(entry_value1?)), - "block-device" => Some(StonePayloadLayoutEntry::BlockDevice(entry_value1?)), - "fifo" => Some(StonePayloadLayoutEntry::Fifo(entry_value1?)), - "socket" => Some(StonePayloadLayoutEntry::Socket(entry_value1?)), + "symlink" => Some(StonePayloadLayoutFile::Symlink(entry_value1?, entry_value2?)), + "directory" => Some(StonePayloadLayoutFile::Directory(entry_value1?)), + "character-device" => Some(StonePayloadLayoutFile::CharacterDevice(entry_value1?)), + "block-device" => Some(StonePayloadLayoutFile::BlockDevice(entry_value1?)), + "fifo" => Some(StonePayloadLayoutFile::Fifo(entry_value1?)), + "socket" => Some(StonePayloadLayoutFile::Socket(entry_value1?)), _ => None, } } -fn encode_entry(entry: StonePayloadLayoutEntry) -> (&'static str, Option, Option) { +fn encode_entry(entry: StonePayloadLayoutFile) -> (&'static str, Option, Option) { match entry { - StonePayloadLayoutEntry::Regular(hash, name) => ("regular", Some(hash.to_string()), Some(name)), - StonePayloadLayoutEntry::Symlink(a, b) => ("symlink", Some(a), Some(b)), - StonePayloadLayoutEntry::Directory(name) => ("directory", Some(name), None), - StonePayloadLayoutEntry::CharacterDevice(name) => ("character-device", Some(name), None), - StonePayloadLayoutEntry::BlockDevice(name) => ("block-device", Some(name), None), - StonePayloadLayoutEntry::Fifo(name) => ("fifo", Some(name), None), - StonePayloadLayoutEntry::Socket(name) => ("socket", Some(name), None), + StonePayloadLayoutFile::Regular(hash, name) => ("regular", Some(hash.to_string()), Some(name)), + StonePayloadLayoutFile::Symlink(a, b) => ("symlink", Some(a), Some(b)), + StonePayloadLayoutFile::Directory(name) => ("directory", Some(name), None), + StonePayloadLayoutFile::CharacterDevice(name) => ("character-device", Some(name), None), + StonePayloadLayoutFile::BlockDevice(name) => ("block-device", Some(name), None), + StonePayloadLayoutFile::Fifo(name) => ("fifo", Some(name), None), + StonePayloadLayoutFile::Socket(name) => ("socket", Some(name), None), } } diff --git a/moss/src/package/meta.rs b/moss/src/package/meta.rs index 6271677f..e1aa5966 100644 --- a/moss/src/package/meta.rs +++ b/moss/src/package/meta.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use derive_more::{AsRef, Display, From, Into}; -use stone::{StonePayloadMeta, StonePayloadMetaKind, StonePayloadMetaTag}; +use stone::{StonePayloadMetaPrimitive, StonePayloadMetaRecord, StonePayloadMetaTag}; use thiserror::Error; use crate::{dependency, Dependency, Provider}; @@ -62,7 +62,7 @@ pub struct Meta { } impl Meta { - pub fn from_stone_payload(payload: &[StonePayloadMeta]) -> Result { + pub fn from_stone_payload(payload: &[StonePayloadMetaRecord]) -> Result { let name = find_meta_string(payload, StonePayloadMetaTag::Name)?; let version_identifier = find_meta_string(payload, StonePayloadMetaTag::Version)?; let source_release = find_meta_u64(payload, StonePayloadMetaTag::Release)?; @@ -112,64 +112,71 @@ impl Meta { }) } - pub fn to_stone_payload(self) -> Vec { + pub fn to_stone_payload(self) -> Vec { vec![ ( StonePayloadMetaTag::Name, - StonePayloadMetaKind::String(self.name.to_string()), + StonePayloadMetaPrimitive::String(self.name.to_string()), ), ( StonePayloadMetaTag::Version, - StonePayloadMetaKind::String(self.version_identifier), + StonePayloadMetaPrimitive::String(self.version_identifier), ), ( StonePayloadMetaTag::Release, - StonePayloadMetaKind::Uint64(self.source_release), + StonePayloadMetaPrimitive::Uint64(self.source_release), ), ( StonePayloadMetaTag::BuildRelease, - StonePayloadMetaKind::Uint64(self.build_release), + StonePayloadMetaPrimitive::Uint64(self.build_release), ), ( StonePayloadMetaTag::Architecture, - StonePayloadMetaKind::String(self.architecture), + StonePayloadMetaPrimitive::String(self.architecture), + ), + ( + StonePayloadMetaTag::Summary, + StonePayloadMetaPrimitive::String(self.summary), ), - (StonePayloadMetaTag::Summary, StonePayloadMetaKind::String(self.summary)), ( StonePayloadMetaTag::Description, - StonePayloadMetaKind::String(self.description), + StonePayloadMetaPrimitive::String(self.description), ), ( StonePayloadMetaTag::SourceID, - StonePayloadMetaKind::String(self.source_id), + StonePayloadMetaPrimitive::String(self.source_id), ), ( StonePayloadMetaTag::Homepage, - StonePayloadMetaKind::String(self.homepage), + StonePayloadMetaPrimitive::String(self.homepage), ), ] .into_iter() .chain( self.uri - .map(|uri| (StonePayloadMetaTag::PackageURI, StonePayloadMetaKind::String(uri))), - ) - .chain( - self.hash - .map(|hash| (StonePayloadMetaTag::PackageHash, StonePayloadMetaKind::String(hash))), - ) - .chain( - self.download_size - .map(|size| (StonePayloadMetaTag::PackageSize, StonePayloadMetaKind::Uint64(size))), + .map(|uri| (StonePayloadMetaTag::PackageURI, StonePayloadMetaPrimitive::String(uri))), ) + .chain(self.hash.map(|hash| { + ( + StonePayloadMetaTag::PackageHash, + StonePayloadMetaPrimitive::String(hash), + ) + })) + .chain(self.download_size.map(|size| { + ( + StonePayloadMetaTag::PackageSize, + StonePayloadMetaPrimitive::Uint64(size), + ) + })) .chain( self.licenses .into_iter() - .map(|license| (StonePayloadMetaTag::License, StonePayloadMetaKind::String(license))), + .map(|license| (StonePayloadMetaTag::License, StonePayloadMetaPrimitive::String(license))), ) .chain(self.dependencies.into_iter().map(|dep| { ( StonePayloadMetaTag::Depends, - StonePayloadMetaKind::Dependency(dep.kind.into(), dep.name), + StonePayloadMetaPrimitive::Dependency(dep.kind.into(), dep.name), ) })) .chain( @@ -180,7 +187,7 @@ impl Meta { .map(|provider| { ( StonePayloadMetaTag::Provides, - StonePayloadMetaKind::Provider(provider.kind.into(), provider.name), + StonePayloadMetaPrimitive::Provider(provider.kind.into(), provider.name), ) }), ) @@ -191,11 +198,11 @@ impl Meta { .map(|conflict| { ( StonePayloadMetaTag::Conflicts, - StonePayloadMetaKind::Provider(conflict.kind.into(), conflict.name), + StonePayloadMetaPrimitive::Provider(conflict.kind.into(), conflict.name), ) }), ) - .map(|(tag, kind)| StonePayloadMeta { tag, kind }) + .map(|(tag, kind)| StonePayloadMetaRecord { tag, primitive: kind }) .collect() } @@ -208,29 +215,32 @@ impl Meta { } } -fn find_meta_string(meta: &[StonePayloadMeta], tag: StonePayloadMetaTag) -> Result { +fn find_meta_string( + meta: &[StonePayloadMetaRecord], + tag: StonePayloadMetaTag, +) -> Result { meta.iter() .find_map(|meta| meta_string(meta, tag)) .ok_or(MissingMetaFieldError(tag)) } -fn find_meta_u64(meta: &[StonePayloadMeta], tag: StonePayloadMetaTag) -> Result { +fn find_meta_u64(meta: &[StonePayloadMetaRecord], tag: StonePayloadMetaTag) -> Result { meta.iter() .find_map(|meta| meta_u64(meta, tag)) .ok_or(MissingMetaFieldError(tag)) } -fn meta_u64(meta: &StonePayloadMeta, tag: StonePayloadMetaTag) -> Option { +fn meta_u64(meta: &StonePayloadMetaRecord, tag: StonePayloadMetaTag) -> Option { if meta.tag == tag { - Some(match meta.kind { - StonePayloadMetaKind::Int8(i) => i as _, - StonePayloadMetaKind::Uint8(i) => i as _, - StonePayloadMetaKind::Int16(i) => i as _, - StonePayloadMetaKind::Uint16(i) => i as _, - StonePayloadMetaKind::Int32(i) => i as _, - StonePayloadMetaKind::Uint32(i) => i as _, - StonePayloadMetaKind::Int64(i) => i as _, - StonePayloadMetaKind::Uint64(i) => i, + Some(match meta.primitive { + StonePayloadMetaPrimitive::Int8(i) => i as _, + StonePayloadMetaPrimitive::Uint8(i) => i as _, + StonePayloadMetaPrimitive::Int16(i) => i as _, + StonePayloadMetaPrimitive::Uint16(i) => i as _, + StonePayloadMetaPrimitive::Int32(i) => i as _, + StonePayloadMetaPrimitive::Uint32(i) => i as _, + StonePayloadMetaPrimitive::Int64(i) => i as _, + StonePayloadMetaPrimitive::Uint64(i) => i, _ => return None, }) } else { @@ -238,15 +248,15 @@ fn meta_u64(meta: &StonePayloadMeta, tag: StonePayloadMetaTag) -> Option { } } -fn meta_string(meta: &StonePayloadMeta, tag: StonePayloadMetaTag) -> Option { - match (meta.tag, &meta.kind) { - (meta_tag, StonePayloadMetaKind::String(value)) if meta_tag == tag => Some(value.clone()), +fn meta_string(meta: &StonePayloadMetaRecord, tag: StonePayloadMetaTag) -> Option { + match (meta.tag, &meta.primitive) { + (meta_tag, StonePayloadMetaPrimitive::String(value)) if meta_tag == tag => Some(value.clone()), _ => None, } } -fn meta_dependency(meta: &StonePayloadMeta) -> Option { - if let StonePayloadMetaKind::Dependency(kind, name) = meta.kind.clone() { +fn meta_dependency(meta: &StonePayloadMetaRecord) -> Option { + if let StonePayloadMetaPrimitive::Dependency(kind, name) = meta.primitive.clone() { Some(Dependency { kind: dependency::Kind::from(kind), name, @@ -256,9 +266,9 @@ fn meta_dependency(meta: &StonePayloadMeta) -> Option { } } -fn meta_provider(meta: &StonePayloadMeta) -> Option { - match (meta.tag, meta.kind.clone()) { - (StonePayloadMetaTag::Provides, StonePayloadMetaKind::Provider(kind, name)) => Some(Provider { +fn meta_provider(meta: &StonePayloadMetaRecord) -> Option { + match (meta.tag, meta.primitive.clone()) { + (StonePayloadMetaTag::Provides, StonePayloadMetaPrimitive::Provider(kind, name)) => Some(Provider { kind: dependency::Kind::from(kind), name: name.clone(), }), @@ -266,9 +276,9 @@ fn meta_provider(meta: &StonePayloadMeta) -> Option { } } -fn meta_conflict(meta: &StonePayloadMeta) -> Option { - match (meta.tag, meta.kind.clone()) { - (StonePayloadMetaTag::Conflicts, StonePayloadMetaKind::Provider(kind, name)) => Some(Provider { +fn meta_conflict(meta: &StonePayloadMetaRecord) -> Option { + match (meta.tag, meta.primitive.clone()) { + (StonePayloadMetaTag::Conflicts, StonePayloadMetaPrimitive::Provider(kind, name)) => Some(Provider { kind: dependency::Kind::from(kind), name: name.clone(), }), From ee8aa0ec265c3f385acab8027ef37b1816b8d823 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Wed, 30 Oct 2024 14:57:35 -0700 Subject: [PATCH 06/12] Rename to libstone in root --- Cargo.lock | 574 +++++++++++------- Cargo.toml | 2 +- justfile | 16 +- {crates/stone-ffi => libstone}/Cargo.toml | 15 +- libstone/build.rs | 9 + {crates/stone-ffi => libstone}/cbindgen.toml | 0 .../test.c => libstone/examples/read.c | 41 +- {crates/stone-ffi => libstone}/src/lib.rs | 23 +- {crates/stone-ffi => libstone}/src/payload.rs | 0 .../src/payload/attribute.rs | 0 .../src/payload/index.rs | 0 .../src/payload/layout.rs | 0 .../src/payload/meta.rs | 0 libstone/src/stone.h | 328 ++++++++++ moss/src/cli/inspect.rs | 6 +- 15 files changed, 773 insertions(+), 241 deletions(-) rename {crates/stone-ffi => libstone}/Cargo.toml (56%) create mode 100644 libstone/build.rs rename {crates/stone-ffi => libstone}/cbindgen.toml (100%) rename crates/stone-ffi/test.c => libstone/examples/read.c (93%) rename {crates/stone-ffi => libstone}/src/lib.rs (92%) rename {crates/stone-ffi => libstone}/src/payload.rs (100%) rename {crates/stone-ffi => libstone}/src/payload/attribute.rs (100%) rename {crates/stone-ffi => libstone}/src/payload/index.rs (100%) rename {crates/stone-ffi => libstone}/src/payload/layout.rs (100%) rename {crates/stone-ffi => libstone}/src/payload/meta.rs (100%) create mode 100644 libstone/src/stone.h diff --git a/Cargo.lock b/Cargo.lock index 29d4bcca..aefee9d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,9 +125,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-broadcast" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" dependencies = [ "event-listener", "event-listener-strategy", @@ -244,7 +244,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -273,13 +273,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "1b1244b10dcd56c92219da4e14caa97e312079e185f04ba3eea25061561dc0a0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -288,6 +288,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -315,6 +326,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.6.0" @@ -370,7 +387,7 @@ dependencies = [ "serde", "serde_json", "superblock", - "thiserror 2.0.3", + "thiserror 2.0.9", "topology", ] @@ -379,7 +396,7 @@ name = "boulder" version = "0.24.7" dependencies = [ "chrono", - "clap", + "clap 4.5.23", "clap_complete", "clap_mangen", "config", @@ -404,7 +421,7 @@ dependencies = [ "stone", "stone_recipe", "strum", - "thiserror 2.0.3", + "thiserror 2.0.9", "thread-priority", "tokio", "tui", @@ -457,11 +474,30 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" +dependencies = [ + "clap 3.2.25", + "heck 0.4.1", + "indexmap 1.9.3", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 1.0.109", + "tempfile", + "toml 0.5.11", +] + [[package]] name = "cc" -version = "1.2.2" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "jobserver", "libc", @@ -492,9 +528,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -533,9 +569,24 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "strsim 0.10.0", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -543,23 +594,23 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", - "clap_lex", - "strsim", + "clap_lex 0.7.4", + "strsim 0.11.1", ] [[package]] name = "clap_complete" -version = "4.5.38" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9647a559c112175f17cf724dc72d3645680a883c58481332779192b0d8e7a01" +checksum = "ac2e663e3e3bed2d32d065a8404024dad306e699a04263ec59919529f803aee9" dependencies = [ - "clap", + "clap 4.5.23", ] [[package]] @@ -568,17 +619,26 @@ version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "clap_mangen" @@ -586,7 +646,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbae9cbfdc5d4fa8711c09bd7b83f644cb48281ac35bf97af3e47b0675864bdf" dependencies = [ - "clap", + "clap 4.5.23", "roff", ] @@ -613,20 +673,20 @@ dependencies = [ "fs-err", "serde", "serde_yaml", - "thiserror 2.0.3", + "thiserror 2.0.9", ] [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width 0.1.14", - "windows-sys 0.52.0", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", ] [[package]] @@ -642,7 +702,7 @@ dependencies = [ "fs-err", "nix 0.27.1", "strum", - "thiserror 2.0.3", + "thiserror 2.0.9", ] [[package]] @@ -693,7 +753,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap", + "clap 4.5.23", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -722,9 +782,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -741,9 +801,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" @@ -751,7 +811,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.6.0", "crossterm_winapi", "mio", "parking_lot", @@ -813,8 +873,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.95", ] [[package]] @@ -825,7 +885,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -860,7 +920,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", "unicode-xid", ] @@ -879,9 +939,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf9649c05e0a9dbd6d0b0b8301db5182b972d0fd02f0a7c6736cf632d7c0fd5" +checksum = "ccf1bedf64cdb9643204a36dd15b19a6ce8e7aa7f7b105868e9f1fad5ffa7d12" dependencies = [ "diesel_derives", "libsqlite3-sys", @@ -898,7 +958,7 @@ dependencies = [ "dsl_auto_type", "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -918,7 +978,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ - "syn", + "syn 2.0.95", ] [[package]] @@ -960,7 +1020,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -971,10 +1031,10 @@ checksum = "c5d9abe6314103864cc2d8901b7ae224e0ab1a103a0a416661b4097b0779b607" dependencies = [ "darling", "either", - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -991,9 +1051,9 @@ checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" @@ -1028,7 +1088,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -1070,9 +1130,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fixedbitset" @@ -1096,7 +1156,7 @@ version = "0.24.7" dependencies = [ "regex", "serde", - "thiserror 2.0.3", + "thiserror 2.0.9", ] [[package]] @@ -1166,7 +1226,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -1229,9 +1289,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "gpt" @@ -1239,7 +1299,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8283e7331b8c93b9756e0cfdbcfb90312852f953c6faf9bf741e684cc3b6ad69" dependencies = [ - "bitflags", + "bitflags 2.6.0", "crc", "log", "uuid", @@ -1257,7 +1317,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -1274,18 +1334,39 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.4.0" @@ -1300,9 +1381,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1340,9 +1421,9 @@ checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -1360,9 +1441,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1533,7 +1614,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -1565,12 +1646,22 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1593,7 +1684,7 @@ dependencies = [ "proc-macro2", "quote", "strum", - "syn", + "syn 2.0.95", "thiserror 1.0.69", ] @@ -1606,7 +1697,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width", "web-time", ] @@ -1622,7 +1713,7 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] @@ -1668,25 +1759,19 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "libc" -version = "0.2.167" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libredox" @@ -1694,7 +1779,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags", + "bitflags 2.6.0", "libc", ] @@ -1709,6 +1794,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libstone" +version = "0.24.7" +dependencies = [ + "cbindgen", + "libc", + "stone", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1779,7 +1873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" dependencies = [ "serde", - "toml", + "toml 0.8.19", ] [[package]] @@ -1807,9 +1901,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -1833,7 +1927,7 @@ dependencies = [ "blsforme", "bytes", "chrono", - "clap", + "clap 4.5.23", "clap_complete", "clap_mangen", "config", @@ -1857,7 +1951,7 @@ dependencies = [ "sha2", "stone", "strum", - "thiserror 2.0.3", + "thiserror 2.0.9", "tokio", "tokio-util", "triggers", @@ -1874,7 +1968,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cfg-if", "libc", ] @@ -1885,7 +1979,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -1925,9 +2019,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -1960,6 +2054,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "parking" version = "2.2.1" @@ -2002,14 +2102,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.7.0", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2070,7 +2170,7 @@ checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.4.0", "pin-project-lite", "rustix", "tracing", @@ -2129,7 +2229,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.9", "tokio", "tracing", ] @@ -2148,7 +2248,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.9", "tinyvec", "tracing", "web-time", @@ -2156,9 +2256,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -2170,9 +2270,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2235,11 +2335,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags", + "bitflags 2.6.0", ] [[package]] @@ -2284,9 +2384,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "async-compression", "base64", @@ -2319,6 +2419,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", + "tower", "tower-service", "url", "wasm-bindgen", @@ -2364,22 +2465,22 @@ checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ - "bitflags", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring", @@ -2400,9 +2501,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -2420,9 +2521,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -2447,29 +2548,29 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -2485,7 +2586,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -2515,7 +2616,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.7.0", "itoa", "ryu", "serde", @@ -2527,7 +2628,7 @@ name = "serpent_buildinfo" version = "0.24.7" dependencies = [ "chrono", - "thiserror 2.0.3", + "thiserror 2.0.9", ] [[package]] @@ -2632,19 +2733,11 @@ version = "0.24.7" dependencies = [ "criterion", "strum", - "thiserror 2.0.3", + "thiserror 2.0.9", "xxhash-rust", "zstd", ] -[[package]] -name = "stone-ffi" -version = "0.24.7" -dependencies = [ - "libc", - "stone", -] - [[package]] name = "stone_recipe" version = "0.24.7" @@ -2653,10 +2746,16 @@ dependencies = [ "serde", "serde_yaml", "strum", - "thiserror 2.0.3", + "thiserror 2.0.9", "url", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -2678,11 +2777,11 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.95", ] [[package]] @@ -2697,15 +2796,26 @@ version = "0.1.0" source = "git+https://github.com/serpent-os/blsforme.git?rev=3cb315d6e9b4f2168927bded8b326b55c92f0e84#3cb315d6e9b4f2168927bded8b326b55c92f0e84" dependencies = [ "log", - "thiserror 2.0.3", + "thiserror 2.0.9", "uuid", ] [[package]] name = "syn" -version = "2.0.90" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -2729,22 +2839,38 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + [[package]] name = "thiserror" version = "1.0.69" @@ -2756,11 +2882,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.9", ] [[package]] @@ -2771,18 +2897,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -2791,7 +2917,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe075d7053dae61ac5413a34ea7d4913b6e6207844fd726bdd858b37ff72bf5" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cfg-if", "libc", "log", @@ -2801,9 +2927,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -2822,9 +2948,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -2852,9 +2978,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -2867,9 +2993,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -2891,25 +3017,24 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -2918,6 +3043,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.8.19" @@ -2945,7 +3079,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -2961,9 +3095,30 @@ dependencies = [ "log", "nix 0.29.0", "superblock", - "thiserror 2.0.3", + "thiserror 2.0.9", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2989,7 +3144,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -3009,7 +3164,7 @@ dependencies = [ "fnmatch", "serde", "serde_yaml", - "thiserror 2.0.3", + "thiserror 2.0.9", ] [[package]] @@ -3050,12 +3205,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.0" @@ -3136,7 +3285,7 @@ name = "vfs" version = "0.1.0" dependencies = [ "indextree", - "thiserror 2.0.3", + "thiserror 2.0.9", ] [[package]] @@ -3166,9 +3315,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -3177,24 +3326,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.95", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -3205,9 +3353,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3215,22 +3363,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" @@ -3247,9 +3395,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -3494,9 +3642,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" dependencies = [ "memchr", ] @@ -3525,9 +3673,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.12" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yaml" @@ -3553,15 +3701,15 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", "synstructure", ] [[package]] name = "zbus" -version = "5.1.1" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1162094dc63b1629fcc44150bcceeaa80798cd28bcbe7fa987b65a034c258608" +checksum = "fb67eadba43784b6fb14857eba0d8fc518686d3ee537066eb6086dc318e2c8a1" dependencies = [ "async-broadcast", "async-executor", @@ -3595,14 +3743,14 @@ dependencies = [ [[package]] name = "zbus_macros" -version = "5.1.1" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd2dcdce3e2727f7d74b7e33b5a89539b3cc31049562137faf7ae4eb86cd16d" +checksum = "2c9d49ebc960ceb660f2abe40a5904da975de6986f2af0d7884b39eec6528c57" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.95", "zbus_names", "zvariant", "zvariant_utils", @@ -3638,7 +3786,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -3658,7 +3806,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", "synstructure", ] @@ -3687,7 +3835,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.95", ] [[package]] @@ -3742,7 +3890,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.95", "zvariant_utils", ] @@ -3756,6 +3904,6 @@ dependencies = [ "quote", "serde", "static_assertions", - "syn", + "syn 2.0.95", "winnow", ] diff --git a/Cargo.toml b/Cargo.toml index bb4fa6dc..fc97cd3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = ["boulder", "moss", "crates/*"] default-members = ["moss"] +members = ["boulder", "moss", "libstone", "crates/*"] resolver = "2" [workspace.package] diff --git a/justfile b/justfile index 971721b7..3c46ee1c 100644 --- a/justfile +++ b/justfile @@ -85,10 +85,14 @@ diesel db +ARGS: --database-url sqlite://{{root-dir}}/moss/src/db/{{db}}/test.db \ {{ARGS}} -test-ffi: +# Run libstone example +libstone example="read" *ARGS="./test/bash-completion-2.11-1-1-x86_64.stone": #!/bin/bash - cargo cbuild -p stone-ffi - gcc -o stone-ffi-test ./crates/stone-ffi/test.c -L./target/x86_64-unknown-linux-gnu/debug/ -lstone -I./target/x86_64-unknown-linux-gnu/debug/ - ln -sf $(pwd)/target/x86_64-unknown-linux-gnu/debug/libstone.so ./target/x86_64-unknown-linux-gnu/debug/libstone.so.0.24 - LD_LIBRARY_PATH=$(pwd)/target/x86_64-unknown-linux-gnu/debug/ valgrind --track-origins=yes ./stone-ffi-test - rm stone-ffi-test + output=$(mktemp) + cargo build -p libstone --release + clang libstone/examples/{{example}}.c -o $output -I./libstone/src/ -lstone -L./target/release/ -Wl,-rpath,./target/release/ + if [ "$USE_VALGRIND" == "1" ]; + then time valgrind --track-origins=yes $output {{ARGS}}; + else time $output {{ARGS}}; + fi + rm "$output" diff --git a/crates/stone-ffi/Cargo.toml b/libstone/Cargo.toml similarity index 56% rename from crates/stone-ffi/Cargo.toml rename to libstone/Cargo.toml index 9e108e4b..33811bf0 100644 --- a/crates/stone-ffi/Cargo.toml +++ b/libstone/Cargo.toml @@ -1,22 +1,31 @@ [package] -name = "stone-ffi" +name = "libstone" version.workspace = true edition.workspace = true rust-version.workspace = true +[lib] +name = "stone" +crate-type = ["cdylib", "staticlib"] + [features] capi = [] [dependencies] -stone = { path = "../stone", features = ["ffi"] } +stone = { path = "../crates/stone", features = ["ffi"] } libc.workspace = true +[build-dependencies] +cbindgen.workspace = true + [package.metadata.capi.header] name = "stone" +subdirectory = "libstone" [package.metadata.capi.pkg_config] -name = "stone" +name = "libstone" +filename = "libstone" [package.metadata.capi.library] name = "stone" diff --git a/libstone/build.rs b/libstone/build.rs new file mode 100644 index 00000000..f6547cf2 --- /dev/null +++ b/libstone/build.rs @@ -0,0 +1,9 @@ +use std::path::Path; + +fn main() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + + let bindings = cbindgen::generate(env!("CARGO_MANIFEST_DIR")).unwrap(); + + bindings.write_to_file(root.join("src/stone.h")); +} diff --git a/crates/stone-ffi/cbindgen.toml b/libstone/cbindgen.toml similarity index 100% rename from crates/stone-ffi/cbindgen.toml rename to libstone/cbindgen.toml diff --git a/crates/stone-ffi/test.c b/libstone/examples/read.c similarity index 93% rename from crates/stone-ffi/test.c rename to libstone/examples/read.c index 908bcf8d..504489d0 100644 --- a/crates/stone-ffi/test.c +++ b/libstone/examples/read.c @@ -170,6 +170,14 @@ void print_payload_attribute_record(StonePayloadAttributeRecord *record) { printf("}\n"); } +void print_inspect_output(StonePayloadMetaRecord *metas, int num_metas, + StonePayloadLayoutRecord *layouts, int num_layotus) { + + for (int i = 0; i < num_metas; i++) { + StonePayloadMetaRecord *meta = &metas[i]; + } +} + void process_records(StonePayload *payload, StonePayloadHeader *payload_header, void **records, int *num_records, int record_size, int (*next_record)(StonePayload *, void *), @@ -251,21 +259,22 @@ void process_reader(StoneReader *reader, StoneHeaderVersion version) { break; } case STONE_PAYLOAD_KIND_CONTENT: { - void *data = malloc(payload_header.plain_size); - - if (data == NULL) { - exit(1); - } + FILE *fptr; + fptr = fopen("/dev/null", "w+"); - stone_reader_unpack_content_payload(reader, payload, data); + stone_reader_unpack_content_payload_to_file(reader, payload, fileno(fptr)); - free(data); + fclose(fptr); } } current_payload += 1; } + if (num_metas + num_layouts > 0) { + print_inspect_output(metas, num_metas, layouts, num_layouts); + } + if (layouts != NULL) { free(layouts); } @@ -290,20 +299,24 @@ int main(int argc, char *argv[]) { FILE *fptr; StoneReader *reader; StoneHeaderVersion version; - char *file = "./test/bash-completion-2.11-1-1-x86_64.stone"; - printf("Reading stone from '%s'\n\n", file); - fptr = fopen(file, "r"); - stone_reader_read_file(fileno(fptr), &reader, &version); + if (argc != 2) { + printf("usage: %s \n", argv[0]); + exit(1); + } + + printf("Reading stone header from buffer\n\n"); + stone_reader_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); process_reader(reader, version); stone_reader_destroy(reader); - fclose(fptr); printf("\n"); - printf("Reading stone header from buffer\n\n"); - stone_reader_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); + printf("Reading stone from '%s'\n\n", argv[1]); + fptr = fopen(argv[1], "r"); + stone_reader_read_file(fileno(fptr), &reader, &version); process_reader(reader, version); stone_reader_destroy(reader); + fclose(fptr); return 0; } diff --git a/crates/stone-ffi/src/lib.rs b/libstone/src/lib.rs similarity index 92% rename from crates/stone-ffi/src/lib.rs rename to libstone/src/lib.rs index f43c4c20..24dee754 100644 --- a/crates/stone-ffi/src/lib.rs +++ b/libstone/src/lib.rs @@ -160,7 +160,28 @@ pub unsafe extern "C" fn stone_reader_next_payload( } #[no_mangle] -pub unsafe extern "C" fn stone_reader_unpack_content_payload( +pub unsafe extern "C" fn stone_reader_unpack_content_payload_to_file( + reader: *mut StoneReader, + payload: *const StonePayload, + file: c_int, +) -> c_int { + fallible(|| { + let mut reader = NonNull::new(reader).ok_or("")?; + let payload = NonNull::new(payload as *mut StonePayload).ok_or("")?; + let mut file = File::from_raw_fd(file); + + if let StoneDecodedPayload::Content(content) = &payload.as_ref().decoded { + reader.as_mut().unpack_content(content, &mut file)?; + } else { + Err("incorrect payload kind")?; + } + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_reader_unpack_content_payload_to_buf( reader: *mut StoneReader, payload: *const StonePayload, data: *mut u8, diff --git a/crates/stone-ffi/src/payload.rs b/libstone/src/payload.rs similarity index 100% rename from crates/stone-ffi/src/payload.rs rename to libstone/src/payload.rs diff --git a/crates/stone-ffi/src/payload/attribute.rs b/libstone/src/payload/attribute.rs similarity index 100% rename from crates/stone-ffi/src/payload/attribute.rs rename to libstone/src/payload/attribute.rs diff --git a/crates/stone-ffi/src/payload/index.rs b/libstone/src/payload/index.rs similarity index 100% rename from crates/stone-ffi/src/payload/index.rs rename to libstone/src/payload/index.rs diff --git a/crates/stone-ffi/src/payload/layout.rs b/libstone/src/payload/layout.rs similarity index 100% rename from crates/stone-ffi/src/payload/layout.rs rename to libstone/src/payload/layout.rs diff --git a/crates/stone-ffi/src/payload/meta.rs b/libstone/src/payload/meta.rs similarity index 100% rename from crates/stone-ffi/src/payload/meta.rs rename to libstone/src/payload/meta.rs diff --git a/libstone/src/stone.h b/libstone/src/stone.h new file mode 100644 index 00000000..ed416f02 --- /dev/null +++ b/libstone/src/stone.h @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: Copyright © 2020-2024 Serpent OS Developers +// +// SPDX-License-Identifier: MPL-2.0 + + +#ifndef STONE_H +#define STONE_H + +#include +#include +#include +#include + +/** + * Well known file type for a v1 stone container + * + * Some types are now legacy as we're going to use Ion to define them. + * + */ +enum StoneHeaderV1FileType { + /** + * Binary package + */ + STONE_HEADER_V1_FILE_TYPE_BINARY = 1, + /** + * Delta package + */ + STONE_HEADER_V1_FILE_TYPE_DELTA, + /** + * (Legacy) repository index + */ + STONE_HEADER_V1_FILE_TYPE_REPOSITORY, + /** + * (Legacy) build manifest + */ + STONE_HEADER_V1_FILE_TYPE_BUILD_MANIFEST, +}; +typedef uint8_t StoneHeaderV1FileType; + +/** + * Format versions are defined as u32, to allow further mangling in future + */ +enum StoneHeaderVersion { + STONE_HEADER_VERSION_V1 = 1, +}; +typedef uint32_t StoneHeaderVersion; + +enum StonePayloadCompression { + STONE_PAYLOAD_COMPRESSION_NONE = 1, + STONE_PAYLOAD_COMPRESSION_ZSTD = 2, +}; +typedef uint8_t StonePayloadCompression; + +enum StonePayloadKind { + STONE_PAYLOAD_KIND_META = 1, + STONE_PAYLOAD_KIND_CONTENT = 2, + STONE_PAYLOAD_KIND_LAYOUT = 3, + STONE_PAYLOAD_KIND_INDEX = 4, + STONE_PAYLOAD_KIND_ATTRIBUTES = 5, + STONE_PAYLOAD_KIND_DUMB = 6, +}; +typedef uint8_t StonePayloadKind; + +/** + * Layout entries record their target file type so they can be rebuilt on + * the target installation. + */ +enum StonePayloadLayoutFileType { + /** + * Regular file + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_REGULAR = 1, + /** + * Symbolic link (source + target set) + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_SYMLINK, + /** + * Directory node + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_DIRECTORY, + /** + * Character device + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_CHARACTER_DEVICE, + /** + * Block device + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_BLOCK_DEVICE, + /** + * FIFO node + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_FIFO, + /** + * UNIX Socket + */ + STONE_PAYLOAD_LAYOUT_FILE_TYPE_SOCKET, +}; +typedef uint8_t StonePayloadLayoutFileType; + +enum StonePayloadMetaDependency { + /** + * Just the plain name of a package + */ + STONE_PAYLOAD_META_DEPENDENCY_PACKAGE_NAME = 0, + /** + * A soname based dependency + */ + STONE_PAYLOAD_META_DEPENDENCY_SHARED_LIBRARY, + /** + * A pkgconfig `.pc` based dependency + */ + STONE_PAYLOAD_META_DEPENDENCY_PKG_CONFIG, + /** + * Special interpreter (PT_INTERP/etc) to run the binaries + */ + STONE_PAYLOAD_META_DEPENDENCY_INTERPRETER, + /** + * A CMake module + */ + STONE_PAYLOAD_META_DEPENDENCY_C_MAKE, + /** + * A Python module + */ + STONE_PAYLOAD_META_DEPENDENCY_PYTHON, + /** + * A binary in /usr/bin + */ + STONE_PAYLOAD_META_DEPENDENCY_BINARY, + /** + * A binary in /usr/sbin + */ + STONE_PAYLOAD_META_DEPENDENCY_SYSTEM_BINARY, + /** + * An emul32-compatible pkgconfig .pc dependency (lib32/*.pc) + */ + STONE_PAYLOAD_META_DEPENDENCY_PKG_CONFIG32, +}; +typedef uint8_t StonePayloadMetaDependency; + +typedef enum StonePayloadMetaPrimitiveType { + STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT8, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT8, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT16, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT16, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT32, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT32, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_INT64, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_UINT64, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_STRING, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_DEPENDENCY, + STONE_PAYLOAD_META_PRIMITIVE_TYPE_PROVIDER, +} StonePayloadMetaPrimitiveType; + +enum StonePayloadMetaTag { + STONE_PAYLOAD_META_TAG_NAME = 1, + STONE_PAYLOAD_META_TAG_ARCHITECTURE = 2, + STONE_PAYLOAD_META_TAG_VERSION = 3, + STONE_PAYLOAD_META_TAG_SUMMARY = 4, + STONE_PAYLOAD_META_TAG_DESCRIPTION = 5, + STONE_PAYLOAD_META_TAG_HOMEPAGE = 6, + STONE_PAYLOAD_META_TAG_SOURCE_ID = 7, + STONE_PAYLOAD_META_TAG_DEPENDS = 8, + STONE_PAYLOAD_META_TAG_PROVIDES = 9, + STONE_PAYLOAD_META_TAG_CONFLICTS = 10, + STONE_PAYLOAD_META_TAG_RELEASE = 11, + STONE_PAYLOAD_META_TAG_LICENSE = 12, + STONE_PAYLOAD_META_TAG_BUILD_RELEASE = 13, + STONE_PAYLOAD_META_TAG_PACKAGE_URI = 14, + STONE_PAYLOAD_META_TAG_PACKAGE_HASH = 15, + STONE_PAYLOAD_META_TAG_PACKAGE_SIZE = 16, + STONE_PAYLOAD_META_TAG_BUILD_DEPENDS = 17, + STONE_PAYLOAD_META_TAG_SOURCE_URI = 18, + STONE_PAYLOAD_META_TAG_SOURCE_PATH = 19, + STONE_PAYLOAD_META_TAG_SOURCE_REF = 20, +}; +typedef uint16_t StonePayloadMetaTag; + +typedef struct StonePayload StonePayload; + +typedef struct StoneReader StoneReader; + +/** + * Header for the v1 format version + */ +typedef struct StoneHeaderV1 { + uint16_t num_payloads; + StoneHeaderV1FileType file_type; +} StoneHeaderV1; + +typedef struct StonePayloadHeader { + uint64_t stored_size; + uint64_t plain_size; + uint8_t checksum[8]; + uintptr_t num_records; + uint16_t version; + StonePayloadKind kind; + StonePayloadCompression compression; +} StonePayloadHeader; + +typedef struct StoneString { + const uint8_t *buf; + size_t size; +} StoneString; + +typedef struct StonePayloadLayoutFileRegular { + uint8_t hash[16]; + struct StoneString name; +} StonePayloadLayoutFileRegular; + +typedef struct StonePayloadLayoutFileSymlink { + struct StoneString source; + struct StoneString target; +} StonePayloadLayoutFileSymlink; + +typedef union StonePayloadLayoutFilePayload { + struct StonePayloadLayoutFileRegular regular; + struct StonePayloadLayoutFileSymlink symlink; + struct StoneString directory; + struct StoneString character_device; + struct StoneString block_device; + struct StoneString fifo; + struct StoneString socket; +} StonePayloadLayoutFilePayload; + +typedef struct StonePayloadLayoutRecord { + uint32_t uid; + uint32_t gid; + uint32_t mode; + uint32_t tag; + StonePayloadLayoutFileType file_type; + union StonePayloadLayoutFilePayload file_payload; +} StonePayloadLayoutRecord; + +typedef struct StonePayloadMetaDependencyValue { + StonePayloadMetaDependency kind; + struct StoneString name; +} StonePayloadMetaDependencyValue; + +typedef struct StonePayloadMetaProviderValue { + StonePayloadMetaDependency kind; + struct StoneString name; +} StonePayloadMetaProviderValue; + +typedef union StonePayloadMetaPrimitivePayload { + int8_t int8; + uint8_t uint8; + int16_t int16; + uint16_t uint16; + int32_t int32; + uint32_t uint32; + int64_t int64; + uint64_t uint64; + struct StoneString string; + struct StonePayloadMetaDependencyValue dependency; + struct StonePayloadMetaProviderValue provider; +} StonePayloadMetaPrimitivePayload; + +typedef struct StonePayloadMetaRecord { + StonePayloadMetaTag tag; + enum StonePayloadMetaPrimitiveType primitive_type; + union StonePayloadMetaPrimitivePayload primitive_payload; +} StonePayloadMetaRecord; + +typedef struct StonePayloadIndexRecord { + uint64_t start; + uint64_t end; + uint8_t digest[16]; +} StonePayloadIndexRecord; + +typedef struct StonePayloadAttributeRecord { + uintptr_t key_size; + const uint8_t *key_buf; + uintptr_t value_size; + const uint8_t *value_buf; +} StonePayloadAttributeRecord; + + + +int stone_reader_read_file(int file, struct StoneReader **reader_ptr, StoneHeaderVersion *version); + +int stone_reader_read_buf(const uint8_t *buf, + uintptr_t len, + struct StoneReader **reader_ptr, + StoneHeaderVersion *version); + +int stone_reader_header_v1(const struct StoneReader *reader, struct StoneHeaderV1 *header); + +int stone_reader_next_payload(struct StoneReader *reader, struct StonePayload **payload_ptr); + +int stone_reader_unpack_content_payload_to_file(struct StoneReader *reader, + const struct StonePayload *payload, + int file); + +int stone_reader_unpack_content_payload_to_buf(struct StoneReader *reader, + const struct StonePayload *payload, + uint8_t *data); + +void stone_reader_destroy(struct StoneReader *reader); + +int stone_payload_header(const struct StonePayload *payload, struct StonePayloadHeader *header); + +int stone_payload_next_layout_record(struct StonePayload *payload, + struct StonePayloadLayoutRecord *record); + +int stone_payload_next_meta_record(struct StonePayload *payload, + struct StonePayloadMetaRecord *record); + +int stone_payload_next_index_record(struct StonePayload *payload, + struct StonePayloadIndexRecord *record); + +int stone_payload_next_attribute_record(struct StonePayload *payload, + struct StonePayloadAttributeRecord *record); + +void stone_payload_destroy(struct StonePayload *payload); + +void stone_format_header_v1_file_type(StoneHeaderV1FileType file_type, uint8_t *buf); + +void stone_format_payload_compression(StonePayloadCompression compression, uint8_t *buf); + +void stone_format_payload_kind(StonePayloadKind kind, uint8_t *buf); + +void stone_format_payload_layout_file_type(StonePayloadLayoutFileType file_type, uint8_t *buf); + +void stone_format_payload_meta_tag(StonePayloadMetaTag tag, uint8_t *buf); + +void stone_format_payload_meta_dependency(StonePayloadMetaDependency dependency, uint8_t *buf); + +#endif /* STONE_H */ diff --git a/moss/src/cli/inspect.rs b/moss/src/cli/inspect.rs index d9690176..304d9b33 100644 --- a/moss/src/cli/inspect.rs +++ b/moss/src/cli/inspect.rs @@ -61,15 +61,15 @@ pub fn handle(args: &ArgMatches) -> Result<(), Error> { StonePayloadMetaPrimitive::Provider(k, p) if record.tag == StonePayloadMetaTag::Provides => { - provs.push(format!("{k}({p})")) + provs.push(format!("{k}({p})")); } StonePayloadMetaPrimitive::Provider(k, p) if record.tag == StonePayloadMetaTag::Conflicts => { - cnfls.push(format!("{k}({p})")) + cnfls.push(format!("{k}({p})")); } StonePayloadMetaPrimitive::Dependency(k, d) => { - deps.push(format!("{}({})", k, d)); + deps.push(format!("{k}({d})")); } StonePayloadMetaPrimitive::String(s) => { println!("{name:COLUMN_WIDTH$} : {s}"); From 08c8f6ab1471048bf9a9aa46b6e86aa266c50a3c Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Fri, 1 Nov 2024 08:55:30 -0700 Subject: [PATCH 07/12] Add read vtable & payload read method --- crates/stone/src/lib.rs | 2 + crates/stone/src/read/digest.rs | 2 +- crates/stone/src/read/mod.rs | 55 ++++++++- crates/stone/src/read/zstd.rs | 4 + libstone/cbindgen.toml | 5 +- libstone/examples/read.c | 45 ++++++- libstone/src/lib.rs | 210 +++++++++++++++++++++++++------- libstone/src/stone.h | 64 +++++++--- 8 files changed, 314 insertions(+), 73 deletions(-) diff --git a/crates/stone/src/lib.rs b/crates/stone/src/lib.rs index 32cb10d6..4ec8f345 100644 --- a/crates/stone/src/lib.rs +++ b/crates/stone/src/lib.rs @@ -18,6 +18,8 @@ pub use self::payload::{ StonePayloadLayoutFileType, StonePayloadLayoutRecord, StonePayloadMetaDependency, StonePayloadMetaPrimitive, StonePayloadMetaRecord, StonePayloadMetaTag, }; +#[cfg(feature = "ffi")] +pub use self::read::StonePayloadContentReader; pub use self::read::{read, read_bytes, StoneDecodedPayload, StoneReadError, StoneReader}; pub use self::write::{ StoneContentWriter, StoneDigestWriter, StoneDigestWriterHasher, StoneWriteError, StoneWritePayload, StoneWriter, diff --git a/crates/stone/src/read/digest.rs b/crates/stone/src/read/digest.rs index 33a29c32..2f366976 100644 --- a/crates/stone/src/read/digest.rs +++ b/crates/stone/src/read/digest.rs @@ -10,7 +10,7 @@ pub type Hasher = Xxh3; pub struct Reader<'a, R: Read> { inner: R, - hasher: &'a mut Hasher, + pub hasher: &'a mut Hasher, } impl<'a, R> Reader<'a, R> diff --git a/crates/stone/src/read/mod.rs b/crates/stone/src/read/mod.rs index dec320c1..eba86484 100644 --- a/crates/stone/src/read/mod.rs +++ b/crates/stone/src/read/mod.rs @@ -72,13 +72,10 @@ impl StoneReader { self.reader.seek(SeekFrom::Start(content.body.offset))?; self.hasher.reset(); - let mut hashed = digest::Reader::new(&mut self.reader, &mut self.hasher); - let mut framed = (&mut hashed).take(content.header.stored_size); + let hashed = digest::Reader::new(&mut self.reader, &mut self.hasher); + let framed = hashed.take(content.header.stored_size); - io::copy( - &mut PayloadReader::new(&mut framed, content.header.compression)?, - writer, - )?; + io::copy(&mut PayloadReader::new(framed, content.header.compression)?, writer)?; // Validate checksum validate_checksum(&self.hasher, &content.header)?; @@ -100,6 +97,45 @@ impl StoneReader { Ok(None) } } + + pub fn read_content<'a>( + &'a mut self, + content: &StonePayload, + ) -> Result, StoneReadError> { + self.reader.seek(SeekFrom::Start(content.body.offset))?; + self.hasher.reset(); + + let hashed = digest::Reader::new(&mut self.reader, &mut self.hasher); + let framed = hashed.take(content.header.stored_size); + + Ok(StonePayloadContentReader { + reader: PayloadReader::new(framed, content.header.compression)?, + header_checksum: u64::from_be_bytes(content.header.checksum), + is_checksum_valid: false, + }) + } +} + +#[cfg(feature = "ffi")] +pub struct StonePayloadContentReader<'a, R: Read> { + reader: PayloadReader>>, + header_checksum: u64, + pub is_checksum_valid: bool, +} + +#[cfg(feature = "ffi")] +impl<'a, R: Read> Read for StonePayloadContentReader<'a, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self.reader.read(buf) { + Ok(read) if !buf.is_empty() && read == 0 => { + self.is_checksum_valid = self.header_checksum == self.reader.get_mut().get_mut().hasher.digest(); + + Ok(read) + } + Ok(read) => Ok(read), + e @ Err(_) => e, + } + } } enum PayloadReader { @@ -114,6 +150,13 @@ impl PayloadReader { StonePayloadCompression::Zstd => PayloadReader::Zstd(Zstd::new(reader)?), }) } + + fn get_mut(&mut self) -> &mut R { + match self { + PayloadReader::Plain(reader) => reader, + PayloadReader::Zstd(reader) => reader.get_mut(), + } + } } impl Read for PayloadReader { diff --git a/crates/stone/src/read/zstd.rs b/crates/stone/src/read/zstd.rs index e82758e9..c6b868f8 100644 --- a/crates/stone/src/read/zstd.rs +++ b/crates/stone/src/read/zstd.rs @@ -17,6 +17,10 @@ impl Zstd { Ok(Self { decoder }) } + + pub fn get_mut(&mut self) -> &mut R { + self.decoder.get_mut().get_mut() + } } impl Read for Zstd { diff --git a/libstone/cbindgen.toml b/libstone/cbindgen.toml index 2f6c68c8..23771edc 100644 --- a/libstone/cbindgen.toml +++ b/libstone/cbindgen.toml @@ -12,9 +12,12 @@ prefix_with_name = true rename_variants = "ScreamingSnakeCase" [export] -include = ["StonePayloadLayoutRecord"] +include = [] +exclude = [] [export.rename] +"StoneReader_StoneReadImpl" = "StoneReader" +"StonePayloadContentReader_StoneReadImpl" = "StonePayloadContentReader" [parse] parse_deps = true diff --git a/libstone/examples/read.c b/libstone/examples/read.c index 504489d0..1259da2c 100644 --- a/libstone/examples/read.c +++ b/libstone/examples/read.c @@ -3,6 +3,7 @@ #include #include #include +#include #include // 32 byte stone header @@ -260,10 +261,34 @@ void process_reader(StoneReader *reader, StoneHeaderVersion version) { } case STONE_PAYLOAD_KIND_CONTENT: { FILE *fptr; + StonePayloadContentReader *content_reader; fptr = fopen("/dev/null", "w+"); + char buf[1024]; + int read = 0; - stone_reader_unpack_content_payload_to_file(reader, payload, fileno(fptr)); + // We can instead unpack directly to file as a convenience + // stone_reader_unpack_content_payload(reader, payload, fileno(fptr)); + stone_reader_read_content_payload(reader, payload, &content_reader); + + while ((read = stone_payload_content_reader_read( + content_reader, (void *)buf, sizeof buf)) > 0) { + int total = 0, n = 0; + + while (total < read) { + n = fwrite(buf, 1, read - total, fptr); + if (n == 0) { + exit(1); + } + total += n; + } + } + + assert(stone_payload_content_reader_is_checksum_valid(content_reader) == + 1); + + stone_payload_content_reader_destroy(content_reader); + fflush(fptr); fclose(fptr); } } @@ -295,6 +320,20 @@ void process_reader(StoneReader *reader, StoneHeaderVersion version) { } } +size_t read_shim(void *fptr, char *buf, size_t n) { + return fread(buf, 1, n, fptr); +} + +uint64_t seek_shim(void *fptr, int64_t offset, StoneSeekFrom from) { + fseek(fptr, offset, from); + return ftell(fptr); +} + +StoneReadVTable vtable = { + .read = read_shim, + .seek = seek_shim, +}; + int main(int argc, char *argv[]) { FILE *fptr; StoneReader *reader; @@ -306,14 +345,14 @@ int main(int argc, char *argv[]) { } printf("Reading stone header from buffer\n\n"); - stone_reader_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); + stone_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); process_reader(reader, version); stone_reader_destroy(reader); printf("\n"); printf("Reading stone from '%s'\n\n", argv[1]); fptr = fopen(argv[1], "r"); - stone_reader_read_file(fileno(fptr), &reader, &version); + stone_read(fptr, vtable, &reader, &version); process_reader(reader, version); stone_reader_destroy(reader); fclose(fptr); diff --git a/libstone/src/lib.rs b/libstone/src/lib.rs index 24dee754..cbca4316 100644 --- a/libstone/src/lib.rs +++ b/libstone/src/lib.rs @@ -6,17 +6,17 @@ use std::{ fs::File, - io::{Cursor, Write}, + io::{Cursor, Read, Seek}, os::fd::FromRawFd, ptr::NonNull, slice, }; -use libc::{c_int, size_t}; +use libc::{c_char, c_int, c_void, size_t}; use stone::{ StoneDecodedPayload, StoneHeader, StoneHeaderV1, StoneHeaderV1FileType, StoneHeaderVersion, StonePayloadCompression, StonePayloadHeader, StonePayloadKind, StonePayloadLayoutFileType, - StonePayloadMetaDependency, StonePayloadMetaTag, StoneReadError, + StonePayloadMetaDependency, StonePayloadMetaTag, }; pub use self::payload::{ @@ -26,6 +26,9 @@ pub use self::payload::{ mod payload; +pub type StoneReader<'a> = stone::StoneReader>; +pub type StonePayloadContentReader<'a> = stone::StonePayloadContentReader<'a, StoneReadImpl<'a>>; + #[derive(Debug, Clone, Copy)] #[repr(C)] pub struct StoneString { @@ -42,38 +45,55 @@ impl StoneString { } } -pub enum StoneReader<'a> { - File(stone::StoneReader), - Buffer(stone::StoneReader>), +pub enum StoneReadImpl<'a> { + File(File), + Buffer(Cursor<&'a [u8]>), + Shim(StoneReadShim), } -impl<'a> StoneReader<'a> { - pub fn header(&self) -> &StoneHeader { - match self { - StoneReader::File(reader) => &reader.header, - StoneReader::Buffer(reader) => &reader.header, +macro_rules! delegate { + ($self:expr,$($t:tt)*) => { + match $self { + StoneReadImpl::File(reader) => reader.$($t)*, + StoneReadImpl::Buffer(reader) => reader.$($t)*, + StoneReadImpl::Shim(reader) => reader.$($t)*, } + }; +} + +impl<'a> Read for StoneReadImpl<'a> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + delegate!(self, read(buf)) } - pub fn next_payload(&mut self) -> Result, StoneReadError> { - match self { - StoneReader::File(reader) => reader.next_payload(), - StoneReader::Buffer(reader) => reader.next_payload(), - } + fn read_vectored(&mut self, bufs: &mut [std::io::IoSliceMut<'_>]) -> std::io::Result { + delegate!(self, read_vectored(bufs)) } - pub fn unpack_content( - &mut self, - content: &stone::StonePayload, - writer: &mut W, - ) -> Result<(), StoneReadError> - where - W: Write, - { - match self { - StoneReader::File(reader) => reader.unpack_content(content, writer), - StoneReader::Buffer(reader) => reader.unpack_content(content, writer), - } + fn read_to_end(&mut self, buf: &mut Vec) -> std::io::Result { + delegate!(self, read_to_end(buf)) + } + + fn read_to_string(&mut self, buf: &mut String) -> std::io::Result { + delegate!(self, read_to_string(buf)) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { + delegate!(self, read_exact(buf)) + } +} + +impl<'a> Seek for StoneReadImpl<'a> { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + delegate!(self, seek(pos)) + } + + fn rewind(&mut self) -> std::io::Result<()> { + delegate!(self, rewind()) + } + + fn stream_position(&mut self) -> std::io::Result { + delegate!(self, stream_position()) } } @@ -85,20 +105,63 @@ fn fallible(f: impl FnOnce() -> Result<(), Box>) -> c_int } } +#[repr(u8)] +enum StoneSeekFrom { + Start = 0, + Current = 1, + End = 2, +} + +#[repr(C)] +pub struct StoneReadVTable { + read: Option usize>, + seek: Option u64>, +} + +pub struct StoneReadShim { + data: *mut c_void, + read: unsafe extern "C" fn(*mut c_void, *mut c_char, usize) -> usize, + seek: unsafe extern "C" fn(*mut c_void, i64, StoneSeekFrom) -> u64, +} + +impl Read for StoneReadShim { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + unsafe { Ok((self.read)(self.data, buf.as_mut_ptr() as *mut _, buf.len())) } + } +} + +impl Seek for StoneReadShim { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + let (from, offset) = match pos { + std::io::SeekFrom::Start(i) => (StoneSeekFrom::Start, i as i64), + std::io::SeekFrom::Current(i) => (StoneSeekFrom::Current, i), + std::io::SeekFrom::End(i) => (StoneSeekFrom::End, i), + }; + + unsafe { Ok((self.seek)(self.data, offset, from)) } + } +} + #[no_mangle] -pub unsafe extern "C" fn stone_reader_read_file( - file: c_int, +pub unsafe extern "C" fn stone_read( + data: *mut c_void, + vtable: StoneReadVTable, reader_ptr: *mut *mut StoneReader, version: *mut StoneHeaderVersion, ) -> c_int { fallible(|| { // TODO: Errors + let data = NonNull::new(data).ok_or("")?; let reader_ptr = NonNull::new(reader_ptr).ok_or("")?; let mut version = NonNull::new(version).ok_or("")?; - let reader = stone::read(File::from_raw_fd(file)).map(StoneReader::File)?; + let reader = stone::read(StoneReadImpl::Shim(StoneReadShim { + data: data.as_ptr(), + read: vtable.read.ok_or("")?, + seek: vtable.seek.ok_or("")?, + }))?; - *version.as_mut() = reader.header().version(); + *version.as_mut() = reader.header.version(); *reader_ptr.as_ptr() = Box::into_raw(Box::new(reader)); Ok(()) @@ -106,7 +169,26 @@ pub unsafe extern "C" fn stone_reader_read_file( } #[no_mangle] -pub unsafe extern "C" fn stone_reader_read_buf( +pub unsafe extern "C" fn stone_read_file( + file: c_int, + reader_ptr: *mut *mut StoneReader, + version: *mut StoneHeaderVersion, +) -> c_int { + fallible(|| { + let reader_ptr = NonNull::new(reader_ptr).ok_or("")?; + let mut version = NonNull::new(version).ok_or("")?; + + let reader = stone::read(StoneReadImpl::File(File::from_raw_fd(file)))?; + + *version.as_mut() = reader.header.version(); + *reader_ptr.as_ptr() = Box::into_raw(Box::new(reader)); + + Ok(()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_read_buf( buf: *const u8, len: usize, reader_ptr: *mut *mut StoneReader, @@ -117,9 +199,12 @@ pub unsafe extern "C" fn stone_reader_read_buf( let reader_ptr = NonNull::new(reader_ptr).ok_or("")?; let mut version = NonNull::new(version).ok_or("")?; - let reader = stone::read_bytes(slice::from_raw_parts(buf.as_ptr(), len)).map(StoneReader::Buffer)?; + let reader = stone::read(StoneReadImpl::Buffer(Cursor::new(slice::from_raw_parts( + buf.as_ptr(), + len, + ))))?; - *version.as_mut() = reader.header().version(); + *version.as_mut() = reader.header.version(); *reader_ptr.as_ptr() = Box::into_raw(Box::new(reader)); Ok(()) @@ -132,7 +217,7 @@ pub unsafe extern "C" fn stone_reader_header_v1(reader: *const StoneReader, head let reader = NonNull::new(reader as *mut StoneReader).ok_or("")?; let mut header = NonNull::new(header).ok_or("")?; - match reader.as_ref().header() { + match &reader.as_ref().header { StoneHeader::V1(v1) => *header.as_mut() = *v1, } @@ -160,7 +245,7 @@ pub unsafe extern "C" fn stone_reader_next_payload( } #[no_mangle] -pub unsafe extern "C" fn stone_reader_unpack_content_payload_to_file( +pub unsafe extern "C" fn stone_reader_unpack_content_payload( reader: *mut StoneReader, payload: *const StonePayload, file: c_int, @@ -181,23 +266,18 @@ pub unsafe extern "C" fn stone_reader_unpack_content_payload_to_file( } #[no_mangle] -pub unsafe extern "C" fn stone_reader_unpack_content_payload_to_buf( - reader: *mut StoneReader, +pub unsafe extern "C" fn stone_reader_read_content_payload<'a>( + reader: *mut StoneReader<'a>, payload: *const StonePayload, - data: *mut u8, + content_reader: *mut *mut StonePayloadContentReader<'a>, ) -> c_int { fallible(|| { let mut reader = NonNull::new(reader).ok_or("")?; let payload = NonNull::new(payload as *mut StonePayload).ok_or("")?; - let data = NonNull::new(data).ok_or("")?; - - let mut cursor = Cursor::new(slice::from_raw_parts_mut( - data.as_ptr(), - payload.as_ref().header().stored_size as usize, - )); + let content_reader = NonNull::new(content_reader).ok_or("")?; if let StoneDecodedPayload::Content(content) = &payload.as_ref().decoded { - reader.as_mut().unpack_content(content, &mut cursor)?; + *content_reader.as_ptr() = Box::into_raw(Box::new(reader.as_mut().read_content(content)?)); } else { Err("incorrect payload kind")?; } @@ -215,6 +295,42 @@ pub unsafe extern "C" fn stone_reader_destroy(reader: *mut StoneReader) { drop(Box::from_raw(reader.as_ptr())); } +#[no_mangle] +pub unsafe extern "C" fn stone_payload_content_reader_read( + content_reader: *mut StonePayloadContentReader, + buf: *mut u8, + size: size_t, +) -> size_t { + let Some(mut content_reader) = NonNull::new(content_reader) else { + return 0; + }; + + content_reader + .as_mut() + .read(slice::from_raw_parts_mut(buf, size)) + .unwrap_or(0) +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_content_reader_is_checksum_valid( + content_reader: *const StonePayloadContentReader, +) -> c_int { + let Some(content_reader) = NonNull::new(content_reader as *mut StonePayloadContentReader) else { + return -1; + }; + + content_reader.as_ref().is_checksum_valid as c_int +} + +#[no_mangle] +pub unsafe extern "C" fn stone_payload_content_reader_destroy(content_reader: *mut StonePayloadContentReader) { + let Some(content_reader) = NonNull::new(content_reader) else { + return; + }; + + drop(Box::from_raw(content_reader.as_ptr())); +} + #[no_mangle] pub unsafe extern "C" fn stone_payload_header(payload: *const StonePayload, header: *mut StonePayloadHeader) -> c_int { fallible(|| { diff --git a/libstone/src/stone.h b/libstone/src/stone.h index ed416f02..cdc00598 100644 --- a/libstone/src/stone.h +++ b/libstone/src/stone.h @@ -11,6 +11,8 @@ #include #include +#define STONE_HEADER_SIZE 32 + /** * Well known file type for a v1 stone container * @@ -131,7 +133,7 @@ enum StonePayloadMetaDependency { */ STONE_PAYLOAD_META_DEPENDENCY_SYSTEM_BINARY, /** - * An emul32-compatible pkgconfig .pc dependency (lib32/*.pc) + * An emul32-compatible pkgconfig .pc dependency (lib32*.pc) */ STONE_PAYLOAD_META_DEPENDENCY_PKG_CONFIG32, }; @@ -175,8 +177,24 @@ enum StonePayloadMetaTag { }; typedef uint16_t StonePayloadMetaTag; +enum StoneSeekFrom { + STONE_SEEK_FROM_START = 0, + STONE_SEEK_FROM_CURRENT = 1, + STONE_SEEK_FROM_END = 2, +}; +typedef uint8_t StoneSeekFrom; + typedef struct StonePayload StonePayload; +typedef struct StonePayloadContentReader StonePayloadContentReader; + +typedef struct StoneReader StoneReader; + +typedef struct StoneReadVTable { + uintptr_t (*read)(void*, char*, uintptr_t); + int64_t (*seek)(void*, int64_t, StoneSeekFrom); +} StoneReadVTable; + typedef struct StoneReader StoneReader; /** @@ -187,6 +205,8 @@ typedef struct StoneHeaderV1 { StoneHeaderV1FileType file_type; } StoneHeaderV1; +typedef struct StonePayloadContentReader StonePayloadContentReader; + typedef struct StonePayloadHeader { uint64_t stored_size; uint64_t plain_size; @@ -274,28 +294,42 @@ typedef struct StonePayloadAttributeRecord { const uint8_t *value_buf; } StonePayloadAttributeRecord; +int stone_read(void *data, + struct StoneReadVTable vtable, + StoneReader **reader_ptr, + StoneHeaderVersion *version); + +int stone_read_file(int file, StoneReader **reader_ptr, StoneHeaderVersion *version); + +int stone_read_buf(const uint8_t *buf, + uintptr_t len, + StoneReader **reader_ptr, + StoneHeaderVersion *version); + +int stone_reader_header_v1(const StoneReader *reader, struct StoneHeaderV1 *header); +int stone_reader_next_payload(StoneReader *reader, struct StonePayload **payload_ptr); -int stone_reader_read_file(int file, struct StoneReader **reader_ptr, StoneHeaderVersion *version); +int stone_reader_unpack_content_payload(StoneReader *reader, + const struct StonePayload *payload, + int file); -int stone_reader_read_buf(const uint8_t *buf, - uintptr_t len, - struct StoneReader **reader_ptr, - StoneHeaderVersion *version); +int stone_reader_read_content_payload(StoneReader *reader, + const struct StonePayload *payload, + StonePayloadContentReader **content_reader); -int stone_reader_header_v1(const struct StoneReader *reader, struct StoneHeaderV1 *header); +void stone_reader_destroy(StoneReader *reader); -int stone_reader_next_payload(struct StoneReader *reader, struct StonePayload **payload_ptr); +size_t stone_payload_content_reader_read(StonePayloadContentReader *content_reader, + uint8_t *buf, + size_t size); -int stone_reader_unpack_content_payload_to_file(struct StoneReader *reader, - const struct StonePayload *payload, - int file); +int stone_payload_content_reader_buf_hint(const StonePayloadContentReader *content_reader, + uintptr_t *hint); -int stone_reader_unpack_content_payload_to_buf(struct StoneReader *reader, - const struct StonePayload *payload, - uint8_t *data); +int stone_payload_content_reader_is_checksum_valid(const StonePayloadContentReader *content_reader); -void stone_reader_destroy(struct StoneReader *reader); +void stone_payload_content_reader_destroy(StonePayloadContentReader *content_reader); int stone_payload_header(const struct StonePayload *payload, struct StonePayloadHeader *header); From fe6590f8cc0b0022bb0240bd6e3025149d59a426 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Fri, 1 Nov 2024 09:18:20 -0700 Subject: [PATCH 08/12] Add hint for read buffer size if compression is used --- crates/stone/src/read/mod.rs | 14 +++++++++++++- crates/stone/src/read/zstd.rs | 4 ++++ libstone/examples/read.c | 16 ++++++++++++++-- libstone/src/lib.rs | 14 ++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/crates/stone/src/read/mod.rs b/crates/stone/src/read/mod.rs index eba86484..b05326dc 100644 --- a/crates/stone/src/read/mod.rs +++ b/crates/stone/src/read/mod.rs @@ -107,11 +107,15 @@ impl StoneReader { let hashed = digest::Reader::new(&mut self.reader, &mut self.hasher); let framed = hashed.take(content.header.stored_size); + let reader = PayloadReader::new(framed, content.header.compression)?; + + let buf_hint = reader.buf_hint(); Ok(StonePayloadContentReader { - reader: PayloadReader::new(framed, content.header.compression)?, + reader, header_checksum: u64::from_be_bytes(content.header.checksum), is_checksum_valid: false, + buf_hint, }) } } @@ -121,6 +125,7 @@ pub struct StonePayloadContentReader<'a, R: Read> { reader: PayloadReader>>, header_checksum: u64, pub is_checksum_valid: bool, + pub buf_hint: Option, } #[cfg(feature = "ffi")] @@ -157,6 +162,13 @@ impl PayloadReader { PayloadReader::Zstd(reader) => reader.get_mut(), } } + + fn buf_hint(&self) -> Option { + match self { + PayloadReader::Plain(_) => None, + PayloadReader::Zstd(z) => Some(z.capacity()), + } + } } impl Read for PayloadReader { diff --git a/crates/stone/src/read/zstd.rs b/crates/stone/src/read/zstd.rs index c6b868f8..7d362ebf 100644 --- a/crates/stone/src/read/zstd.rs +++ b/crates/stone/src/read/zstd.rs @@ -21,6 +21,10 @@ impl Zstd { pub fn get_mut(&mut self) -> &mut R { self.decoder.get_mut().get_mut() } + + pub fn capacity(&self) -> usize { + self.decoder.get_ref().capacity() + } } impl Read for Zstd { diff --git a/libstone/examples/read.c b/libstone/examples/read.c index 1259da2c..46c12ba5 100644 --- a/libstone/examples/read.c +++ b/libstone/examples/read.c @@ -263,7 +263,8 @@ void process_reader(StoneReader *reader, StoneHeaderVersion version) { FILE *fptr; StonePayloadContentReader *content_reader; fptr = fopen("/dev/null", "w+"); - char buf[1024]; + char *buf; + uint64_t buf_hint = 0; int read = 0; // We can instead unpack directly to file as a convenience @@ -271,8 +272,18 @@ void process_reader(StoneReader *reader, StoneHeaderVersion version) { stone_reader_read_content_payload(reader, payload, &content_reader); + stone_payload_content_reader_buf_hint(content_reader, &buf_hint); + + buf_hint = buf_hint > 0 ? buf_hint : 1024; + printf("Unpacking w/ buffer size: %ld\n", buf_hint); + + buf = malloc(buf_hint); + if (buf == NULL) { + exit(1); + } + while ((read = stone_payload_content_reader_read( - content_reader, (void *)buf, sizeof buf)) > 0) { + content_reader, (void *)buf, buf_hint)) > 0) { int total = 0, n = 0; while (total < read) { @@ -288,6 +299,7 @@ void process_reader(StoneReader *reader, StoneHeaderVersion version) { 1); stone_payload_content_reader_destroy(content_reader); + free(buf); fflush(fptr); fclose(fptr); } diff --git a/libstone/src/lib.rs b/libstone/src/lib.rs index cbca4316..44e14a11 100644 --- a/libstone/src/lib.rs +++ b/libstone/src/lib.rs @@ -311,6 +311,20 @@ pub unsafe extern "C" fn stone_payload_content_reader_read( .unwrap_or(0) } +#[no_mangle] +pub unsafe extern "C" fn stone_payload_content_reader_buf_hint( + content_reader: *const StonePayloadContentReader, + hint: *mut usize, +) -> c_int { + fallible(|| { + let content_reader = NonNull::new(content_reader as *mut StonePayloadContentReader).ok_or("")?; + + *hint = content_reader.as_ref().buf_hint.unwrap_or(0); + + Ok(()) + }) +} + #[no_mangle] pub unsafe extern "C" fn stone_payload_content_reader_is_checksum_valid( content_reader: *const StonePayloadContentReader, From e4ad120dfb199b295315eeef8119a11ed5b95a99 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Fri, 1 Nov 2024 09:19:28 -0700 Subject: [PATCH 09/12] Remove read_buf from example --- libstone/examples/read.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/libstone/examples/read.c b/libstone/examples/read.c index 46c12ba5..84980412 100644 --- a/libstone/examples/read.c +++ b/libstone/examples/read.c @@ -6,12 +6,6 @@ #include #include -// 32 byte stone header -static uint8_t HEADER_BUF[] = {0x00, 0x6d, 0x6f, 0x73, 0x00, 0x04, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03, 0x00, - 0x00, 0x04, 0x00, 0x00, 0x05, 0x00, 0x00, 0x06, - 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x01}; - void print_header_v1(StoneHeaderV1 *header) { uint8_t file_type[100]; @@ -356,11 +350,6 @@ int main(int argc, char *argv[]) { exit(1); } - printf("Reading stone header from buffer\n\n"); - stone_read_buf(HEADER_BUF, sizeof(HEADER_BUF), &reader, &version); - process_reader(reader, version); - stone_reader_destroy(reader); - printf("\n"); printf("Reading stone from '%s'\n\n", argv[1]); fptr = fopen(argv[1], "r"); From 534dfa46a9ee338f0815a582e7f5208e8a14c4d0 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Mon, 4 Nov 2024 14:43:37 -0800 Subject: [PATCH 10/12] Small fixes --- crates/stone/src/payload/meta.rs | 2 +- crates/stone/src/read/mod.rs | 8 ++++---- libstone/cbindgen.toml | 2 +- libstone/examples/read.c | 2 +- libstone/src/lib.rs | 14 +++++++++++--- libstone/src/payload.rs | 16 ++++++++-------- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/crates/stone/src/payload/meta.rs b/crates/stone/src/payload/meta.rs index c53802a9..69418ded 100644 --- a/crates/stone/src/payload/meta.rs +++ b/crates/stone/src/payload/meta.rs @@ -48,7 +48,7 @@ pub enum StonePayloadMetaDependency { #[strum(serialize = "sysbinary")] SystemBinary, - /// An emul32-compatible pkgconfig .pc dependency (lib32/*.pc) + /// An emul32-compatible pkgconfig .pc dependency (lib32*.pc) PkgConfig32, } diff --git a/crates/stone/src/read/mod.rs b/crates/stone/src/read/mod.rs index b05326dc..ea2a29ef 100644 --- a/crates/stone/src/read/mod.rs +++ b/crates/stone/src/read/mod.rs @@ -237,14 +237,14 @@ impl StoneDecodedPayload { )?, }), StonePayloadKind::Content => { - let offset = reader.stream_position()?; - // Skip past, these are read by user later - reader.seek(SeekFrom::Current(header.stored_size as i64))?; + let new_offset = reader.seek(SeekFrom::Current(header.stored_size as i64))?; StoneDecodedPayload::Content(StonePayload { header, - body: StonePayloadContent { offset }, + body: StonePayloadContent { + offset: (new_offset as i64 - header.stored_size as i64) as u64, + }, }) } StonePayloadKind::Dumb => unimplemented!("??"), diff --git a/libstone/cbindgen.toml b/libstone/cbindgen.toml index 23771edc..89974556 100644 --- a/libstone/cbindgen.toml +++ b/libstone/cbindgen.toml @@ -12,7 +12,7 @@ prefix_with_name = true rename_variants = "ScreamingSnakeCase" [export] -include = [] +include = ["STONE_HEADER_SIZE"] exclude = [] [export.rename] diff --git a/libstone/examples/read.c b/libstone/examples/read.c index 84980412..1173915f 100644 --- a/libstone/examples/read.c +++ b/libstone/examples/read.c @@ -330,7 +330,7 @@ size_t read_shim(void *fptr, char *buf, size_t n) { return fread(buf, 1, n, fptr); } -uint64_t seek_shim(void *fptr, int64_t offset, StoneSeekFrom from) { +int64_t seek_shim(void *fptr, int64_t offset, StoneSeekFrom from) { fseek(fptr, offset, from); return ftell(fptr); } diff --git a/libstone/src/lib.rs b/libstone/src/lib.rs index 44e14a11..087a079b 100644 --- a/libstone/src/lib.rs +++ b/libstone/src/lib.rs @@ -26,6 +26,8 @@ pub use self::payload::{ mod payload; +pub const STONE_HEADER_SIZE: usize = 32; + pub type StoneReader<'a> = stone::StoneReader>; pub type StonePayloadContentReader<'a> = stone::StonePayloadContentReader<'a, StoneReadImpl<'a>>; @@ -115,13 +117,13 @@ enum StoneSeekFrom { #[repr(C)] pub struct StoneReadVTable { read: Option usize>, - seek: Option u64>, + seek: Option i64>, } pub struct StoneReadShim { data: *mut c_void, read: unsafe extern "C" fn(*mut c_void, *mut c_char, usize) -> usize, - seek: unsafe extern "C" fn(*mut c_void, i64, StoneSeekFrom) -> u64, + seek: unsafe extern "C" fn(*mut c_void, i64, StoneSeekFrom) -> i64, } impl Read for StoneReadShim { @@ -138,7 +140,13 @@ impl Seek for StoneReadShim { std::io::SeekFrom::End(i) => (StoneSeekFrom::End, i), }; - unsafe { Ok((self.seek)(self.data, offset, from)) } + let ret = unsafe { (self.seek)(self.data, offset, from) }; + + if ret < 0 { + Err(std::io::Error::new(std::io::ErrorKind::Other, ret.to_string())) + } else { + Ok(ret as u64) + } } } diff --git a/libstone/src/payload.rs b/libstone/src/payload.rs index e2e3f620..a8e65f31 100644 --- a/libstone/src/payload.rs +++ b/libstone/src/payload.rs @@ -25,11 +25,11 @@ impl StonePayload { return None; } - self.next_record += 1; - let payload = self.decoded.layout()?; let record = payload.body.get(self.next_record)?; + self.next_record += 1; + Some(record.into()) } @@ -38,11 +38,11 @@ impl StonePayload { return None; } - self.next_record += 1; - let payload = self.decoded.meta()?; let record = payload.body.get(self.next_record)?; + self.next_record += 1; + Some(record.into()) } @@ -51,11 +51,11 @@ impl StonePayload { return None; } - self.next_record += 1; - let payload = self.decoded.index()?; let record = payload.body.get(self.next_record)?; + self.next_record += 1; + Some(record.into()) } @@ -64,11 +64,11 @@ impl StonePayload { return None; } - self.next_record += 1; - let payload = self.decoded.attributes()?; let record = payload.body.get(self.next_record)?; + self.next_record += 1; + Some(record.into()) } } From 748a4a893e8e690d44541d2a07e555626613bffa Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Mon, 4 Nov 2024 15:04:47 -0800 Subject: [PATCH 11/12] Fix cargo-c usage --- crates/stone/src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/stone/src/write.rs b/crates/stone/src/write.rs index a7d0ef0d..1d8680cc 100644 --- a/crates/stone/src/write.rs +++ b/crates/stone/src/write.rs @@ -16,7 +16,7 @@ pub use self::digest::{StoneDigestWriter, StoneDigestWriterHasher}; pub mod digest; mod zstd; -pub struct StoneWriter { +pub struct StoneWriter { writer: W, content: T, file_type: StoneHeaderV1FileType, From 64eafc2a67b9a46fbf35f025bb4908363894cdba Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Mon, 4 Nov 2024 15:22:10 -0800 Subject: [PATCH 12/12] Remove include subdir --- libstone/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libstone/Cargo.toml b/libstone/Cargo.toml index 33811bf0..3ccafdc1 100644 --- a/libstone/Cargo.toml +++ b/libstone/Cargo.toml @@ -21,7 +21,7 @@ cbindgen.workspace = true [package.metadata.capi.header] name = "stone" -subdirectory = "libstone" +subdirectory = false [package.metadata.capi.pkg_config] name = "libstone"