diff --git a/peeko/src/fs/mod.rs b/peeko/src/fs/mod.rs index de93f34..2b3306b 100644 --- a/peeko/src/fs/mod.rs +++ b/peeko/src/fs/mod.rs @@ -2,6 +2,11 @@ use std::fs; use std::io::Result; use std::path::{Path, PathBuf}; +/// Returns a list of image name/tag pairs found under the given OCI directory. +/// +/// The directory is expected to follow the layout created by `docker pull` or +/// this crate's [`RegistryClient`](crate::registry::RegistryClient), where +/// images are stored under `/`. pub fn collect_images>(oci_dir: P) -> Result> { let base_dir = oci_dir.as_ref(); collect_image_directories(base_dir).map(|dirs| { @@ -22,6 +27,8 @@ pub fn collect_images>(oci_dir: P) -> Result> { }) } +/// Recursively walks the given path and collects directories that contain a +/// `manifest.json`, returning their absolute paths. pub fn collect_image_directories>(path: P) -> Result> { let mut result = Vec::new(); let path = path.as_ref(); @@ -51,6 +58,7 @@ fn collect_image_directories_recursive(path: &Path, result: &mut Vec) - Ok(()) } +/// Removes the directory storing the given `image:tag` from the OCI root. pub fn delete_image>(oci_dir: P, image: &str, tag: &str) -> Result<()> { let image_path = oci_dir.as_ref().join(format!("{image}/{tag}")); fs::remove_dir_all(&image_path)?; diff --git a/peeko/src/lib.rs b/peeko/src/lib.rs index 3676a99..7475c7e 100644 --- a/peeko/src/lib.rs +++ b/peeko/src/lib.rs @@ -1,5 +1,16 @@ +//! Core library for interacting with OCI container images that have been +//! downloaded to disk. The crate provides helpers for discovering images on the +//! filesystem, parsing image manifests, reading layer contents, downloading +//! artifacts from registries, and computing simple statistics about virtual +//! filesystems reconstructed from image layers. + +/// Filesystem helpers for working with OCI image layouts stored on disk. pub mod fs; +/// Types that model OCI image manifests and configs. pub mod manifest; +/// Async readers that reconstruct a virtual filesystem view of image layers. pub mod reader; +/// Clients for talking to OCI compatible registries. pub mod registry; +/// Utilities for summarising reconstructed filesystem trees. pub mod stats; diff --git a/peeko/src/manifest.rs b/peeko/src/manifest.rs index 9cb6c12..737a7d3 100644 --- a/peeko/src/manifest.rs +++ b/peeko/src/manifest.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +/// High level representation of OCI manifest documents. #[derive(Debug, Deserialize, Serialize)] #[serde(tag = "mediaType")] pub enum Manifest { @@ -12,64 +13,92 @@ pub enum Manifest { OCIIndex(ManifestList), } +/// Representation of `application/vnd.oci.image.manifest.v1+json`. #[derive(Debug, Deserialize, Serialize)] pub struct ImageManifest { #[serde(rename = "schemaVersion")] + /// Schema version declared by the manifest. pub schema_version: u32, #[serde(rename = "mediaType")] + /// Media type for the manifest. pub media_type: String, + /// Descriptor that points to the configuration blob. pub config: Descriptor, + /// Ordered layer descriptors composing the image. pub layers: Vec, // for oci index #[serde(skip_serializing_if = "Option::is_none")] + /// Optional annotations supplied by the image registry. pub annotations: Option>, } +/// Generic descriptor that points to a blob stored in the registry. #[derive(Debug, Deserialize, Serialize)] pub struct Descriptor { + /// SHA digest of the referenced blob. pub digest: String, #[serde(rename = "mediaType")] + /// Media type of the blob. pub media_type: String, + /// Size in bytes of the blob. pub size: u64, // for oci index #[serde(skip_serializing_if = "Option::is_none")] + /// Additional annotations provided by the registry. pub annotations: Option>, } +/// Representation of `application/vnd.oci.image.index.v1+json`. #[derive(Debug, Deserialize, Serialize)] pub struct ManifestList { #[serde(rename = "schemaVersion")] + /// Schema version declared by the manifest. pub schema_version: u32, #[serde(rename = "mediaType")] + /// Media type for the manifest list. pub media_type: String, + /// Architectures and platforms included in the manifest list. pub manifests: Vec, } +/// Descriptor of a single platform entry inside an OCI index. #[derive(Debug, Deserialize, Serialize)] pub struct PlatformManifest { + /// SHA digest of the platform-specific manifest. pub digest: String, #[serde(rename = "mediaType")] + /// Media type for the manifest. pub media_type: String, + /// Target platform described by this manifest. pub platform: Platform, + /// Size in bytes of the manifest blob. pub size: u64, #[serde(skip_serializing_if = "Option::is_none")] + /// Optional annotations supplied by the registry. pub annotations: Option>, } +/// Platform information attached to a platform manifest descriptor. #[derive(Debug, Deserialize, Serialize)] pub struct Platform { + /// CPU architecture (for example `amd64` or `arm64`). pub architecture: String, + /// Operating system (for example `linux`). pub os: String, #[serde(rename = "os.version", skip_serializing_if = "Option::is_none")] + /// Optional OS version. pub os_version: Option, #[serde(rename = "os.features", skip_serializing_if = "Option::is_none")] + /// Optional OS feature list. pub os_features: Option>, #[serde(skip_serializing_if = "Option::is_none")] + /// CPU variant (for example `v7`). pub variant: Option, } +/// Returns the file extension associated with a descriptor's media type. pub fn get_file_type(media_type: &str) -> &str { match media_type.rsplit_once('+') { Some((_, ext)) => ext, @@ -80,67 +109,94 @@ pub fn get_file_type(media_type: &str) -> &str { } } +/// Runtime configuration extracted from an image config blob. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImageConfig { + /// CPU architecture (for example `amd64` or `arm64`). pub architecture: String, + /// Operating system (for example `linux`). pub os: String, + /// Container runtime settings. pub config: ContainerConfig, + /// Timestamp when the image was created. pub created: String, + /// History describing how the image layers were produced. pub history: Vec, + /// Root filesystem diff IDs. pub rootfs: RootFs, } +/// Container runtime options section inside an image config. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ContainerConfig { #[serde(rename = "Hostname")] + /// Default hostname assigned to containers started from the image. pub hostname: Option, #[serde(rename = "User")] + /// Default user (UID/GID) the container should run as. pub user: Option, #[serde(rename = "Env")] + /// Default environment variables. pub env: Option>, #[serde(rename = "Cmd")] + /// Default command executed by the container runtime. pub cmd: Option>, #[serde(rename = "Entrypoint")] + /// Entrypoint process invoked before `Cmd`. pub entrypoint: Option>, #[serde(rename = "WorkingDir")] + /// Working directory for the default command. pub working_dir: Option, #[serde(rename = "Labels")] + /// Image labels attached to the runtime config. pub labels: Option>, #[serde(rename = "ExposedPorts")] + /// Ports exposed by default. pub exposed_ports: Option>, #[serde(rename = "Volumes")] + /// Named volumes declared by the image. pub volumes: Option>, #[serde(rename = "StopSignal")] + /// Signal used to request graceful shutdown. pub stop_signal: Option, #[serde(rename = "Shell")] + /// Default shell used for command interpretation. pub shell: Option>, } +/// Detailed history line for how an image layer was produced. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HistoryEntry { + /// Timestamp when the layer was created. pub created: String, + /// Command that produced the layer. pub created_by: String, #[serde(default)] + /// Whether the entry represents an empty layer. pub empty_layer: bool, #[serde(default)] + /// Optional comment attached to the history entry. pub comment: Option, } +/// Root filesystem metadata inside an image config. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RootFs { #[serde(rename = "type")] + /// Type of filesystem (typically `layers`). pub fs_type: String, + /// Digest list representing layer diff IDs. pub diff_ids: Vec, } diff --git a/peeko/src/reader/dir_tree.rs b/peeko/src/reader/dir_tree.rs index 8dd7dd7..a89f8a9 100644 --- a/peeko/src/reader/dir_tree.rs +++ b/peeko/src/reader/dir_tree.rs @@ -1,8 +1,11 @@ +//! Utilities that model directory trees generated from OCI layers. + use std::cell::RefCell; use std::collections::HashMap; use std::path::Path; use std::rc::{Rc, Weak}; +/// Node within a directory tree backed by reference-counted pointers. #[derive(Debug)] pub struct TreeNode { pub name: String, @@ -12,6 +15,7 @@ pub struct TreeNode { } impl TreeNode { + /// Returns the full path for the node, optionally including the virtual root. pub fn pwd(&self, with_root: bool) -> String { let mut components = vec![self.name.clone()]; let mut current = self.parent.borrow().upgrade(); @@ -32,6 +36,7 @@ impl TreeNode { } } + /// Prints this node and its descendants up to `max_depth`. pub fn print(&self, depth: usize, max_depth: usize, is_last: bool, prefix: &str) { let new_prefix = if depth == 0 { println!("{}", &self.name); @@ -64,12 +69,14 @@ impl TreeNode { } } +/// Directory tree built from entries in a [`VirtualFileSystem`](super::vfs::VirtualFileSystem). #[derive(Debug)] pub struct DirectoryTree { pub root: Rc, } impl DirectoryTree { + /// Creates an empty directory tree with a root node named `/`. pub fn new() -> Self { Self { root: Rc::new(TreeNode { @@ -81,6 +88,7 @@ impl DirectoryTree { } } + /// Inserts a new path into the tree, creating intermediate directories as needed. pub fn add_path>(&self, path: P, is_dir: bool) { let mut components: Vec<_> = path .as_ref() @@ -119,6 +127,7 @@ impl DirectoryTree { } } + /// Finds a node inside the tree by path returning a shared pointer to it. pub fn find(&self, path: &str) -> Option> { if path.eq("/") { return Some(Rc::clone(&self.root)); @@ -144,6 +153,7 @@ impl DirectoryTree { Some(current) } + /// Prints the entire tree to stdout up to `max_depth`. pub fn print(&self, max_depth: usize) { self.root.print(0, max_depth, true, ""); } diff --git a/peeko/src/reader/image_reader.rs b/peeko/src/reader/image_reader.rs index 117cdc5..34355ce 100644 --- a/peeko/src/reader/image_reader.rs +++ b/peeko/src/reader/image_reader.rs @@ -9,6 +9,7 @@ use super::dir_tree::DirectoryTree; use super::vfs::{FileEntry, VirtualFileSystem}; use crate::manifest::{ImageManifest, get_file_type}; +/// Errors produced when building or using the asynchronous image reader. #[derive(Error, Debug)] pub enum ImageReaderError { #[error("HTTP error: {0}")] @@ -30,6 +31,7 @@ pub enum ImageReaderError { NotAFile(String), } +/// Convenient result alias that uses [`ImageReaderError`]. pub type Result = std::result::Result; async fn load_manifest>(image_dir: P) -> Result { @@ -140,6 +142,11 @@ async fn read_file_from_layer, FP: AsRef>( } } +/// Constructs an `ImageReader` from an OCI image directory on disk. +/// +/// The directory must contain a `manifest.json` and all layer blobs (named +/// `.`). Layers are replayed in order to build the virtual +/// filesystem that powers the reader. pub async fn build_image_reader>(image_dir: P) -> Result { let image_dir = image_dir.as_ref(); let manifest = load_manifest(image_dir).await?; @@ -158,6 +165,7 @@ pub async fn build_image_reader>(image_dir: P) -> Result>(&self, path: P) -> Result> { let path = path.as_ref(); let entry = self @@ -186,11 +197,16 @@ impl ImageReader { } } + /// Builds an in-memory directory tree representing all files and + /// directories contained in the image. pub fn get_dir_tree(&self) -> Result { let tree = self.vfs.get_directory_tree(); Ok(tree) } + /// Prints the directory tree to stdout with an optional depth filter. + /// + /// When `path` is supplied the tree is rooted at that subdirectory. pub fn print_dir_tree(&self, depth: usize, path: Option) -> Result<()> { let tree = self.get_dir_tree()?; let target_node = match &path { @@ -209,6 +225,7 @@ impl ImageReader { } } + /// Returns metadata associated with a path in the virtual filesystem. pub fn get_file_meatadata(&self, path: &str) -> Option<&FileEntry> { self.vfs.get_entry(PathBuf::from(path)) } diff --git a/peeko/src/reader/mod.rs b/peeko/src/reader/mod.rs index c35f907..e1e751d 100644 --- a/peeko/src/reader/mod.rs +++ b/peeko/src/reader/mod.rs @@ -1,7 +1,11 @@ +//! Helpers for reconstructing filesystem content from OCI image layers. + mod archive_utils; mod dir_tree; mod image_reader; pub mod vfs; +/// Error type returned by the asynchronous image reader. pub use image_reader::ImageReaderError; +/// Build a high level image reader from an unpacked OCI image directory. pub use image_reader::build_image_reader; diff --git a/peeko/src/reader/vfs.rs b/peeko/src/reader/vfs.rs index 804485d..4cd9a1a 100644 --- a/peeko/src/reader/vfs.rs +++ b/peeko/src/reader/vfs.rs @@ -1,40 +1,52 @@ +//! Lightweight virtual filesystem for materialising file listings in OCI images. + use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; use super::dir_tree::DirectoryTree; +/// Metadata recorded for each entry tracked by the virtual filesystem. #[derive(Debug, Clone)] pub enum FileEntry { + /// Regular file along with its size and layer index. File { size: u64, layer_index: usize }, + /// Directory created in the given layer. Directory { layer_index: usize }, + /// Symbolic link pointing at `target`. Symlink { target: String, layer_index: usize }, } +/// In-memory index of filesystem entries extracted from image layers. pub struct VirtualFileSystem { // 路径 -> 文件条目 entries: HashMap, } impl VirtualFileSystem { + /// Creates an empty virtual filesystem. pub fn new() -> Self { Self { entries: HashMap::new(), } } + /// Inserts or replaces the entry stored at `path`. pub fn add_entry(&mut self, path: PathBuf, entry: FileEntry) { self.entries.insert(path, entry); } + /// Returns the metadata for a given path if it exists. pub fn get_entry>(&self, path: P) -> Option<&FileEntry> { self.entries.get(path.as_ref()) } + /// Deletes the entry at `path`. pub fn delete_entry(&mut self, path: &PathBuf) { self.entries.remove(path); } + /// Removes all entries contained inside the directory `dir`. pub fn clear_directory(&mut self, dir: &Path) { let dir_str = dir.to_string_lossy(); let dir_prefix = format!("{dir_str}/"); @@ -42,10 +54,12 @@ impl VirtualFileSystem { .retain(|path, _| !path.to_string_lossy().starts_with(&dir_prefix)); } + /// Returns a view of the raw entry map. pub fn get_entries(&self) -> &HashMap { &self.entries } + /// Builds a `DirectoryTree` covering all tracked paths. pub fn get_directory_tree(&self) -> DirectoryTree { let tree = DirectoryTree::new(); diff --git a/peeko/src/registry/client.rs b/peeko/src/registry/client.rs index 7aea31b..d554fb3 100644 --- a/peeko/src/registry/client.rs +++ b/peeko/src/registry/client.rs @@ -11,6 +11,7 @@ use tokio::io::AsyncWriteExt; use super::progress::{NoopProgress, ProgressTracker}; use crate::manifest::{self, Descriptor, Manifest, ManifestList, PlatformManifest}; +/// Failures raised while communicating with the remote registry or filesystem. #[derive(Error, Debug)] pub enum RegistryError { #[error("Header not found: {0}")] @@ -41,6 +42,7 @@ pub enum RegistryError { IoError(#[from] std::io::Error), } +/// Convenient result alias that uses [`RegistryError`]. pub type Result = std::result::Result; #[derive(Debug, Deserialize, Serialize)] @@ -50,15 +52,20 @@ struct TokenResponse { pub expires_in: Option, } +/// Optional filters used to pick a specific platform when downloading multi-arch images. pub struct PlatformParam { + /// Specific CPU architecture to fetch. pub architecture: Option, + /// Specific operating system to fetch. pub os: Option, + /// CPU variant (for example `arm/v7`) to fetch. pub variant: Option, } const DEFAULT_REGISTRY: &str = "https://registry-1.docker.io"; const DEFAULT_CONCURRENT_DOWNLOADS: usize = 3; +/// High level client for retrieving manifests and blobs from an OCI registry. #[derive(Clone)] pub struct RegistryClient { http: reqwest::Client, @@ -87,6 +94,7 @@ impl Default for RegistryClient { } impl RegistryClient { + /// Creates a client targeting the provided registry URL. pub fn new(registry_url: &str) -> Self { Self { registry_url: registry_url.to_string(), @@ -94,6 +102,7 @@ impl RegistryClient { } } + /// Creates a client configured with basic-auth credentials. pub fn with_credentials(registry_url: &str, username: &str, password: &str) -> Self { Self { registry_url: registry_url.to_string(), @@ -103,6 +112,7 @@ impl RegistryClient { } } + /// Creates a client configured with a pre-baked bearer token. pub fn with_token(registry_url: &str, token: &str) -> Self { Self { registry_url: registry_url.to_string(), @@ -111,15 +121,18 @@ impl RegistryClient { } } + /// Sets the directory where downloaded images are written to disk. pub fn set_downloads_dir>(&mut self, dir: P) { self.oci_dir = dir.into(); } + /// Limits the number of concurrent blob downloads. pub fn set_concurrent_downloads(&mut self, concurrent: usize) { self.concurrent_downloads = concurrent; } #[cfg(feature = "progress")] + /// Enables progress reporting using the `indicatif` progress bars. pub fn enable_progress(mut self) -> Self { self.progress = Arc::new(super::progress::IndicatifProgress::new()); self @@ -205,6 +218,7 @@ impl RegistryClient { request } + /// Fetches the manifest (or manifest list) for the specified image reference. pub async fn get_image_manifest( &mut self, image: &str, @@ -248,6 +262,10 @@ impl RegistryClient { } } + /// Downloads an image and all of its layers into the configured downloads directory. + /// + /// When the manifest resolves to a multi-platform index the `platform` + /// parameter filters which architecture to download. pub async fn download_image( &mut self, image: &str, diff --git a/peeko/src/registry/mod.rs b/peeko/src/registry/mod.rs index 463c1d3..8cd0b05 100644 --- a/peeko/src/registry/mod.rs +++ b/peeko/src/registry/mod.rs @@ -1,4 +1,7 @@ +//! OCI registry client capable of fetching manifests and downloading layers. + pub mod client; pub mod progress; +/// Re-export of the high level registry client. pub use client::{PlatformParam, RegistryClient, RegistryError}; diff --git a/peeko/src/registry/progress.rs b/peeko/src/registry/progress.rs index 40f08bf..a6d3713 100644 --- a/peeko/src/registry/progress.rs +++ b/peeko/src/registry/progress.rs @@ -1,14 +1,16 @@ +//! Progress reporting abstraction used when downloading blobs from the registry. + #[cfg(feature = "progress")] use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -// Progress trait abstraction +/// Trait implemented by download progress reporters. pub trait ProgressTracker: Send + Sync { fn start_download(&self, digest: &str, total_bytes: u64); fn update(&self, digest: &str, bytes: u64); fn finish(&self, digest: &str); } -// No-op implementation +/// No-op progress tracker used when no reporting is required. pub struct NoopProgress; impl ProgressTracker for NoopProgress { @@ -17,7 +19,8 @@ impl ProgressTracker for NoopProgress { fn finish(&self, _digest: &str) {} } -// Indicatif implementation (only when feature enabled) +/// Progress tracker backed by `indicatif` progress bars (only available when the +/// `progress` feature is enabled). #[cfg(feature = "progress")] pub struct IndicatifProgress { multi: MultiProgress, @@ -26,6 +29,7 @@ pub struct IndicatifProgress { #[cfg(feature = "progress")] impl IndicatifProgress { + /// Creates a new progress reporter wired to a `MultiProgress` manager. pub fn new() -> Self { Self { multi: MultiProgress::new(), diff --git a/peeko/src/stats/mod.rs b/peeko/src/stats/mod.rs index 5fd2e8e..d0c90e0 100644 --- a/peeko/src/stats/mod.rs +++ b/peeko/src/stats/mod.rs @@ -1,5 +1,8 @@ +//! Helpers for printing summary information about reconstructed filesystems. + use crate::reader::vfs::{FileEntry, VirtualFileSystem}; +/// Prints aggregate counts of files, directories, symlinks and total size. pub fn show_statistics(vfs: &VirtualFileSystem) { let entries = vfs.get_entries(); @@ -29,6 +32,7 @@ pub fn show_statistics(vfs: &VirtualFileSystem) { ); } +/// Lists top-level entries (paths with a single component) in the virtual filesystem. pub fn list_top_level(vfs: &VirtualFileSystem) { println!("\n=== Top-level Entries ===");