diff --git a/crates/e2e_test/src/compression_test.rs b/crates/e2e_test/src/compression_test.rs index 1f6ede334..30b1cdce0 100644 --- a/crates/e2e_test/src/compression_test.rs +++ b/crates/e2e_test/src/compression_test.rs @@ -4,7 +4,8 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use std::fs; -use std::path::PathBuf; +use std::io; +use std::path::{Path, PathBuf}; use std::process::Command; use tracing::info; @@ -31,30 +32,58 @@ fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec { .collect() } -fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec { +fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> io::Result> { let bucket_path = PathBuf::from(temp_dir).join(bucket); let mut part_files = Vec::new(); - fn scan_dir(dir: &PathBuf, target: &str, results: &mut Vec) { - if let Ok(entries) = fs::read_dir(dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - scan_dir(&path, target, results); - } else if path - .file_name() - .map(|n| n.to_string_lossy().starts_with("part.")) - .unwrap_or(false) - && path.to_string_lossy().contains(target) - { - results.push(path); + fn scan_dir(dir: &Path, target: &str, results: &mut Vec) -> io::Result<()> { + let entries = fs::read_dir(dir) + .map_err(|error| io::Error::new(error.kind(), format!("failed to read {}: {error}", dir.display())))?; + for entry in entries { + let entry = entry + .map_err(|error| io::Error::new(error.kind(), format!("failed to read entry in {}: {error}", dir.display())))?; + let path = entry.path(); + let file_type = entry + .file_type() + .map_err(|error| io::Error::new(error.kind(), format!("failed to inspect {}: {error}", path.display())))?; + if file_type.is_dir() { + scan_dir(&path, target, results)?; + } else if path + .file_name() + .map(|n| n.to_string_lossy().starts_with("part.")) + .unwrap_or(false) + && path.to_string_lossy().contains(target) + { + if !file_type.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected regular part file at {}", path.display()), + )); } + results.push(path); } } + Ok(()) } - scan_dir(&bucket_path, object_key, &mut part_files); - part_files + scan_dir(&bucket_path, object_key, &mut part_files)?; + Ok(part_files) +} + +fn part_files_total_size(part_files: &[PathBuf]) -> io::Result { + part_files.iter().try_fold(0, |total, path| { + let metadata = fs::symlink_metadata(path) + .map_err(|error| io::Error::new(error.kind(), format!("failed to stat {}: {error}", path.display())))?; + if !metadata.file_type().is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected regular part file at {}", path.display()), + )); + } + total + .checked_add(metadata.len()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "on-disk part size overflow")) + }) } async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Result<(), Box> { @@ -123,8 +152,9 @@ async fn test_compression_roundtrip() -> Result<(), Box Result<(), Box Result<(), // This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x // margin still proves the parts were stored compressed rather than raw or double-encoded. - let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key); + let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key)?; assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object"); - let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + let total_physical_size = part_files_total_size(&part_files)?; assert!( total_physical_size < (total_size as u64) / 2, "Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data" @@ -522,9 +552,9 @@ async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), B "Content-Length should be the logical object size" ); - let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key); + let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key)?; assert!(!part_files.is_empty(), "expected on-disk part files for the copied object"); - let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + let total_physical_size = part_files_total_size(&part_files)?; assert!( total_physical_size < (total_size / 2) as u64, "Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)" @@ -585,9 +615,9 @@ async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), "Content-Length should be the logical object size" ); - let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key); + let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key)?; assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object"); - let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + let total_physical_size = part_files_total_size(&part_files)?; assert!( total_physical_size < (total_size / 2) as u64, "Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)" @@ -734,9 +764,9 @@ async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box