diff --git a/CHANGELOG.md b/CHANGELOG.md index 869bdae1f..51cb87145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *When editing this file, please respect a line length of 100.* +## [0.36.2] - 2026-10-13 + +### Added +- `DataMapIndex` gives random access to a `DataMap`'s content with `u64` positions on every + platform, including 32-bit targets such as `wasm32`. It does no I/O: it lists the chunks + overlapping a byte range and decrypts single chunks, checking each against the size and source + hash the map declares. Decompression stops at the declared size, so crafted chunks cannot expand + beyond it. + +### Changed +- `DecryptionStream` is built on `DataMapIndex`. Random access uses `u64` positions and binary + searches instead of `usize` sums and linear scans, so it no longer overflows on 32-bit targets, + and every decrypted chunk is checked against its declared size and hash. Construction fails for + a map whose chunk indices are not exactly `0..n`. +- On 32-bit targets, `DecryptionStream::file_size` reports `usize::MAX` for content larger than + `usize::MAX` bytes; `DataMapIndex::file_size` gives the exact size. + +### Fixed +- `DecryptionStream` fails when the fetch callback omits a chunk. Before, `get_range` joined the + remaining chunks into wrong content, and iteration ended early as if the content were complete. + ## [0.36.1] - 2026-10-06 ### Changed diff --git a/Cargo.toml b/Cargo.toml index ad24aed2d..dd47c61e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ license = "MIT OR Apache-2.0" name = "self_encryption" readme = "README.md" repository = "https://github.com/WithAutonomi/self_encryption" -version = "0.36.1" +version = "0.36.2" [features] default = [] diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 1b646bf30..f9661791e 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["cdylib"] hex = "0.4.3" napi = { version = "2.12.2", default-features = false, features = ["napi4", "napi6", "tokio_rt", "serde-json"] } napi-derive = "2.12.2" -self_encryption = { version = "0.36.1", path = ".." } +self_encryption = { version = "0.36.2", path = ".." } [build-dependencies] napi-build = "2.0.1" diff --git a/src/data_map_index.rs b/src/data_map_index.rs new file mode 100644 index 000000000..fa7946c0e --- /dev/null +++ b/src/data_map_index.rs @@ -0,0 +1,356 @@ +// Copyright 2026 MaidSafe.net limited. +// +// This Autonomi Software is licensed under the MIT license or the Apache License, Version 2.0 +// , at your +// option. This file may not be copied, modified, or distributed except +// according to those terms. + +//! Random access to the content of a `DataMap` with 64-bit positions. + +use crate::{ + decrypt::decrypt_chunk_bounded, hash::content_hash, ChunkInfo, DataMap, Error, Result, +}; +use bytes::Bytes; +use std::ops::Range; +use xor_name::XorName; + +/// Random access to the content of a [`DataMap`], with 64-bit positions on +/// every platform. +/// +/// The index performs no I/O. Callers fetch the encrypted chunks listed by +/// [`Self::infos`] however suits them, for example concurrently and +/// asynchronously, and pass each one to [`Self::decrypt_chunk`]. Positions are +/// `u64`, so content larger than `usize::MAX` bytes stays addressable on +/// 32-bit targets such as `wasm32`. +/// +/// The content of a shrunk (child) map is its serialized parent map. Resolve +/// it with [`crate::get_root_data_map`] first to index the original data. +/// +/// # Example +/// +/// ```rust +/// use self_encryption::{encrypt, test_helpers::random_bytes, DataMapIndex}; +/// +/// # fn main() -> self_encryption::Result<()> { +/// let original = random_bytes(10_000); +/// let (data_map, chunks) = encrypt(original.clone())?; +/// let index = DataMapIndex::new(&data_map)?; +/// +/// // Decrypt only the chunks holding bytes 4000..6000. +/// let (start, end) = (4000u64, 6000u64); +/// let mut range = Vec::new(); +/// for chunk in index.chunks_overlapping(start..end) { +/// // Here the chunks come straight from `encrypt`, in index order. A real +/// // caller fetches `index.infos()[chunk].dst_hash` from storage. +/// let plaintext = index.decrypt_chunk(chunk, &chunks[chunk].content)?; +/// let span = index.chunk_range(chunk); +/// let from = (start.max(span.start) - span.start) as usize; +/// let to = (end.min(span.end) - span.start) as usize; +/// range.extend_from_slice(&plaintext[from..to]); +/// } +/// assert_eq!(&range[..], &original[4000..6000]); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct DataMapIndex { + /// The map's chunks, where `infos[i].index == i`. + infos: Vec, + /// Pre-encryption hashes in chunk order. Every chunk's keys derive from + /// its own and its neighbours' hashes. + src_hashes: Vec, + /// Content offset of each chunk, followed by the content size. + offsets: Vec, + /// Key-derivation level the map's chunks were encrypted at. + child_level: usize, +} + +impl DataMapIndex { + /// Index the chunks of `data_map`. + /// + /// # Errors + /// + /// Fails if the chunk indices are not exactly `0..n`, or if the content + /// size overflows `u64`. + pub fn new(data_map: &DataMap) -> Result { + let mut infos = data_map.infos().to_vec(); + infos.sort_by_key(|info| info.index); + let mut offsets = Vec::with_capacity(infos.len() + 1); + let mut end = 0u64; + offsets.push(end); + for (position, info) in infos.iter().enumerate() { + if info.index != position { + return Err(Error::Generic(format!( + "DataMap chunk indices must be contiguous from 0; found {} at position {position}", + info.index + ))); + } + end = end + .checked_add(info.src_size as u64) + .ok_or_else(|| Error::Generic("DataMap content size overflows u64".to_string()))?; + offsets.push(end); + } + Ok(Self { + src_hashes: infos.iter().map(|info| info.src_hash).collect(), + infos, + offsets, + child_level: data_map.child().unwrap_or(0), + }) + } + + /// Size of the content in bytes. + pub fn file_size(&self) -> u64 { + self.offsets.last().copied().unwrap_or_default() + } + + /// The map's chunks in index order: `infos()[i].index == i`. + pub fn infos(&self) -> &[ChunkInfo] { + &self.infos + } + + /// Half-open byte range of the content held by chunk `index`. + /// + /// # Panics + /// + /// Panics if `index` is not below `infos().len()`. + pub fn chunk_range(&self, index: usize) -> Range { + self.offsets[index]..self.offsets[index + 1] + } + + /// Indices of the chunks overlapping the half-open byte range `range`, + /// which is clamped at the end of the content. Empty when nothing of the + /// content lies inside `range`. + pub fn chunks_overlapping(&self, range: Range) -> Range { + let end = range.end.min(self.file_size()); + if range.start >= end { + return 0..0; + } + // `offsets[0]` is 0, so at least one offset is at or before the start. + let first = self + .offsets + .partition_point(|offset| *offset <= range.start) + - 1; + let last = self.offsets.partition_point(|offset| *offset < end); + first..last + } + + /// Decrypt chunk `index` from its encrypted `content`, checking that the + /// result has the size and hash the map declares. + /// + /// Decompression stops at the declared size, so crafted content cannot + /// expand beyond what the map states. + /// + /// # Errors + /// + /// Fails if `index` is out of range, if `content` does not decrypt with + /// the chunk's keys, or if the result differs from the declared size or + /// hash. + pub fn decrypt_chunk(&self, index: usize, content: &Bytes) -> Result { + let info = self.infos.get(index).ok_or_else(|| { + Error::Generic(format!( + "chunk index {index} out of bounds for {} chunks", + self.infos.len() + )) + })?; + let plaintext = decrypt_chunk_bounded( + index, + content, + &self.src_hashes, + self.child_level, + info.src_size, + )?; + if plaintext.len() != info.src_size { + return Err(Error::Generic(format!( + "chunk {index} decrypted to {} bytes, but the DataMap declares {}", + plaintext.len(), + info.src_size + ))); + } + if content_hash(&plaintext) != info.src_hash { + return Err(Error::Generic(format!( + "chunk {index} does not match its DataMap source hash" + ))); + } + Ok(plaintext) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Content of a small map: three chunks, the fewest `encrypt` produces. + const SMALL_CONTENT_BYTES: usize = 10_000; + /// Chunk size for indexes larger than 32-bit `usize` can address. + const LARGE_CHUNK_BYTES: usize = 1 << 30; + /// Chunks in the large index: 6 GiB of content. + const LARGE_CHUNKS: usize = 6; + /// Offset that 32-bit `usize` arithmetic cannot represent. + const FOUR_GIB: u64 = 1 << 32; + + fn small_map() -> Result<(Bytes, DataMap, Vec)> { + let content = crate::test_helpers::random_bytes(SMALL_CONTENT_BYTES); + let (map, chunks) = crate::encrypt(content.clone())?; + let mut by_index = vec![Bytes::new(); map.len()]; + for info in map.infos() { + by_index[info.index] = chunks + .iter() + .find(|chunk| content_hash(&chunk.content) == info.dst_hash) + .map(|chunk| chunk.content.clone()) + .expect("encrypt returns every chunk of its map"); + } + Ok((content, map, by_index)) + } + + fn large_map() -> DataMap { + DataMap::new( + (0..LARGE_CHUNKS) + .map(|index| ChunkInfo { + index, + dst_hash: content_hash(&index.to_le_bytes()), + src_hash: content_hash(&index.to_be_bytes()), + src_size: LARGE_CHUNK_BYTES, + }) + .collect(), + ) + } + + #[test] + fn decrypted_chunks_reassemble_the_content() -> Result<()> { + let (content, map, chunks) = small_map()?; + let index = DataMapIndex::new(&map)?; + assert_eq!(index.file_size(), content.len() as u64); + let mut reassembled = Vec::new(); + for (chunk, encrypted) in chunks.iter().enumerate() { + let plaintext = index.decrypt_chunk(chunk, encrypted)?; + assert_eq!(plaintext.len() as u64, { + let span = index.chunk_range(chunk); + span.end - span.start + }); + reassembled.extend_from_slice(&plaintext); + } + assert_eq!(reassembled, content.to_vec()); + Ok(()) + } + + #[test] + fn overlapping_chunks_follow_half_open_ranges() -> Result<()> { + let (content, map, _) = small_map()?; + let index = DataMapIndex::new(&map)?; + let size = content.len() as u64; + let boundary = index.chunk_range(1).start; + assert_eq!(index.chunks_overlapping(0..1), 0..1); + assert_eq!(index.chunks_overlapping(boundary - 1..boundary), 0..1); + assert_eq!(index.chunks_overlapping(boundary - 1..boundary + 1), 0..2); + assert_eq!(index.chunks_overlapping(boundary..boundary + 1), 1..2); + assert_eq!(index.chunks_overlapping(0..u64::MAX), 0..3); + assert_eq!(index.chunks_overlapping(size - 1..u64::MAX), 2..3); + assert_eq!(index.chunks_overlapping(size..size + 1), 0..0); + assert_eq!(index.chunks_overlapping(5..5), 0..0); + Ok(()) + } + + #[test] + fn positions_beyond_four_gib_need_no_wide_usize() -> Result<()> { + let index = DataMapIndex::new(&large_map())?; + let chunk_bytes = LARGE_CHUNK_BYTES as u64; + assert_eq!(index.file_size(), LARGE_CHUNKS as u64 * chunk_bytes); + let at_four_gib = (FOUR_GIB / chunk_bytes) as usize; + assert_eq!( + index.chunks_overlapping(FOUR_GIB - 1..FOUR_GIB + 1), + at_four_gib - 1..at_four_gib + 1 + ); + let last = LARGE_CHUNKS - 1; + let end = index.file_size(); + assert_eq!(index.chunk_range(last), end - chunk_bytes..end); + assert_eq!(index.chunks_overlapping(end - 1..end), last..LARGE_CHUNKS); + Ok(()) + } + + #[test] + fn positions_map_by_the_maps_own_chunk_sizes() -> Result<()> { + // Reported underflow: a ~15.27 GB file chunked with twice this build's + // MAX_CHUNK_SIZE, read from 4 MiB. + let file_size = 16_404_310_194u64 as usize; + let start = 4_194_304u64; + let max_chunk_size = crate::MAX_CHUNK_SIZE * 2; + let chunks = crate::utils::get_num_chunks_with_variable_max(file_size, max_chunk_size); + let map = DataMap::new( + (0..chunks) + .map(|index| ChunkInfo { + index, + src_size: crate::utils::get_chunk_size_with_variable_max( + file_size, + index, + max_chunk_size, + ), + ..ChunkInfo::default() + }) + .collect(), + ); + let index = DataMapIndex::new(&map)?; + assert_eq!(index.file_size(), file_size as u64); + let chunk = index.chunks_overlapping(start..start + 1).start; + assert!(index.chunk_range(chunk).contains(&start)); + Ok(()) + } + + #[test] + fn indices_must_run_from_zero_without_gaps_or_repeats() { + for indices in [[0, 1, 3], [0, 0, 1], [1, 2, 3]] { + let map = DataMap::new( + indices + .iter() + .map(|&index| ChunkInfo { + index, + src_size: 1, + ..ChunkInfo::default() + }) + .collect(), + ); + assert!(DataMapIndex::new(&map).is_err(), "accepted {:?}", indices); + } + } + + #[test] + fn content_must_have_the_declared_size() -> Result<()> { + let (_, map, chunks) = small_map()?; + for delta in [-1isize, 1] { + let mut infos = map.infos().to_vec(); + infos[0].src_size = (infos[0].src_size as isize + delta) as usize; + let index = DataMapIndex::new(&DataMap::new(infos))?; + assert!( + index.decrypt_chunk(0, &chunks[0]).is_err(), + "size delta {}", + delta + ); + } + Ok(()) + } + + #[test] + fn content_must_have_the_declared_hash() -> Result<()> { + let (content, map, _) = small_map()?; + // A crafted map whose first chunk declares the wrong source hash, with + // every chunk encrypted under keys derived from the declared hashes. + let mut infos = map.infos().to_vec(); + infos[0].src_hash = content_hash(b"not the first chunk"); + let src_hashes: Vec<_> = infos.iter().map(|info| info.src_hash).collect(); + let index = DataMapIndex::new(&DataMap::new(infos))?; + let span = index.chunk_range(0); + let plaintext = content.slice(span.start as usize..span.end as usize); + let keys = crate::utils::get_pad_key_and_nonce(0, &src_hashes, 0)?; + let encrypted = crate::encrypt::encrypt_chunk(plaintext, keys)?; + assert!(index.decrypt_chunk(0, &encrypted).is_err()); + Ok(()) + } + + #[test] + fn chunk_index_must_be_in_range() -> Result<()> { + let (_, map, chunks) = small_map()?; + let index = DataMapIndex::new(&map)?; + assert!(index.decrypt_chunk(chunks.len(), &chunks[0]).is_err()); + Ok(()) + } +} diff --git a/src/decrypt.rs b/src/decrypt.rs index 9f22b341a..b2e5c5efd 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -7,11 +7,15 @@ // according to those terms. use crate::{cipher, utils::get_pad_key_and_nonce, utils::xor, EncryptedChunk, Error, Result}; +use brotli::Decompressor; use bytes::Bytes; use rayon::prelude::*; -use std::io::Cursor; +use std::io::{Cursor, Read}; use xor_name::XorName; +/// Internal buffer of the streaming decompressor, as `brotli::BrotliDecompress` uses. +const DECOMPRESS_BUFFER_BYTES: usize = 4096; + // encrypted_Chunks are sorted !! pub fn decrypt_sorted_set( src_hashes: Vec, @@ -43,6 +47,18 @@ pub fn decrypt_chunk( content: &Bytes, src_hashes: &[XorName], child_level: usize, +) -> Result { + decrypt_chunk_bounded(chunk_index, content, src_hashes, child_level, usize::MAX) +} + +/// As [`decrypt_chunk`], but fails rather than decompress more than +/// `max_size` bytes. +pub(crate) fn decrypt_chunk_bounded( + chunk_index: usize, + content: &Bytes, + src_hashes: &[XorName], + child_level: usize, + max_size: usize, ) -> Result { let pki = get_pad_key_and_nonce(chunk_index, src_hashes, child_level)?; let (pad, key, nonce) = pki; @@ -53,11 +69,17 @@ pub fn decrypt_chunk( // Then decrypt the content let decrypted = cipher::decrypt(xored, &key, &nonce)?; - // Finally decompress + // Finally decompress, reading one byte past the bound to detect an overrun let mut decompressed = Vec::new(); - let mut cursor = Cursor::new(&decrypted); - - brotli::BrotliDecompress(&mut cursor, &mut decompressed).map_err(|_| Error::Compression)?; + let _ = Decompressor::new(Cursor::new(&decrypted), DECOMPRESS_BUFFER_BYTES) + .take((max_size as u64).saturating_add(1)) + .read_to_end(&mut decompressed) + .map_err(|_| Error::Compression)?; + if decompressed.len() > max_size { + return Err(Error::Generic(format!( + "chunk {chunk_index} decompresses to more than {max_size} bytes" + ))); + } Ok(Bytes::from(decompressed)) } diff --git a/src/lib.rs b/src/lib.rs index a8419c513..fb8bf9626 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,7 @@ mod chunk; mod cipher; mod data_map; +mod data_map_index; mod decrypt; mod encrypt; mod error; @@ -110,6 +111,7 @@ pub use xor_name::XorName; pub use self::{ data_map::{ChunkInfo, DataMap}, + data_map_index::DataMapIndex, error::{Error, Result}, stream_decrypt::{streaming_decrypt, streaming_decrypt_with_batch_size, DecryptionStream}, stream_encrypt::{stream_encrypt, ChunkStream, EncryptionStream}, diff --git a/src/stream_decrypt.rs b/src/stream_decrypt.rs index 2d31b319a..5fcb08445 100644 --- a/src/stream_decrypt.rs +++ b/src/stream_decrypt.rs @@ -9,11 +9,11 @@ //! Streaming decryption functionality for memory-efficient processing of large encrypted files. use crate::{ - decrypt::decrypt_chunk, get_root_data_map_parallel, stream_decrypt_batch_size, - utils::extract_hashes, ChunkInfo, DataMap, Result, + get_root_data_map_parallel, stream_decrypt_batch_size, DataMap, DataMapIndex, Error, Result, }; use bytes::Bytes; use std::collections::HashMap; +use std::convert::TryFrom; use std::ops::Range; use xor_name::XorName; @@ -29,9 +29,7 @@ use xor_name::XorName; /// to any byte range within the encrypted file using methods like `get_range()`, /// `range()`, and other convenience methods. pub struct DecryptionStream { - chunk_infos: Vec, - src_hashes: Vec, - child_level: usize, + index: DataMapIndex, get_chunk_parallel: F, current_batch_start: usize, current_batch_chunks: Vec, @@ -83,15 +81,8 @@ where data_map.clone() }; - let child_level = root_map.child().unwrap_or(0); - let mut chunk_infos = root_map.infos().to_vec(); - chunk_infos.sort_by_key(|info| info.index); - let src_hashes = extract_hashes(&root_map); - Ok(Self { - chunk_infos, - src_hashes, - child_level, + index: DataMapIndex::new(&root_map)?, get_chunk_parallel, current_batch_start: 0, current_batch_chunks: Vec::new(), @@ -102,23 +93,23 @@ where /// Fetches and decrypts the next batch of chunks. fn fetch_next_batch(&mut self) -> Result { - if self.current_batch_start >= self.chunk_infos.len() { + let chunk_infos = self.index.infos(); + if self.current_batch_start >= chunk_infos.len() { return Ok(false); // No more chunks } let batch_end = self .current_batch_start .saturating_add(self.batch_size) - .min(self.chunk_infos.len()); - let batch_infos = self - .chunk_infos + .min(chunk_infos.len()); + let batch_infos = chunk_infos .get(self.current_batch_start..batch_end) .ok_or_else(|| { - crate::Error::Generic(format!( + Error::Generic(format!( "batch range {}..{} out of bounds for chunk_infos (len {})", self.current_batch_start, batch_end, - self.chunk_infos.len() + chunk_infos.len() )) })?; @@ -128,33 +119,36 @@ where .map(|info| (info.index, info.dst_hash)) .collect(); - // Fetch chunks in parallel - let mut fetched_chunks = (self.get_chunk_parallel)(&batch_hashes)?; - fetched_chunks.sort_by_key(|(index, _content)| *index); - - // Decrypt each chunk and store the results - self.current_batch_chunks.clear(); - for (info, (_index, encrypted_content)) in batch_infos.iter().zip(fetched_chunks) { - let decrypted_chunk = decrypt_chunk( - info.index, - &encrypted_content, - &self.src_hashes, - self.child_level, - )?; - self.current_batch_chunks.push(decrypted_chunk); - } - + self.current_batch_chunks = self.fetch_and_decrypt(&batch_hashes)?; self.current_batch_start = batch_end; self.current_batch_index = 0; Ok(true) } + /// Fetches one batch and decrypts it in batch order, failing if the + /// callback omits a requested chunk. + fn fetch_and_decrypt(&self, batch_hashes: &[(usize, XorName)]) -> Result> { + let mut fetched_chunks: HashMap = (self.get_chunk_parallel)(batch_hashes)? + .into_iter() + .collect(); + batch_hashes + .iter() + .map(|(chunk_index, _)| { + let encrypted_content = fetched_chunks.remove(chunk_index).ok_or_else(|| { + Error::Generic(format!("chunk {chunk_index} missing from fetched chunks")) + })?; + self.index.decrypt_chunk(*chunk_index, &encrypted_content) + }) + .collect() + } + /// Returns the original file size for random access operations. + /// + /// On 32-bit targets a file larger than `usize::MAX` bytes reports + /// `usize::MAX`; [`DataMapIndex::file_size`] gives the exact size. pub fn file_size(&self) -> usize { - self.chunk_infos - .iter() - .fold(0, |acc, chunk| acc + chunk.src_size) + usize::try_from(self.index.file_size()).unwrap_or(usize::MAX) } /// Decrypts and returns a specific byte range from the encrypted data. @@ -212,107 +206,32 @@ where /// # } /// ``` pub fn get_range(&self, start: usize, len: usize) -> Result { - let file_size = self.file_size(); - - // Validate range - if start >= file_size { + let start = start as u64; + let end = start.saturating_add(len as u64).min(self.index.file_size()); + if start >= end { return Ok(Bytes::new()); } - let end_pos = std::cmp::min(start + len, file_size); - let actual_len = end_pos - start; - - if actual_len == 0 { - return Ok(Bytes::new()); - } - - // Calculate which chunks we need using actual chunk sizes from data map - // This avoids issues with different MAX_CHUNK_SIZE schemes - let start_chunk = self.get_chunk_index_from_infos(start); - let end_chunk = self.get_chunk_index_from_infos(end_pos.saturating_sub(1)); - - // Collect the chunk hashes we need - let mut required_hashes = Vec::new(); - for chunk_info in &self.chunk_infos { - if chunk_info.index >= start_chunk && chunk_info.index <= end_chunk { - required_hashes.push((chunk_info.index, chunk_info.dst_hash)); - } - } - - // Sort by index to ensure correct order - required_hashes.sort_by_key(|(index, _)| *index); + // Collect the hashes of the chunks holding the range, in index order + let chunks = self.index.chunks_overlapping(start..end); + let required_hashes: Vec<_> = self.index.infos()[chunks.clone()] + .iter() + .map(|info| (info.index, info.dst_hash)) + .collect(); // Decrypt the chunks in order and collect the bytes let mut all_bytes = Vec::new(); for batch_hashes in required_hashes.chunks(self.batch_size) { - let fetched_chunks = (self.get_chunk_parallel)(batch_hashes)?; - let chunk_map: HashMap = fetched_chunks.into_iter().collect(); - - for (chunk_index, _) in batch_hashes { - if let Some(encrypted_content) = chunk_map.get(chunk_index) { - let decrypted = decrypt_chunk( - *chunk_index, - encrypted_content, - &self.src_hashes, - self.child_level, - )?; - all_bytes.extend_from_slice(&decrypted); - } + for decrypted in self.fetch_and_decrypt(batch_hashes)? { + all_bytes.extend_from_slice(&decrypted); } } - let bytes = Bytes::from(all_bytes); - - // Calculate the offset within our decrypted data - let start_chunk_pos = self.get_chunk_start_position(start_chunk); - let internal_offset = start - start_chunk_pos; - - if internal_offset >= bytes.len() { - return Ok(Bytes::new()); - } - - // Extract just the range we need - let available_len = bytes.len() - internal_offset; - let range_len = std::cmp::min(actual_len, available_len); - let result = bytes.slice(internal_offset..internal_offset + range_len); - - Ok(result) - } - - /// Helper method to get the starting byte position of a chunk within the file - fn get_chunk_start_position(&self, chunk_index: usize) -> usize { - self.chunk_infos - .iter() - .filter(|info| info.index < chunk_index) - .fold(0, |acc, chunk| acc + chunk.src_size) - } - - /// Calculate chunk index from position based on actual chunk sizes in chunk_infos. - /// This avoids issues when the input datamap was generated using different MAX_CHUNK_SIZE schemes. - /// - /// # Arguments - /// * `position` - Byte position within the file - /// - /// # Returns - /// * `usize` - The chunk index that contains the given position - fn get_chunk_index_from_infos(&self, position: usize) -> usize { - let mut accumulated_size = 0; - - for chunk_info in &self.chunk_infos { - // Check if position falls within this chunk - if position >= accumulated_size && position < accumulated_size + chunk_info.src_size { - return chunk_info.index; - } - accumulated_size += chunk_info.src_size; - } - - // If position is beyond all chunks, return the last chunk index - // This handles the case where position == file_size - if let Some(last_chunk) = self.chunk_infos.last() { - last_chunk.index - } else { - 0 // Fallback for empty chunk_infos (shouldn't happen in practice) - } + // Both offsets lie within the fetched chunks, whose bytes fit in memory + let chunks_start = self.index.chunk_range(chunks.start).start; + let from = (start - chunks_start) as usize; + let to = (end - chunks_start) as usize; + Ok(Bytes::from(all_bytes).slice(from..to)) } } @@ -561,6 +480,9 @@ mod tests { use crate::{encrypt, test_helpers::random_bytes, Error}; use std::collections::HashMap; + /// Enough content for the minimum of three chunks. + const SMALL_FILE_BYTES: usize = 10_000; + #[test] fn test_streaming_decrypt_basic() -> Result<()> { // Create test data @@ -814,6 +736,33 @@ mod tests { Ok(()) } + #[test] + fn test_omitted_chunk_fails_instead_of_truncating() -> Result<()> { + let (data_map, encrypted_chunks) = encrypt(random_bytes(SMALL_FILE_BYTES))?; + let storage: HashMap = encrypted_chunks + .into_iter() + .map(|chunk| (crate::hash::content_hash(&chunk.content), chunk.content)) + .collect(); + let last_chunk = data_map.len() - 1; + + // Succeeds while silently leaving out the last chunk + let get_chunks = |hashes: &[(usize, XorName)]| -> Result> { + Ok(hashes + .iter() + .filter(|(index, _)| *index != last_chunk) + .map(|(index, hash)| (*index, storage[hash].clone())) + .collect()) + }; + let mut stream = streaming_decrypt(&data_map, get_chunks)?; + + assert!(stream.range_full().is_err()); + assert!( + stream.any(|chunk| chunk.is_err()), + "iteration must fail rather than end early" + ); + Ok(()) + } + #[test] fn test_random_access_basic() -> Result<()> { // Create test data @@ -1067,143 +1016,4 @@ mod tests { Ok(()) } - - #[test] - fn test_chunk_boundary_underflow_reproduction() -> Result<()> { - // This test reproduces the exact scenario reported: - // file_size: 16405289714, start_position: 4194304u64 - // We expect start_chunk_pos <= start_position, and if smaller, - // the difference should be less than MAX_CHUNK_SIZE - - let file_size = 16404310194u64 as usize; // ~15.27 GB - let start_position = 4194304u64 as usize; // 4MB before end - - // Simulate different MAX_CHUNK_SIZE cheme - let max_chunk_size = crate::MAX_CHUNK_SIZE * 2; - - println!("Testing with file_size: {file_size}, start_position: {start_position}"); - - // First, create the mock data map to use with get_chunk_index_from_infos - // We need this to avoid dependency on MAX_CHUNK_SIZE utility functions - - // Create a mock data map that simulates how chunks would be distributed - // for a file of this size. We need to generate chunk infos with realistic - // src_size values that match how the encryption algorithm would chunk the data. - - let num_chunks = crate::utils::get_num_chunks_with_variable_max(file_size, max_chunk_size); - println!("Total number of chunks: {num_chunks}"); - - let mut chunk_infos = Vec::new(); - let mut accumulated_size = 0; - - // Generate chunk infos with sizes that match the actual chunking algorithm - for chunk_index in 0..num_chunks { - let chunk_size = crate::utils::get_chunk_size_with_variable_max( - file_size, - chunk_index, - max_chunk_size, - ); - - // Create a ChunkInfo with dummy hashes (as the test notes, only src_size and index matter) - let chunk_info = ChunkInfo { - index: chunk_index, - dst_hash: crate::hash::content_hash(&[chunk_index as u8]), // Dummy hash - src_hash: crate::hash::content_hash(&[(chunk_index + 1) as u8]), // Dummy hash - src_size: chunk_size, - }; - - chunk_infos.push(chunk_info); - accumulated_size += chunk_size; - } - - // Verify the total size matches - assert_eq!( - accumulated_size, file_size, - "Mock data map total size should match file size" - ); - - // Create a mock DecryptionStream to test get_chunk_start_position - let data_map = DataMap::new(chunk_infos); - - // Create a dummy get_chunk_parallel function (won't be used in this test) - let get_chunk_parallel = - |_hashes: &[(usize, XorName)]| -> Result> { Ok(Vec::new()) }; - - // Create a mock DecryptionStream - let mock_stream = DecryptionStream { - chunk_infos: data_map.infos().to_vec(), - src_hashes: vec![crate::hash::content_hash(&[0u8]); num_chunks], // Dummy hashes - child_level: 0, - get_chunk_parallel, - current_batch_start: 0, - current_batch_chunks: Vec::new(), - current_batch_index: 0, - batch_size: stream_decrypt_batch_size(), - }; - - // Use the new get_chunk_index_from_infos method instead of the utility function - let start_chunk_index = mock_stream.get_chunk_index_from_infos(start_position); - println!( - "Calculated start_chunk_index using get_chunk_index_from_infos: {start_chunk_index}" - ); - - // Test get_chunk_start_position - let start_chunk_pos = mock_stream.get_chunk_start_position(start_chunk_index); - - println!("start_chunk_pos: {start_chunk_pos}"); - println!("start_position: {start_position}"); - - // Verify our expectations - if start_chunk_pos <= start_position { - println!("✓ start_chunk_pos <= start_position (as expected)"); - - if start_chunk_pos < start_position { - let diff = start_position - start_chunk_pos; - println!("Difference: {diff}"); - - // The difference should be less than max_chunk_size (the local variable used for this test) - assert!( - diff <= max_chunk_size, - "Difference {} should be less than {}, but got {}", - diff, - max_chunk_size, - diff - ); - println!("✓ Difference {diff} is less than max_chunk_size"); - } else { - println!("start_chunk_pos exactly equals start_position"); - } - } else { - // This is the problematic case that causes underflow - let would_underflow = start_chunk_pos - start_position; - panic!( - "❌ start_chunk_pos ({}) > start_position ({}) by {}, this would cause underflow!", - start_chunk_pos, start_position, would_underflow - ); - } - - // Additional verification: calculate what the internal_offset would be - let internal_offset = start_position - start_chunk_pos; - println!("Calculated internal_offset: {internal_offset}"); - - // Verify this is reasonable (should be less than chunk size) - // Get chunk size from the actual data map instead of utility function - let chunk_size = mock_stream - .chunk_infos - .iter() - .find(|info| info.index == start_chunk_index) - .map(|info| info.src_size) - .unwrap_or(0); - println!("Chunk {start_chunk_index} size: {chunk_size}"); - - assert!( - internal_offset < chunk_size, - "internal_offset {} should be less than chunk size {}", - internal_offset, - chunk_size - ); - - println!("✓ Test passed: No underflow condition detected"); - Ok(()) - } }