From e8aa39ee1337c11f52628356d97fa86ac0d1007d Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:13:44 +0200 Subject: [PATCH 1/2] feat: add DataMapIndex for 64-bit random access DataMapIndex indexes a DataMap's chunks with u64 content offsets on every platform and does no I/O: it lists the chunks overlapping a byte range and decrypts single chunks, checking each against the size and source hash the map declares. Decompression stops at the declared size, so crafted chunks cannot expand beyond it. Callers that fetch chunks asynchronously, such as the wasm32 browser client, no longer need to reimplement the chunk-index and key-derivation contract around decrypt_chunk to read files larger than 4 GiB. DecryptionStream now uses the index, so its random access works with u64 positions and binary searches instead of usize sums and linear scans, and get_range fails when the fetch callback omits a chunk instead of joining the remaining chunks into wrong content. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 21 +++ Cargo.toml | 2 +- nodejs/Cargo.toml | 2 +- src/data_map_index.rs | 356 ++++++++++++++++++++++++++++++++++++++++++ src/decrypt.rs | 32 +++- src/lib.rs | 2 + src/stream_decrypt.rs | 288 ++++------------------------------ 7 files changed, 442 insertions(+), 261 deletions(-) create mode 100644 src/data_map_index.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 869bdae1f..1445022ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *When editing this file, please respect a line length of 100.* +## [0.36.2] - 2026-10-13 + +### Added +- `DataMapIndex` gives random access to a `DataMap`'s content with `u64` positions on every + platform, including 32-bit targets such as `wasm32`. It does no I/O: it lists the chunks + overlapping a byte range and decrypts single chunks, checking each against the size and source + hash the map declares. Decompression stops at the declared size, so crafted chunks cannot expand + beyond it. + +### Changed +- `DecryptionStream` is built on `DataMapIndex`. Random access uses `u64` positions and binary + searches instead of `usize` sums and linear scans, so it no longer overflows on 32-bit targets, + and every decrypted chunk is checked against its declared size and hash. Construction fails for + a map whose chunk indices are not exactly `0..n`. +- On 32-bit targets, `DecryptionStream::file_size` reports `usize::MAX` for content larger than + `usize::MAX` bytes; `DataMapIndex::file_size` gives the exact size. + +### Fixed +- `DecryptionStream::get_range` fails when the fetch callback omits a chunk, instead of joining the + remaining chunks into wrong content. + ## [0.36.1] - 2026-10-06 ### Changed diff --git a/Cargo.toml b/Cargo.toml index ad24aed2d..dd47c61e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ license = "MIT OR Apache-2.0" name = "self_encryption" readme = "README.md" repository = "https://github.com/WithAutonomi/self_encryption" -version = "0.36.1" +version = "0.36.2" [features] default = [] diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 1b646bf30..f9661791e 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["cdylib"] hex = "0.4.3" napi = { version = "2.12.2", default-features = false, features = ["napi4", "napi6", "tokio_rt", "serde-json"] } napi-derive = "2.12.2" -self_encryption = { version = "0.36.1", path = ".." } +self_encryption = { version = "0.36.2", path = ".." } [build-dependencies] napi-build = "2.0.1" diff --git a/src/data_map_index.rs b/src/data_map_index.rs new file mode 100644 index 000000000..fa7946c0e --- /dev/null +++ b/src/data_map_index.rs @@ -0,0 +1,356 @@ +// Copyright 2026 MaidSafe.net limited. +// +// This Autonomi Software is licensed under the MIT license or the Apache License, Version 2.0 +// , at your +// option. This file may not be copied, modified, or distributed except +// according to those terms. + +//! Random access to the content of a `DataMap` with 64-bit positions. + +use crate::{ + decrypt::decrypt_chunk_bounded, hash::content_hash, ChunkInfo, DataMap, Error, Result, +}; +use bytes::Bytes; +use std::ops::Range; +use xor_name::XorName; + +/// Random access to the content of a [`DataMap`], with 64-bit positions on +/// every platform. +/// +/// The index performs no I/O. Callers fetch the encrypted chunks listed by +/// [`Self::infos`] however suits them, for example concurrently and +/// asynchronously, and pass each one to [`Self::decrypt_chunk`]. Positions are +/// `u64`, so content larger than `usize::MAX` bytes stays addressable on +/// 32-bit targets such as `wasm32`. +/// +/// The content of a shrunk (child) map is its serialized parent map. Resolve +/// it with [`crate::get_root_data_map`] first to index the original data. +/// +/// # Example +/// +/// ```rust +/// use self_encryption::{encrypt, test_helpers::random_bytes, DataMapIndex}; +/// +/// # fn main() -> self_encryption::Result<()> { +/// let original = random_bytes(10_000); +/// let (data_map, chunks) = encrypt(original.clone())?; +/// let index = DataMapIndex::new(&data_map)?; +/// +/// // Decrypt only the chunks holding bytes 4000..6000. +/// let (start, end) = (4000u64, 6000u64); +/// let mut range = Vec::new(); +/// for chunk in index.chunks_overlapping(start..end) { +/// // Here the chunks come straight from `encrypt`, in index order. A real +/// // caller fetches `index.infos()[chunk].dst_hash` from storage. +/// let plaintext = index.decrypt_chunk(chunk, &chunks[chunk].content)?; +/// let span = index.chunk_range(chunk); +/// let from = (start.max(span.start) - span.start) as usize; +/// let to = (end.min(span.end) - span.start) as usize; +/// range.extend_from_slice(&plaintext[from..to]); +/// } +/// assert_eq!(&range[..], &original[4000..6000]); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct DataMapIndex { + /// The map's chunks, where `infos[i].index == i`. + infos: Vec, + /// Pre-encryption hashes in chunk order. Every chunk's keys derive from + /// its own and its neighbours' hashes. + src_hashes: Vec, + /// Content offset of each chunk, followed by the content size. + offsets: Vec, + /// Key-derivation level the map's chunks were encrypted at. + child_level: usize, +} + +impl DataMapIndex { + /// Index the chunks of `data_map`. + /// + /// # Errors + /// + /// Fails if the chunk indices are not exactly `0..n`, or if the content + /// size overflows `u64`. + pub fn new(data_map: &DataMap) -> Result { + let mut infos = data_map.infos().to_vec(); + infos.sort_by_key(|info| info.index); + let mut offsets = Vec::with_capacity(infos.len() + 1); + let mut end = 0u64; + offsets.push(end); + for (position, info) in infos.iter().enumerate() { + if info.index != position { + return Err(Error::Generic(format!( + "DataMap chunk indices must be contiguous from 0; found {} at position {position}", + info.index + ))); + } + end = end + .checked_add(info.src_size as u64) + .ok_or_else(|| Error::Generic("DataMap content size overflows u64".to_string()))?; + offsets.push(end); + } + Ok(Self { + src_hashes: infos.iter().map(|info| info.src_hash).collect(), + infos, + offsets, + child_level: data_map.child().unwrap_or(0), + }) + } + + /// Size of the content in bytes. + pub fn file_size(&self) -> u64 { + self.offsets.last().copied().unwrap_or_default() + } + + /// The map's chunks in index order: `infos()[i].index == i`. + pub fn infos(&self) -> &[ChunkInfo] { + &self.infos + } + + /// Half-open byte range of the content held by chunk `index`. + /// + /// # Panics + /// + /// Panics if `index` is not below `infos().len()`. + pub fn chunk_range(&self, index: usize) -> Range { + self.offsets[index]..self.offsets[index + 1] + } + + /// Indices of the chunks overlapping the half-open byte range `range`, + /// which is clamped at the end of the content. Empty when nothing of the + /// content lies inside `range`. + pub fn chunks_overlapping(&self, range: Range) -> Range { + let end = range.end.min(self.file_size()); + if range.start >= end { + return 0..0; + } + // `offsets[0]` is 0, so at least one offset is at or before the start. + let first = self + .offsets + .partition_point(|offset| *offset <= range.start) + - 1; + let last = self.offsets.partition_point(|offset| *offset < end); + first..last + } + + /// Decrypt chunk `index` from its encrypted `content`, checking that the + /// result has the size and hash the map declares. + /// + /// Decompression stops at the declared size, so crafted content cannot + /// expand beyond what the map states. + /// + /// # Errors + /// + /// Fails if `index` is out of range, if `content` does not decrypt with + /// the chunk's keys, or if the result differs from the declared size or + /// hash. + pub fn decrypt_chunk(&self, index: usize, content: &Bytes) -> Result { + let info = self.infos.get(index).ok_or_else(|| { + Error::Generic(format!( + "chunk index {index} out of bounds for {} chunks", + self.infos.len() + )) + })?; + let plaintext = decrypt_chunk_bounded( + index, + content, + &self.src_hashes, + self.child_level, + info.src_size, + )?; + if plaintext.len() != info.src_size { + return Err(Error::Generic(format!( + "chunk {index} decrypted to {} bytes, but the DataMap declares {}", + plaintext.len(), + info.src_size + ))); + } + if content_hash(&plaintext) != info.src_hash { + return Err(Error::Generic(format!( + "chunk {index} does not match its DataMap source hash" + ))); + } + Ok(plaintext) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Content of a small map: three chunks, the fewest `encrypt` produces. + const SMALL_CONTENT_BYTES: usize = 10_000; + /// Chunk size for indexes larger than 32-bit `usize` can address. + const LARGE_CHUNK_BYTES: usize = 1 << 30; + /// Chunks in the large index: 6 GiB of content. + const LARGE_CHUNKS: usize = 6; + /// Offset that 32-bit `usize` arithmetic cannot represent. + const FOUR_GIB: u64 = 1 << 32; + + fn small_map() -> Result<(Bytes, DataMap, Vec)> { + let content = crate::test_helpers::random_bytes(SMALL_CONTENT_BYTES); + let (map, chunks) = crate::encrypt(content.clone())?; + let mut by_index = vec![Bytes::new(); map.len()]; + for info in map.infos() { + by_index[info.index] = chunks + .iter() + .find(|chunk| content_hash(&chunk.content) == info.dst_hash) + .map(|chunk| chunk.content.clone()) + .expect("encrypt returns every chunk of its map"); + } + Ok((content, map, by_index)) + } + + fn large_map() -> DataMap { + DataMap::new( + (0..LARGE_CHUNKS) + .map(|index| ChunkInfo { + index, + dst_hash: content_hash(&index.to_le_bytes()), + src_hash: content_hash(&index.to_be_bytes()), + src_size: LARGE_CHUNK_BYTES, + }) + .collect(), + ) + } + + #[test] + fn decrypted_chunks_reassemble_the_content() -> Result<()> { + let (content, map, chunks) = small_map()?; + let index = DataMapIndex::new(&map)?; + assert_eq!(index.file_size(), content.len() as u64); + let mut reassembled = Vec::new(); + for (chunk, encrypted) in chunks.iter().enumerate() { + let plaintext = index.decrypt_chunk(chunk, encrypted)?; + assert_eq!(plaintext.len() as u64, { + let span = index.chunk_range(chunk); + span.end - span.start + }); + reassembled.extend_from_slice(&plaintext); + } + assert_eq!(reassembled, content.to_vec()); + Ok(()) + } + + #[test] + fn overlapping_chunks_follow_half_open_ranges() -> Result<()> { + let (content, map, _) = small_map()?; + let index = DataMapIndex::new(&map)?; + let size = content.len() as u64; + let boundary = index.chunk_range(1).start; + assert_eq!(index.chunks_overlapping(0..1), 0..1); + assert_eq!(index.chunks_overlapping(boundary - 1..boundary), 0..1); + assert_eq!(index.chunks_overlapping(boundary - 1..boundary + 1), 0..2); + assert_eq!(index.chunks_overlapping(boundary..boundary + 1), 1..2); + assert_eq!(index.chunks_overlapping(0..u64::MAX), 0..3); + assert_eq!(index.chunks_overlapping(size - 1..u64::MAX), 2..3); + assert_eq!(index.chunks_overlapping(size..size + 1), 0..0); + assert_eq!(index.chunks_overlapping(5..5), 0..0); + Ok(()) + } + + #[test] + fn positions_beyond_four_gib_need_no_wide_usize() -> Result<()> { + let index = DataMapIndex::new(&large_map())?; + let chunk_bytes = LARGE_CHUNK_BYTES as u64; + assert_eq!(index.file_size(), LARGE_CHUNKS as u64 * chunk_bytes); + let at_four_gib = (FOUR_GIB / chunk_bytes) as usize; + assert_eq!( + index.chunks_overlapping(FOUR_GIB - 1..FOUR_GIB + 1), + at_four_gib - 1..at_four_gib + 1 + ); + let last = LARGE_CHUNKS - 1; + let end = index.file_size(); + assert_eq!(index.chunk_range(last), end - chunk_bytes..end); + assert_eq!(index.chunks_overlapping(end - 1..end), last..LARGE_CHUNKS); + Ok(()) + } + + #[test] + fn positions_map_by_the_maps_own_chunk_sizes() -> Result<()> { + // Reported underflow: a ~15.27 GB file chunked with twice this build's + // MAX_CHUNK_SIZE, read from 4 MiB. + let file_size = 16_404_310_194u64 as usize; + let start = 4_194_304u64; + let max_chunk_size = crate::MAX_CHUNK_SIZE * 2; + let chunks = crate::utils::get_num_chunks_with_variable_max(file_size, max_chunk_size); + let map = DataMap::new( + (0..chunks) + .map(|index| ChunkInfo { + index, + src_size: crate::utils::get_chunk_size_with_variable_max( + file_size, + index, + max_chunk_size, + ), + ..ChunkInfo::default() + }) + .collect(), + ); + let index = DataMapIndex::new(&map)?; + assert_eq!(index.file_size(), file_size as u64); + let chunk = index.chunks_overlapping(start..start + 1).start; + assert!(index.chunk_range(chunk).contains(&start)); + Ok(()) + } + + #[test] + fn indices_must_run_from_zero_without_gaps_or_repeats() { + for indices in [[0, 1, 3], [0, 0, 1], [1, 2, 3]] { + let map = DataMap::new( + indices + .iter() + .map(|&index| ChunkInfo { + index, + src_size: 1, + ..ChunkInfo::default() + }) + .collect(), + ); + assert!(DataMapIndex::new(&map).is_err(), "accepted {:?}", indices); + } + } + + #[test] + fn content_must_have_the_declared_size() -> Result<()> { + let (_, map, chunks) = small_map()?; + for delta in [-1isize, 1] { + let mut infos = map.infos().to_vec(); + infos[0].src_size = (infos[0].src_size as isize + delta) as usize; + let index = DataMapIndex::new(&DataMap::new(infos))?; + assert!( + index.decrypt_chunk(0, &chunks[0]).is_err(), + "size delta {}", + delta + ); + } + Ok(()) + } + + #[test] + fn content_must_have_the_declared_hash() -> Result<()> { + let (content, map, _) = small_map()?; + // A crafted map whose first chunk declares the wrong source hash, with + // every chunk encrypted under keys derived from the declared hashes. + let mut infos = map.infos().to_vec(); + infos[0].src_hash = content_hash(b"not the first chunk"); + let src_hashes: Vec<_> = infos.iter().map(|info| info.src_hash).collect(); + let index = DataMapIndex::new(&DataMap::new(infos))?; + let span = index.chunk_range(0); + let plaintext = content.slice(span.start as usize..span.end as usize); + let keys = crate::utils::get_pad_key_and_nonce(0, &src_hashes, 0)?; + let encrypted = crate::encrypt::encrypt_chunk(plaintext, keys)?; + assert!(index.decrypt_chunk(0, &encrypted).is_err()); + Ok(()) + } + + #[test] + fn chunk_index_must_be_in_range() -> Result<()> { + let (_, map, chunks) = small_map()?; + let index = DataMapIndex::new(&map)?; + assert!(index.decrypt_chunk(chunks.len(), &chunks[0]).is_err()); + Ok(()) + } +} diff --git a/src/decrypt.rs b/src/decrypt.rs index 9f22b341a..b2e5c5efd 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -7,11 +7,15 @@ // according to those terms. use crate::{cipher, utils::get_pad_key_and_nonce, utils::xor, EncryptedChunk, Error, Result}; +use brotli::Decompressor; use bytes::Bytes; use rayon::prelude::*; -use std::io::Cursor; +use std::io::{Cursor, Read}; use xor_name::XorName; +/// Internal buffer of the streaming decompressor, as `brotli::BrotliDecompress` uses. +const DECOMPRESS_BUFFER_BYTES: usize = 4096; + // encrypted_Chunks are sorted !! pub fn decrypt_sorted_set( src_hashes: Vec, @@ -43,6 +47,18 @@ pub fn decrypt_chunk( content: &Bytes, src_hashes: &[XorName], child_level: usize, +) -> Result { + decrypt_chunk_bounded(chunk_index, content, src_hashes, child_level, usize::MAX) +} + +/// As [`decrypt_chunk`], but fails rather than decompress more than +/// `max_size` bytes. +pub(crate) fn decrypt_chunk_bounded( + chunk_index: usize, + content: &Bytes, + src_hashes: &[XorName], + child_level: usize, + max_size: usize, ) -> Result { let pki = get_pad_key_and_nonce(chunk_index, src_hashes, child_level)?; let (pad, key, nonce) = pki; @@ -53,11 +69,17 @@ pub fn decrypt_chunk( // Then decrypt the content let decrypted = cipher::decrypt(xored, &key, &nonce)?; - // Finally decompress + // Finally decompress, reading one byte past the bound to detect an overrun let mut decompressed = Vec::new(); - let mut cursor = Cursor::new(&decrypted); - - brotli::BrotliDecompress(&mut cursor, &mut decompressed).map_err(|_| Error::Compression)?; + let _ = Decompressor::new(Cursor::new(&decrypted), DECOMPRESS_BUFFER_BYTES) + .take((max_size as u64).saturating_add(1)) + .read_to_end(&mut decompressed) + .map_err(|_| Error::Compression)?; + if decompressed.len() > max_size { + return Err(Error::Generic(format!( + "chunk {chunk_index} decompresses to more than {max_size} bytes" + ))); + } Ok(Bytes::from(decompressed)) } diff --git a/src/lib.rs b/src/lib.rs index a8419c513..fb8bf9626 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,7 @@ mod chunk; mod cipher; mod data_map; +mod data_map_index; mod decrypt; mod encrypt; mod error; @@ -110,6 +111,7 @@ pub use xor_name::XorName; pub use self::{ data_map::{ChunkInfo, DataMap}, + data_map_index::DataMapIndex, error::{Error, Result}, stream_decrypt::{streaming_decrypt, streaming_decrypt_with_batch_size, DecryptionStream}, stream_encrypt::{stream_encrypt, ChunkStream, EncryptionStream}, diff --git a/src/stream_decrypt.rs b/src/stream_decrypt.rs index 2d31b319a..f958e2375 100644 --- a/src/stream_decrypt.rs +++ b/src/stream_decrypt.rs @@ -9,11 +9,11 @@ //! Streaming decryption functionality for memory-efficient processing of large encrypted files. use crate::{ - decrypt::decrypt_chunk, get_root_data_map_parallel, stream_decrypt_batch_size, - utils::extract_hashes, ChunkInfo, DataMap, Result, + get_root_data_map_parallel, stream_decrypt_batch_size, DataMap, DataMapIndex, Error, Result, }; use bytes::Bytes; use std::collections::HashMap; +use std::convert::TryFrom; use std::ops::Range; use xor_name::XorName; @@ -29,9 +29,7 @@ use xor_name::XorName; /// to any byte range within the encrypted file using methods like `get_range()`, /// `range()`, and other convenience methods. pub struct DecryptionStream { - chunk_infos: Vec, - src_hashes: Vec, - child_level: usize, + index: DataMapIndex, get_chunk_parallel: F, current_batch_start: usize, current_batch_chunks: Vec, @@ -83,15 +81,8 @@ where data_map.clone() }; - let child_level = root_map.child().unwrap_or(0); - let mut chunk_infos = root_map.infos().to_vec(); - chunk_infos.sort_by_key(|info| info.index); - let src_hashes = extract_hashes(&root_map); - Ok(Self { - chunk_infos, - src_hashes, - child_level, + index: DataMapIndex::new(&root_map)?, get_chunk_parallel, current_batch_start: 0, current_batch_chunks: Vec::new(), @@ -102,23 +93,23 @@ where /// Fetches and decrypts the next batch of chunks. fn fetch_next_batch(&mut self) -> Result { - if self.current_batch_start >= self.chunk_infos.len() { + let chunk_infos = self.index.infos(); + if self.current_batch_start >= chunk_infos.len() { return Ok(false); // No more chunks } let batch_end = self .current_batch_start .saturating_add(self.batch_size) - .min(self.chunk_infos.len()); - let batch_infos = self - .chunk_infos + .min(chunk_infos.len()); + let batch_infos = chunk_infos .get(self.current_batch_start..batch_end) .ok_or_else(|| { - crate::Error::Generic(format!( + Error::Generic(format!( "batch range {}..{} out of bounds for chunk_infos (len {})", self.current_batch_start, batch_end, - self.chunk_infos.len() + chunk_infos.len() )) })?; @@ -135,12 +126,7 @@ where // Decrypt each chunk and store the results self.current_batch_chunks.clear(); for (info, (_index, encrypted_content)) in batch_infos.iter().zip(fetched_chunks) { - let decrypted_chunk = decrypt_chunk( - info.index, - &encrypted_content, - &self.src_hashes, - self.child_level, - )?; + let decrypted_chunk = self.index.decrypt_chunk(info.index, &encrypted_content)?; self.current_batch_chunks.push(decrypted_chunk); } @@ -151,10 +137,11 @@ where } /// Returns the original file size for random access operations. + /// + /// On 32-bit targets a file larger than `usize::MAX` bytes reports + /// `usize::MAX`; [`DataMapIndex::file_size`] gives the exact size. pub fn file_size(&self) -> usize { - self.chunk_infos - .iter() - .fold(0, |acc, chunk| acc + chunk.src_size) + usize::try_from(self.index.file_size()).unwrap_or(usize::MAX) } /// Decrypts and returns a specific byte range from the encrypted data. @@ -212,35 +199,18 @@ where /// # } /// ``` pub fn get_range(&self, start: usize, len: usize) -> Result { - let file_size = self.file_size(); - - // Validate range - if start >= file_size { - return Ok(Bytes::new()); - } - - let end_pos = std::cmp::min(start + len, file_size); - let actual_len = end_pos - start; - - if actual_len == 0 { + let start = start as u64; + let end = start.saturating_add(len as u64).min(self.index.file_size()); + if start >= end { return Ok(Bytes::new()); } - // Calculate which chunks we need using actual chunk sizes from data map - // This avoids issues with different MAX_CHUNK_SIZE schemes - let start_chunk = self.get_chunk_index_from_infos(start); - let end_chunk = self.get_chunk_index_from_infos(end_pos.saturating_sub(1)); - - // Collect the chunk hashes we need - let mut required_hashes = Vec::new(); - for chunk_info in &self.chunk_infos { - if chunk_info.index >= start_chunk && chunk_info.index <= end_chunk { - required_hashes.push((chunk_info.index, chunk_info.dst_hash)); - } - } - - // Sort by index to ensure correct order - required_hashes.sort_by_key(|(index, _)| *index); + // Collect the hashes of the chunks holding the range, in index order + let chunks = self.index.chunks_overlapping(start..end); + let required_hashes: Vec<_> = self.index.infos()[chunks.clone()] + .iter() + .map(|info| (info.index, info.dst_hash)) + .collect(); // Decrypt the chunks in order and collect the bytes let mut all_bytes = Vec::new(); @@ -249,70 +219,19 @@ where let chunk_map: HashMap = fetched_chunks.into_iter().collect(); for (chunk_index, _) in batch_hashes { - if let Some(encrypted_content) = chunk_map.get(chunk_index) { - let decrypted = decrypt_chunk( - *chunk_index, - encrypted_content, - &self.src_hashes, - self.child_level, - )?; - all_bytes.extend_from_slice(&decrypted); - } + let encrypted_content = chunk_map.get(chunk_index).ok_or_else(|| { + Error::Generic(format!("chunk {chunk_index} missing from fetched chunks")) + })?; + let decrypted = self.index.decrypt_chunk(*chunk_index, encrypted_content)?; + all_bytes.extend_from_slice(&decrypted); } } - let bytes = Bytes::from(all_bytes); - - // Calculate the offset within our decrypted data - let start_chunk_pos = self.get_chunk_start_position(start_chunk); - let internal_offset = start - start_chunk_pos; - - if internal_offset >= bytes.len() { - return Ok(Bytes::new()); - } - - // Extract just the range we need - let available_len = bytes.len() - internal_offset; - let range_len = std::cmp::min(actual_len, available_len); - let result = bytes.slice(internal_offset..internal_offset + range_len); - - Ok(result) - } - - /// Helper method to get the starting byte position of a chunk within the file - fn get_chunk_start_position(&self, chunk_index: usize) -> usize { - self.chunk_infos - .iter() - .filter(|info| info.index < chunk_index) - .fold(0, |acc, chunk| acc + chunk.src_size) - } - - /// Calculate chunk index from position based on actual chunk sizes in chunk_infos. - /// This avoids issues when the input datamap was generated using different MAX_CHUNK_SIZE schemes. - /// - /// # Arguments - /// * `position` - Byte position within the file - /// - /// # Returns - /// * `usize` - The chunk index that contains the given position - fn get_chunk_index_from_infos(&self, position: usize) -> usize { - let mut accumulated_size = 0; - - for chunk_info in &self.chunk_infos { - // Check if position falls within this chunk - if position >= accumulated_size && position < accumulated_size + chunk_info.src_size { - return chunk_info.index; - } - accumulated_size += chunk_info.src_size; - } - - // If position is beyond all chunks, return the last chunk index - // This handles the case where position == file_size - if let Some(last_chunk) = self.chunk_infos.last() { - last_chunk.index - } else { - 0 // Fallback for empty chunk_infos (shouldn't happen in practice) - } + // Both offsets lie within the fetched chunks, whose bytes fit in memory + let chunks_start = self.index.chunk_range(chunks.start).start; + let from = (start - chunks_start) as usize; + let to = (end - chunks_start) as usize; + Ok(Bytes::from(all_bytes).slice(from..to)) } } @@ -1067,143 +986,4 @@ mod tests { Ok(()) } - - #[test] - fn test_chunk_boundary_underflow_reproduction() -> Result<()> { - // This test reproduces the exact scenario reported: - // file_size: 16405289714, start_position: 4194304u64 - // We expect start_chunk_pos <= start_position, and if smaller, - // the difference should be less than MAX_CHUNK_SIZE - - let file_size = 16404310194u64 as usize; // ~15.27 GB - let start_position = 4194304u64 as usize; // 4MB before end - - // Simulate different MAX_CHUNK_SIZE cheme - let max_chunk_size = crate::MAX_CHUNK_SIZE * 2; - - println!("Testing with file_size: {file_size}, start_position: {start_position}"); - - // First, create the mock data map to use with get_chunk_index_from_infos - // We need this to avoid dependency on MAX_CHUNK_SIZE utility functions - - // Create a mock data map that simulates how chunks would be distributed - // for a file of this size. We need to generate chunk infos with realistic - // src_size values that match how the encryption algorithm would chunk the data. - - let num_chunks = crate::utils::get_num_chunks_with_variable_max(file_size, max_chunk_size); - println!("Total number of chunks: {num_chunks}"); - - let mut chunk_infos = Vec::new(); - let mut accumulated_size = 0; - - // Generate chunk infos with sizes that match the actual chunking algorithm - for chunk_index in 0..num_chunks { - let chunk_size = crate::utils::get_chunk_size_with_variable_max( - file_size, - chunk_index, - max_chunk_size, - ); - - // Create a ChunkInfo with dummy hashes (as the test notes, only src_size and index matter) - let chunk_info = ChunkInfo { - index: chunk_index, - dst_hash: crate::hash::content_hash(&[chunk_index as u8]), // Dummy hash - src_hash: crate::hash::content_hash(&[(chunk_index + 1) as u8]), // Dummy hash - src_size: chunk_size, - }; - - chunk_infos.push(chunk_info); - accumulated_size += chunk_size; - } - - // Verify the total size matches - assert_eq!( - accumulated_size, file_size, - "Mock data map total size should match file size" - ); - - // Create a mock DecryptionStream to test get_chunk_start_position - let data_map = DataMap::new(chunk_infos); - - // Create a dummy get_chunk_parallel function (won't be used in this test) - let get_chunk_parallel = - |_hashes: &[(usize, XorName)]| -> Result> { Ok(Vec::new()) }; - - // Create a mock DecryptionStream - let mock_stream = DecryptionStream { - chunk_infos: data_map.infos().to_vec(), - src_hashes: vec![crate::hash::content_hash(&[0u8]); num_chunks], // Dummy hashes - child_level: 0, - get_chunk_parallel, - current_batch_start: 0, - current_batch_chunks: Vec::new(), - current_batch_index: 0, - batch_size: stream_decrypt_batch_size(), - }; - - // Use the new get_chunk_index_from_infos method instead of the utility function - let start_chunk_index = mock_stream.get_chunk_index_from_infos(start_position); - println!( - "Calculated start_chunk_index using get_chunk_index_from_infos: {start_chunk_index}" - ); - - // Test get_chunk_start_position - let start_chunk_pos = mock_stream.get_chunk_start_position(start_chunk_index); - - println!("start_chunk_pos: {start_chunk_pos}"); - println!("start_position: {start_position}"); - - // Verify our expectations - if start_chunk_pos <= start_position { - println!("✓ start_chunk_pos <= start_position (as expected)"); - - if start_chunk_pos < start_position { - let diff = start_position - start_chunk_pos; - println!("Difference: {diff}"); - - // The difference should be less than max_chunk_size (the local variable used for this test) - assert!( - diff <= max_chunk_size, - "Difference {} should be less than {}, but got {}", - diff, - max_chunk_size, - diff - ); - println!("✓ Difference {diff} is less than max_chunk_size"); - } else { - println!("start_chunk_pos exactly equals start_position"); - } - } else { - // This is the problematic case that causes underflow - let would_underflow = start_chunk_pos - start_position; - panic!( - "❌ start_chunk_pos ({}) > start_position ({}) by {}, this would cause underflow!", - start_chunk_pos, start_position, would_underflow - ); - } - - // Additional verification: calculate what the internal_offset would be - let internal_offset = start_position - start_chunk_pos; - println!("Calculated internal_offset: {internal_offset}"); - - // Verify this is reasonable (should be less than chunk size) - // Get chunk size from the actual data map instead of utility function - let chunk_size = mock_stream - .chunk_infos - .iter() - .find(|info| info.index == start_chunk_index) - .map(|info| info.src_size) - .unwrap_or(0); - println!("Chunk {start_chunk_index} size: {chunk_size}"); - - assert!( - internal_offset < chunk_size, - "internal_offset {} should be less than chunk size {}", - internal_offset, - chunk_size - ); - - println!("✓ Test passed: No underflow condition detected"); - Ok(()) - } } From 869b2f94d5b3b2c5a488e74c477939b5544ffc22 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:08:08 +0200 Subject: [PATCH 2/2] fix: fail stream iteration when the fetch callback omits a chunk The iterator paired fetched chunks with the batch by position, so a callback that returned fewer chunks than requested ended the stream early, as if the content were complete. Iteration and range reads now share one batch fetch that looks chunks up by index and fails on a missing one. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +-- src/stream_decrypt.rs | 68 +++++++++++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1445022ff..51cb87145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `usize::MAX` bytes; `DataMapIndex::file_size` gives the exact size. ### Fixed -- `DecryptionStream::get_range` fails when the fetch callback omits a chunk, instead of joining the - remaining chunks into wrong content. +- `DecryptionStream` fails when the fetch callback omits a chunk. Before, `get_range` joined the + remaining chunks into wrong content, and iteration ended early as if the content were complete. ## [0.36.1] - 2026-10-06 diff --git a/src/stream_decrypt.rs b/src/stream_decrypt.rs index f958e2375..5fcb08445 100644 --- a/src/stream_decrypt.rs +++ b/src/stream_decrypt.rs @@ -119,23 +119,30 @@ where .map(|info| (info.index, info.dst_hash)) .collect(); - // Fetch chunks in parallel - let mut fetched_chunks = (self.get_chunk_parallel)(&batch_hashes)?; - fetched_chunks.sort_by_key(|(index, _content)| *index); - - // Decrypt each chunk and store the results - self.current_batch_chunks.clear(); - for (info, (_index, encrypted_content)) in batch_infos.iter().zip(fetched_chunks) { - let decrypted_chunk = self.index.decrypt_chunk(info.index, &encrypted_content)?; - self.current_batch_chunks.push(decrypted_chunk); - } - + self.current_batch_chunks = self.fetch_and_decrypt(&batch_hashes)?; self.current_batch_start = batch_end; self.current_batch_index = 0; Ok(true) } + /// Fetches one batch and decrypts it in batch order, failing if the + /// callback omits a requested chunk. + fn fetch_and_decrypt(&self, batch_hashes: &[(usize, XorName)]) -> Result> { + let mut fetched_chunks: HashMap = (self.get_chunk_parallel)(batch_hashes)? + .into_iter() + .collect(); + batch_hashes + .iter() + .map(|(chunk_index, _)| { + let encrypted_content = fetched_chunks.remove(chunk_index).ok_or_else(|| { + Error::Generic(format!("chunk {chunk_index} missing from fetched chunks")) + })?; + self.index.decrypt_chunk(*chunk_index, &encrypted_content) + }) + .collect() + } + /// Returns the original file size for random access operations. /// /// On 32-bit targets a file larger than `usize::MAX` bytes reports @@ -215,14 +222,7 @@ where // Decrypt the chunks in order and collect the bytes let mut all_bytes = Vec::new(); for batch_hashes in required_hashes.chunks(self.batch_size) { - let fetched_chunks = (self.get_chunk_parallel)(batch_hashes)?; - let chunk_map: HashMap = fetched_chunks.into_iter().collect(); - - for (chunk_index, _) in batch_hashes { - let encrypted_content = chunk_map.get(chunk_index).ok_or_else(|| { - Error::Generic(format!("chunk {chunk_index} missing from fetched chunks")) - })?; - let decrypted = self.index.decrypt_chunk(*chunk_index, encrypted_content)?; + for decrypted in self.fetch_and_decrypt(batch_hashes)? { all_bytes.extend_from_slice(&decrypted); } } @@ -480,6 +480,9 @@ mod tests { use crate::{encrypt, test_helpers::random_bytes, Error}; use std::collections::HashMap; + /// Enough content for the minimum of three chunks. + const SMALL_FILE_BYTES: usize = 10_000; + #[test] fn test_streaming_decrypt_basic() -> Result<()> { // Create test data @@ -733,6 +736,33 @@ mod tests { Ok(()) } + #[test] + fn test_omitted_chunk_fails_instead_of_truncating() -> Result<()> { + let (data_map, encrypted_chunks) = encrypt(random_bytes(SMALL_FILE_BYTES))?; + let storage: HashMap = encrypted_chunks + .into_iter() + .map(|chunk| (crate::hash::content_hash(&chunk.content), chunk.content)) + .collect(); + let last_chunk = data_map.len() - 1; + + // Succeeds while silently leaving out the last chunk + let get_chunks = |hashes: &[(usize, XorName)]| -> Result> { + Ok(hashes + .iter() + .filter(|(index, _)| *index != last_chunk) + .map(|(index, hash)| (*index, storage[hash].clone())) + .collect()) + }; + let mut stream = streaming_decrypt(&data_map, get_chunks)?; + + assert!(stream.range_full().is_err()); + assert!( + stream.any(|chunk| chunk.is_err()), + "iteration must fail rather than end early" + ); + Ok(()) + } + #[test] fn test_random_access_basic() -> Result<()> { // Create test data