chunker.rs raw

use std::io::Read;

use crate::{Error, crypto::RepoConfig};

/// Splits a byte stream into content-defined chunks with FastCDC.
///
/// Content-defined boundaries mean that an insertion or deletion early in
/// a file only changes the chunks around the edit, so unchanged content
/// keeps deduplicating across snapshots.
pub(crate) fn chunks(
    source: impl Read,
    config: &RepoConfig,
) -> impl Iterator<Item = Result<Vec<u8>, Error>> {
    fastcdc::v2020::StreamCDC::new(source, config.chunk_min, config.chunk_avg, config.chunk_max)
        .map(|result| match result {
            Ok(chunk) => Ok(chunk.data),
            Err(fastcdc::v2020::Error::IoError(err)) => Err(Error::IO(err)),
            Err(err) => Err(Error::Chunking(err.to_string())),
        })
}