//! Conditional parsing with until combinators. //! //! This module provides parsers that consume input until a specified end condition //! is met. It includes both byte-level ([`Until`]) and UTF-8 character-level //! ([`Utf8Until`]) variants for different parsing needs. //! //! Until parsers are useful for extracting content between delimiters, parsing //! string literals, comments, or any scenario where you need to consume input //! up to a specific boundary condition. use crate::{ cache::ParsingCache, parser::{Parsable, Parser, Source}, result::{Error, ParseResult}, utf8util::read_utf8_char, }; /// A parser that consumes characters until an end condition is met. /// /// Until reads characters one by one from the input until the end parser /// would match at the current position. The end parser is not consumed - /// it's just used to determine when to stop reading. /// /// This is useful for parsing content up to a specific delimiter without /// consuming the delimiter itself. /// /// # Examples /// /// ```rust /// use neotoma::{until::Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read characters until we hit "end" /// let parser = Until::new(Literal::from_str("end")); /// /// let mut input = Cursor::new(b"hello world end more text"); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// assert_eq!(result, b"hello world ".to_vec()); /// ``` #[derive(Clone, Debug, PartialEq, Eq)] pub struct Until { end: E, min_length: usize, max_length: Option, } impl Until { /// Create a new Until parser with default bounds (0 or more characters). /// /// # Examples /// /// ```rust /// use neotoma::{until::Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read until we see "." /// let parser = Until::new(Literal::from_str(".")); /// /// let mut input = Cursor::new(b"hello world.more"); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// assert_eq!(result, b"hello world".to_vec()); /// ``` pub fn new(end: E) -> Self { Self { end, min_length: 0, max_length: None, } } /// Create an Until parser that requires at least `min` characters. /// /// # Examples /// /// ```rust /// use neotoma::{until::Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Require at least 5 characters before "." /// let parser = Until::with_min(Literal::from_str("."), 5); /// /// let mut input1 = Cursor::new(b"hello.world"); /// let mut source1 = Source::new(input1); /// let result1 = parse(parser, &mut source1).unwrap(); /// assert_eq!(result1, b"hello".to_vec()); /// /// // This would fail with only 2 characters /// let parser2 = Until::with_min(Literal::from_str("."), 5); /// let mut input2 = Cursor::new(b"hi.there"); /// let mut source2 = Source::new(input2); /// let result2 = parse(parser2, &mut source2); /// assert!(result2.is_err()); /// ``` pub fn with_min(end: E, min: usize) -> Self { Self { end, min_length: min, max_length: None, } } /// Create an Until parser that reads at most `max` characters. /// /// # Examples /// /// ```rust /// use neotoma::{until::Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read at most 5 characters before "." /// let parser = Until::with_max(Literal::from_str("."), 5); /// /// let mut input = Cursor::new(b"hello world.more"); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// assert_eq!(result, b"hello".to_vec()); // Stops at 5 chars /// ``` pub fn with_max(end: E, max: usize) -> Self { Self { end, min_length: 0, max_length: Some(max), } } /// Create an Until parser with both minimum and maximum bounds. /// /// # Examples /// /// ```rust /// use neotoma::{until::Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read 3-7 characters before "." /// let parser = Until::with_bounds(Literal::from_str("."), 3, 7); /// /// let mut input = Cursor::new(b"hello.world"); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// assert_eq!(result, b"hello".to_vec()); /// ``` pub fn with_bounds(end: E, min: usize, max: usize) -> Self { Self { end, min_length: min, max_length: Some(max), } } } impl Parser for Until where E: Parser, { type Output = Vec; fn id(&self) -> u64 { use std::any::TypeId; use std::hash::{DefaultHasher, Hash, Hasher}; let mut hasher = DefaultHasher::new(); TypeId::of::().hash(&mut hasher); self.end.id().hash(&mut hasher); self.min_length.hash(&mut hasher); self.max_length.hash(&mut hasher); hasher.finish() } fn read( &self, source: &mut Source, cache: &mut impl ParsingCache, context: &mut Ctx, ) -> ParseResult where S: Parsable, { let mut result = Vec::new(); loop { // Check if we've hit the maximum length if let Some(max) = self.max_length { if result.len() >= max { break; } } // Check if the end condition matches at current position (without consuming) source.push(); match self.end.parse(source, cache, context) { Ok(_) => { // End condition matched - stop here source.pop(); // Backtrack to before the end match break; } Err(Error::NoMatch) => { // End condition doesn't match - continue reading source.pop(); // Backtrack to before the end attempt } Err(err) => { // Other error (like IO error) - propagate it source.pop(); return Err(err); } } // Try to read one byte match source.peek1() { Ok(byte) => { result.push(byte); source.advance(1); } Err(Error::NoMatch) => { // End of input - we're done break; } Err(err) => { // Other error - propagate it return Err(err); } } } // Check minimum requirement if result.len() < self.min_length { return Err(Error::NoMatch); } Ok(result) } } /// A UTF-8 aware parser that consumes characters until an end condition is met. /// /// Utf8Until reads UTF-8 characters one by one from the input until the end parser /// would match at the current position. Unlike `Until`, this parser respects UTF-8 /// character boundaries and operates on character counts rather than byte counts. /// /// # Examples /// /// ```rust /// use neotoma::{until::Utf8Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read UTF-8 characters until we hit "end" /// let parser = Utf8Until::new(Literal::from_str("end")); /// /// let mut input = Cursor::new("hello 世界 end more".as_bytes()); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// assert_eq!(result, "hello 世界 "); /// ``` #[derive(Clone, Debug, PartialEq, Eq)] pub struct Utf8Until { end: E, min_chars: usize, max_chars: Option, } impl Utf8Until { /// Create a new Utf8Until parser with default bounds (0 or more characters). /// /// # Examples /// /// ```rust /// use neotoma::{until::Utf8Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read until we see "." /// let parser = Utf8Until::new(Literal::from_str(".")); /// /// let mut input = Cursor::new("hello 世界.more".as_bytes()); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// assert_eq!(result, "hello 世界"); /// ``` pub fn new(end: E) -> Self { Self { end, min_chars: 0, max_chars: None, } } /// Create a Utf8Until parser that requires at least `min` characters. /// /// # Examples /// /// ```rust /// use neotoma::{until::Utf8Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Require at least 3 characters before "." /// let parser = Utf8Until::with_min(Literal::from_str("."), 3); /// /// let mut input1 = Cursor::new("hello.world".as_bytes()); /// let mut source1 = Source::new(input1); /// let result1 = parse(parser, &mut source1).unwrap(); /// assert_eq!(result1, "hello"); /// /// // This would fail with only 2 characters /// let parser2 = Utf8Until::with_min(Literal::from_str("."), 3); /// let mut input2 = Cursor::new("hi.there".as_bytes()); /// let mut source2 = Source::new(input2); /// let result2 = parse(parser2, &mut source2); /// assert!(result2.is_err()); /// ``` pub fn with_min(end: E, min: usize) -> Self { Self { end, min_chars: min, max_chars: None, } } /// Create a Utf8Until parser that reads at most `max` characters. /// /// # Examples /// /// ```rust /// use neotoma::{until::Utf8Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read at most 2 characters before "." /// let parser = Utf8Until::with_max(Literal::from_str("."), 2); /// /// let mut input = Cursor::new("hello 世界.more".as_bytes()); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// // Note: this reads 2 UTF-8 characters, not 2 bytes /// ``` pub fn with_max(end: E, max: usize) -> Self { Self { end, min_chars: 0, max_chars: Some(max), } } /// Create a Utf8Until parser with both minimum and maximum character bounds. /// /// # Examples /// /// ```rust /// use neotoma::{until::Utf8Until, literal::Literal, parser::{parse, Source}}; /// use std::io::Cursor; /// /// // Read 1-3 characters before "." /// let parser = Utf8Until::with_bounds(Literal::from_str("."), 1, 3); /// /// let mut input = Cursor::new("hello.world".as_bytes()); /// let mut source = Source::new(input); /// let result = parse(parser, &mut source).unwrap(); /// ``` pub fn with_bounds(end: E, min: usize, max: usize) -> Self { Self { end, min_chars: min, max_chars: Some(max), } } } impl Parser for Utf8Until where E: Parser, { type Output = String; fn id(&self) -> u64 { use std::any::TypeId; use std::hash::{DefaultHasher, Hash, Hasher}; let mut hasher = DefaultHasher::new(); TypeId::of::().hash(&mut hasher); self.end.id().hash(&mut hasher); self.min_chars.hash(&mut hasher); self.max_chars.hash(&mut hasher); hasher.finish() } fn read( &self, source: &mut Source, cache: &mut impl ParsingCache, context: &mut Ctx, ) -> ParseResult where S: Parsable, { let mut result = String::new(); let mut char_count = 0; loop { // Check if we've hit the maximum character count if let Some(max) = self.max_chars { if char_count >= max { break; } } // Check if the end condition matches at current position (without consuming) source.push(); match self.end.parse(source, cache, context) { Ok(_) => { // End condition matched - stop here source.pop(); // Backtrack to before the end match break; } Err(Error::NoMatch) => { // End condition doesn't match - continue reading source.pop(); // Backtrack to before the end attempt } Err(err) => { // Other error (like IO error) - propagate it source.pop(); return Err(err); } } // Try to read one UTF-8 character match read_utf8_char(source) { Ok(ch) => { result.push(ch); char_count += 1; } Err(Error::NoMatch) => { // End of input - we're done break; } Err(err) => { // Other error - propagate it return Err(err); } } } // Check minimum requirement if char_count < self.min_chars { return Err(Error::NoMatch); } Ok(result) } } #[cfg(test)] mod tests { use super::*; use crate::{literal::Literal, parser::parse}; use std::io::Cursor; #[test] fn test_id_implementation_different_until_parsers() { // Test that Until implements proper id() method // Different Until parsers should have different IDs to avoid cache conflicts let until1 = Until::new(Literal::from_str("end")); let until2 = Until::new(Literal::from_str("stop")); // These should have different IDs because they have different end conditions // This test will FAIL if Until uses default id() implementation assert_ne!( as crate::parser::Parser<()>>::id(&until1), as crate::parser::Parser<()>>::id(&until2), "Different Until instances should have different IDs to avoid cache collisions" ); } #[test] fn test_id_implementation_same_until_parsers() { // Test that identical Until parsers have the same ID let until1 = Until::new(Literal::from_str("end")); let until2 = Until::new(Literal::from_str("end")); assert_eq!( as crate::parser::Parser<()>>::id(&until1), as crate::parser::Parser<()>>::id(&until2), "Identical Until instances should have the same ID for cache efficiency" ); } #[test] fn test_id_implementation_until_different_bounds() { // Test Until parsers with different bounds let until1 = Until::with_min(Literal::from_str("end"), 1); let until2 = Until::with_min(Literal::from_str("end"), 2); // These should have different IDs because they have different minimum bounds // This test will FAIL if Until uses default id() implementation assert_ne!( as crate::parser::Parser<()>>::id(&until1), as crate::parser::Parser<()>>::id(&until2), "Until instances with different bounds should have different IDs" ); } #[test] fn test_id_implementation_utf8_until_parsers() { // Test that Utf8Until implements proper id() method let utf8_until1 = Utf8Until::new(Literal::from_str("end")); let utf8_until2 = Utf8Until::new(Literal::from_str("stop")); // These should have different IDs because they have different end conditions // This test will FAIL if Utf8Until uses default id() implementation assert_ne!( as crate::parser::Parser<()>>::id(&utf8_until1), as crate::parser::Parser<()>>::id(&utf8_until2), "Different Utf8Until instances should have different IDs to avoid cache collisions" ); } #[test] fn test_until_basic() { let parser = Until::new(Literal::from_str("end")); let mut input = Cursor::new(b"hello world end more text"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"hello world ".to_vec()); // The "end" should still be available to parse let end_parser = Literal::from_str("end"); let end_result = parse(end_parser, &mut source).unwrap(); assert_eq!(end_result, b"end".as_slice().into()); } #[test] fn test_until_single_character_end() { let parser = Until::new(Literal::from_str(".")); let mut input = Cursor::new(b"hello.world"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"hello".to_vec()); // The "." should still be available let dot_parser = Literal::from_str("."); let dot_result = parse(dot_parser, &mut source).unwrap(); assert_eq!(dot_result, b".".as_slice().into()); } #[test] fn test_until_with_min() { let parser = Until::with_min(Literal::from_str("."), 5); // Should succeed with 5+ characters let mut input1 = Cursor::new(b"hello.world"); let mut source1 = crate::parser::Source::new(&mut input1); let result1 = parse(parser, &mut source1).unwrap(); assert_eq!(result1, b"hello".to_vec()); // Should fail with only 2 characters let parser2 = Until::with_min(Literal::from_str("."), 5); let mut input2 = Cursor::new(b"hi.there"); let mut source2 = crate::parser::Source::new(&mut input2); let result2 = parse(parser2, &mut source2); assert!(result2.is_err()); } #[test] fn test_until_with_max() { let parser = Until::with_max(Literal::from_str("."), 5); let mut input = Cursor::new(b"hello world.more"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"hello".to_vec()); // Stops at 5 characters // Should still have " world.more" available let remaining = source.peek(11).unwrap(); assert_eq!(remaining, b" world.more"); } #[test] fn test_until_with_bounds() { let parser = Until::with_bounds(Literal::from_str("."), 3, 7); let mut input = Cursor::new(b"hello world.more"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"hello w".to_vec()); // Stops at 7 characters (max) } #[test] fn test_until_no_end_found() { let parser = Until::new(Literal::from_str(".")); let mut input = Cursor::new(b"hello world"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"hello world".to_vec()); // Consumes all input } #[test] fn test_until_immediate_end() { let parser = Until::new(Literal::from_str("hello")); let mut input = Cursor::new(b"hello world"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"".to_vec()); // No characters consumed before "hello" // "hello" should still be available let hello_parser = Literal::from_str("hello"); let hello_result = parse(hello_parser, &mut source).unwrap(); assert_eq!(hello_result, b"hello".as_slice().into()); } #[test] fn test_until_empty_input() { let parser = Until::new(Literal::from_str(".")); let mut input = Cursor::new(b""); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"".to_vec()); // No matches on empty input } #[test] fn test_until_multichar_delimiter() { let parser = Until::new(Literal::from_str("end")); let mut input = Cursor::new(b"start middle end final"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"start middle ".to_vec()); // "end" should still be available, then we can parse " final" let end_parser = Literal::from_str("end"); let end_result = parse(end_parser, &mut source).unwrap(); assert_eq!(end_result, b"end".as_slice().into()); let final_parser = Literal::from_str(" final"); let final_result = parse(final_parser, &mut source).unwrap(); assert_eq!(final_result, b" final".as_slice().into()); } #[test] fn test_until_in_sequence() { use crate::seq; let parser = seq![ Literal::from_str("start"), Literal::from_str("123"), Literal::from_str("_"), Until::new(Literal::from_str("end")), Literal::from_str("end") ]; let mut input = Cursor::new(b"start123_abcend"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result.0, b"start".as_slice().into()); assert_eq!(result.1.0, b"123".as_slice().into()); assert_eq!(result.1.1.0, b"_".as_slice().into()); assert_eq!(result.1.1.1.0, b"abc".to_vec()); assert_eq!(result.1.1.1.1.0, b"end".as_slice().into()); } #[test] fn test_utf8_until_basic() { let parser = Utf8Until::new(Literal::from_str("end")); let mut input = Cursor::new("hello 世界 end more".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, "hello 世界 "); // The "end" should still be available to parse let end_parser = Literal::from_str("end"); let end_result = parse(end_parser, &mut source).unwrap(); assert_eq!(end_result, b"end".as_slice().into()); } #[test] fn test_utf8_until_character_boundaries() { let parser = Utf8Until::new(Literal::from_str(".")); // Mix of ASCII and multi-byte UTF-8 characters let mut input = Cursor::new("café 世界.more".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, "café 世界"); // The "." should still be available let dot_parser = Literal::from_str("."); let dot_result = parse(dot_parser, &mut source).unwrap(); assert_eq!(dot_result, b".".as_slice().into()); } #[test] fn test_utf8_until_with_max_chars() { let parser = Utf8Until::with_max(Literal::from_str("."), 3); // Should stop at 3 UTF-8 characters, not 3 bytes let mut input = Cursor::new("café 世界 more.end".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); // "café" = 4 bytes but 4 characters, " " = 1 char, "世" = 1 char = 3 chars total assert_eq!(result, "caf"); } #[test] fn test_utf8_until_with_min_chars() { let parser = Utf8Until::with_min(Literal::from_str("."), 3); // Should succeed with 3+ characters let mut input1 = Cursor::new("世界测.test".as_bytes()); let mut source1 = crate::parser::Source::new(&mut input1); let result1 = parse(parser, &mut source1).unwrap(); assert_eq!(result1, "世界测"); // Should fail with only 2 characters let parser2 = Utf8Until::with_min(Literal::from_str("."), 3); let mut input2 = Cursor::new("世界.test".as_bytes()); let mut source2 = crate::parser::Source::new(&mut input2); let result2 = parse(parser2, &mut source2); assert!(result2.is_err()); } #[test] fn test_utf8_until_invalid_utf8() { let parser = Utf8Until::with_min(Literal::from_str("end"), 1); // Create invalid UTF-8 sequence at the start (0xFF is not valid UTF-8) let mut input = Cursor::new(b"\xFF\xFEend"); let mut source = crate::parser::Source::new(&mut input); // Should fail when it tries to read invalid UTF-8 let result = parse(parser, &mut source); assert!(result.is_err()); } #[test] fn test_utf8_until_emoji() { let parser = Utf8Until::new(Literal::from_str("!")); // Test with emoji (4-byte UTF-8 characters) let mut input = Cursor::new("Hello 👋 World 🌍!".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, "Hello 👋 World 🌍"); // The "!" should still be available let bang_parser = Literal::from_str("!"); let bang_result = parse(bang_parser, &mut source).unwrap(); assert_eq!(bang_result, b"!".as_slice().into()); } #[test] fn test_utf8_until_bounds() { let parser = Utf8Until::with_bounds(Literal::from_str("."), 2, 4); let mut input = Cursor::new("世界测试.more".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, "世界测试"); // 4 characters (max) } #[test] fn test_until_complex_end_parser() { use crate::sequence::Sequence; // Use a sequence as the end condition let end_parser = Sequence::new(Literal::from_str("end"), Literal::from_str("_tag")); let parser = Until::new(end_parser); let mut input = Cursor::new(b"some content before end_tag and after"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"some content before ".to_vec()); // The end parser should still be available let end_parser2 = Sequence::new(Literal::from_str("end"), Literal::from_str("_tag")); let end_result = parse(end_parser2, &mut source).unwrap(); assert_eq!(end_result.0, b"end".as_slice().into()); assert_eq!(end_result.1, b"_tag".as_slice().into()); } #[test] fn test_until_with_repeat_end_parser() { use crate::repeat::Repeat; // End condition is repeated 'x' let end_parser = Repeat::with_min(Literal::from_str("x"), 2); let parser = Until::new(end_parser); let mut input = Cursor::new(b"content before xxxx and after"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"content before ".to_vec()); } #[test] fn test_until_multiple_potential_ends() { // Test content with multiple instances of the end pattern let parser = Until::new(Literal::from_str("end")); let mut input = Cursor::new(b"start end middle end final"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"start ".to_vec()); // Stops at first "end" // Should be positioned at the first "end" let remaining = source.peek(20).unwrap(); assert_eq!(remaining, b"end middle end final"); } #[test] fn test_until_overlapping_end_patterns() { // Test where end pattern overlaps with content let parser = Until::new(Literal::from_str("aba")); let mut input = Cursor::new(b"xyzababadef"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"xyz".to_vec()); // Stops when it finds "aba" // Should be positioned at "aba" let remaining = source.peek(8).unwrap(); assert_eq!(remaining, b"ababadef"); } #[test] fn test_until_boundary_edge_cases() { // Test exactly at minimum bound let parser = Until::with_min(Literal::from_str("."), 5); let mut input1 = Cursor::new(b"12345.rest"); let mut source1 = crate::parser::Source::new(&mut input1); let result1 = parse(parser, &mut source1).unwrap(); assert_eq!(result1, b"12345".to_vec()); // Test just below minimum bound let parser2 = Until::with_min(Literal::from_str("."), 5); let mut input2 = Cursor::new(b"1234.rest"); let mut source2 = crate::parser::Source::new(&mut input2); let result2 = parse(parser2, &mut source2); assert!(result2.is_err()); // Test exactly at maximum bound let parser3 = Until::with_max(Literal::from_str("."), 3); let mut input3 = Cursor::new(b"12345678.rest"); let mut source3 = crate::parser::Source::new(&mut input3); let result3 = parse(parser3, &mut source3).unwrap(); assert_eq!(result3, b"123".to_vec()); // Stops at max } #[test] fn test_until_very_large_content() { // Test with reasonably large content let large_content = b"x".repeat(10000); let mut input_bytes = Vec::new(); input_bytes.extend_from_slice(&large_content); input_bytes.extend_from_slice(b"END"); let parser = Until::new(Literal::from_str("END")); let mut input = Cursor::new(&input_bytes); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result.len(), 10000); assert!(result.iter().all(|&b| b == b'x')); } #[test] fn test_utf8_until_complex_end_condition() { use crate::utf8class::Utf8Class; // End condition is Unicode digits let end_parser = Utf8Class::unicode_digits(); let parser = Utf8Until::new(end_parser); let mut input = Cursor::new("Hello 世界 ১২৩ more text".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, "Hello 世界 "); } #[test] fn test_utf8_until_mixed_content() { // Test with mixed ASCII, Latin-1, and Unicode content let parser = Utf8Until::new(Literal::from_str("🔚")); let mut input = Cursor::new("ASCII café 世界 привет 🔚 end".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, "ASCII café 世界 привет "); // Should be positioned at the emoji let remaining_bytes = source.peek(8).unwrap(); assert_eq!(remaining_bytes, "🔚 end".as_bytes()); } #[test] fn test_until_position_after_parsing() { // Verify position tracking through complex scenarios let parser = Until::new(Literal::from_str("||")); let mut input = Cursor::new(b"item1|item2||item3|item4"); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); assert_eq!(result, b"item1|item2".to_vec()); // Should be positioned at "||" let delimiter = source.peek(2).unwrap(); assert_eq!(delimiter, b"||"); // Advance past delimiter source.advance(2); // Should now be at "item3" let remaining = source.peek(5).unwrap(); assert_eq!(remaining, b"item3"); } #[test] fn test_until_with_empty_end_condition() { use crate::literal::Literal; // Create a literal that matches empty string (this might be a special case) let empty_literal = Literal::from_str(""); let parser = Until::new(empty_literal); let mut input = Cursor::new(b"some content"); let mut source = crate::parser::Source::new(&mut input); // This should immediately match at the start since empty string matches anywhere let result = parse(parser, &mut source).unwrap(); assert_eq!(result, Vec::::new()); } #[test] fn test_utf8_until_character_counting_accuracy() { // Verify character counting vs byte counting let parser = Utf8Until::with_max(Literal::from_str("!"), 3); // "🇺🇸" is 8 bytes but 1 grapheme cluster, "é" is 2 bytes but 1 character let mut input = Cursor::new("🇺🇸é!more".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let result = parse(parser, &mut source).unwrap(); // Should get the flag emoji, é, and then stop due to max chars = 3 // But this might depend on how Unicode characters are counted println!("Result: '{}', len: {}", result, result.chars().count()); } }