//! UTF-8 decoding utilities for character-level parsing. //! //! This module provides low-level UTF-8 decoding functionality used by //! character-aware parsers like [`Utf8Class`](crate::utf8class::Utf8Class) and [`Utf8Until`](crate::until::Utf8Until). It includes //! manual UTF-8 validation and decoding with optimizations for ASCII characters. //! //! The utilities handle proper UTF-8 validation, preventing overlong encodings //! and ensuring continuation bytes are correctly formatted. ASCII characters //! are handled through a fast path for optimal performance. use crate::{ parser::{Parsable, Source}, result::Error, }; /// Reads a single UTF-8 character from the source, returning the character. /// /// This function handles UTF-8 decoding with proper validation and error handling. /// It includes optimizations for ASCII characters and manual decoding for multi-byte /// UTF-8 sequences. /// /// # Examples /// /// ```rust /// use neotoma::utf8util::read_utf8_char; /// use neotoma::parser::Source; /// use std::io::Cursor; /// /// let mut input = Cursor::new("Hello 世界".as_bytes()); /// let mut source = Source::new(&mut input); /// /// let ch1 = read_utf8_char(&mut source).unwrap(); /// assert_eq!(ch1, 'H'); /// /// // Skip "ello " (5 characters) /// for _ in 0..5 { read_utf8_char(&mut source).unwrap(); } /// let ch2 = read_utf8_char(&mut source).unwrap(); /// assert_eq!(ch2, '世'); /// ``` pub fn read_utf8_char(source: &mut Source) -> Result where S: Parsable, { // Read the first byte to determine character length let first_byte = source.peek1()?; // Fast path for ASCII characters - handle immediately if first_byte & 0x80 == 0 { source.advance(1); return Ok(first_byte as char); } // Determine multi-byte character length let char_len = if first_byte & 0xE0 == 0xC0 { // 2-byte character (110xxxxx) 2 } else if first_byte & 0xF0 == 0xE0 { // 3-byte character (1110xxxx) 3 } else if first_byte & 0xF8 == 0xF0 { // 4-byte character (11110xxx) 4 } else { // Invalid UTF-8 start byte return Err(Error::NoMatch); }; // Read the full character bytes for multi-byte sequences let bytes = source.read(char_len)?; // Validate continuation bytes and decode manually for better performance let mut code_point = (first_byte & (0xFF >> (char_len + 1))) as u32; for byte in bytes.iter().take(char_len).skip(1) { if byte & 0xC0 != 0x80 { // Invalid continuation byte return Err(Error::NoMatch); } code_point = (code_point << 6) | ((byte & 0x3F) as u32); } // Check for overlong encodings let min_code_point = match char_len { 2 => 0x80, 3 => 0x800, 4 => 0x10000, _ => 0, }; if code_point < min_code_point { // This is an overlong encoding return Err(Error::NoMatch); } // Convert to char (this validates the code point) char::from_u32(code_point).ok_or(Error::NoMatch) } #[cfg(test)] mod tests { use super::*; use std::io::Cursor; #[test] fn test_read_ascii_char() { let mut input = Cursor::new(b"Hello"); let mut source = crate::parser::Source::new(&mut input); let ch = read_utf8_char(&mut source).unwrap(); assert_eq!(ch, 'H'); } #[test] fn test_read_two_byte_utf8() { let mut input = Cursor::new("café".as_bytes()); let mut source = crate::parser::Source::new(&mut input); // Skip 'c', 'a' read_utf8_char(&mut source).unwrap(); read_utf8_char(&mut source).unwrap(); let ch = read_utf8_char(&mut source).unwrap(); assert_eq!(ch, 'f'); let ch = read_utf8_char(&mut source).unwrap(); assert_eq!(ch, 'é'); // 2-byte UTF-8 } #[test] fn test_read_three_byte_utf8() { let mut input = Cursor::new("世界".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let ch1 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch1, '世'); // 3-byte UTF-8 let ch2 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch2, '界'); // 3-byte UTF-8 } #[test] fn test_read_four_byte_utf8() { let mut input = Cursor::new("👋🌍".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let ch1 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch1, '👋'); // 4-byte UTF-8 (emoji) let ch2 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch2, '🌍'); // 4-byte UTF-8 (emoji) } #[test] fn test_read_mixed_utf8() { let mut input = Cursor::new("A世👋".as_bytes()); let mut source = crate::parser::Source::new(&mut input); let ch1 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch1, 'A'); // ASCII let ch2 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch2, '世'); // 3-byte UTF-8 let ch3 = read_utf8_char(&mut source).unwrap(); assert_eq!(ch3, '👋'); // 4-byte UTF-8 } #[test] fn test_invalid_utf8_start_byte() { let mut input = Cursor::new(b"\xFF\xFE"); let mut source = crate::parser::Source::new(&mut input); let result = read_utf8_char(&mut source); assert!(result.is_err()); } #[test] fn test_invalid_continuation_byte() { // 0xC2 expects a continuation byte, but 0xFF is not valid let mut input = Cursor::new(b"\xC2\xFF"); let mut source = crate::parser::Source::new(&mut input); let result = read_utf8_char(&mut source); assert!(result.is_err()); } #[test] fn test_truncated_utf8() { // 0xC2 expects a continuation byte, but input ends let mut input = Cursor::new(b"\xC2"); let mut source = crate::parser::Source::new(&mut input); let result = read_utf8_char(&mut source); assert!(result.is_err()); } #[test] fn test_overlong_encoding() { // Overlong encoding of 'A' (should be 0x41, not 0xC1 0x81) let mut input = Cursor::new(b"\xC1\x81"); let mut source = crate::parser::Source::new(&mut input); let result = read_utf8_char(&mut source); assert!(result.is_err()); } #[test] fn test_empty_input() { let mut input = Cursor::new(b""); let mut source = crate::parser::Source::new(&mut input); let result = read_utf8_char(&mut source); assert!(result.is_err()); } }