utf8class.rs
raw
//! UTF-8 character class parsing with Unicode support.
//!
//! This module provides the [`Utf8Class`] parser for matching sequences of UTF-8
//! characters that belong to a specific character class. Unlike the byte-level
//! [`Class`](crate::class::Class) parser, this handles full Unicode characters correctly.
//!
//! The parser includes built-in support for Unicode-aware character classes
//! such as alphabetic characters, digits, alphanumeric, and whitespace across
//! all Unicode categories. It uses efficient UTF-8 decoding with ASCII
//! optimization for performance.
use std::collections::HashSet;
use crate::{
cache::ParsingCache,
parser::{Parsable, Parser, Source},
result::{Error, ParseResult},
utf8util::read_utf8_char,
};
/// A parser that matches a sequence of UTF-8 characters that are all present in a character class.
///
/// The Utf8Class parser consumes UTF-8 encoded characters from the input as long as each
/// character is present in the specified set. It returns a `String` containing all matched
/// characters. The parser succeeds even if it matches zero characters (unless a minimum
/// length is specified).
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// // Match digits: parses "123" from "123abc"
/// let digits = Utf8Class::new("0123456789");
/// let mut input1 = Cursor::new("123abc".as_bytes());
/// let mut source1 = Source::new(input1);
/// let result1 = parse(digits, &mut source1).unwrap();
/// assert_eq!(result1, "123");
///
/// // Match whitespace including Unicode spaces
/// let whitespace = Utf8Class::new(" \t\r\n\u{00A0}\u{2000}\u{2001}");
/// let mut input2 = Cursor::new(" \t\u{00A0}hello".as_bytes());
/// let mut source2 = Source::new(input2);
/// let result2 = parse(whitespace, &mut source2).unwrap();
/// assert_eq!(result2, " \t\u{00A0}");
///
/// // Match emoji
/// let emoji = Utf8Class::new("😀😁😂🤣😃😄😅");
/// let mut input3 = Cursor::new("😀😁😂abc".as_bytes());
/// let mut source3 = Source::new(input3);
/// let result3 = parse(emoji, &mut source3).unwrap();
/// assert_eq!(result3, "😀😁😂");
/// ```
#[derive(Debug, Clone)]
pub struct Utf8Class<F = fn(char) -> bool> {
allowed: HashSet<char>,
predicate: Option<F>,
min_length: usize,
max_length: Option<usize>,
negated: bool,
}
impl<F> PartialEq for Utf8Class<F> {
fn eq(&self, other: &Self) -> bool {
self.allowed == other.allowed
&& self.min_length == other.min_length
&& self.max_length == other.max_length
&& self.negated == other.negated
// Note: We can't compare function pointers reliably, so we ignore predicate
}
}
impl<F> Eq for Utf8Class<F> {}
impl Utf8Class<fn(char) -> bool> {
/// Create a new Utf8Class parser that matches any character in the given string.
///
/// The parser will match zero or more characters that are present in the class.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let digits = Utf8Class::new("0123456789");
///
/// // Matches: "", "1", "123", "999999", etc.
/// let mut input1 = Cursor::new("123abc".as_bytes());
/// let mut source1 = Source::new(input1);
/// let result1 = parse(digits, &mut source1).unwrap();
/// assert_eq!(result1, "123");
///
/// // Stops at first non-digit
/// let digits2 = Utf8Class::new("0123456789");
/// let mut input2 = Cursor::new("abc123".as_bytes());
/// let mut source2 = Source::new(input2);
/// let result2 = parse(digits2, &mut source2).unwrap();
/// assert_eq!(result2, ""); // matches empty string
/// ```
pub fn new(chars: &str) -> Self {
Self {
allowed: chars.chars().collect(),
predicate: None,
min_length: 0,
max_length: None,
negated: false,
}
}
/// Create a new Utf8Class parser that matches any character NOT in the given string.
///
/// The parser will match zero or more characters that are NOT present in the class.
/// This uses the negation flag instead of predicates for better performance.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let not_digits = Utf8Class::not_in("0123456789");
///
/// // Matches: "abc", "hello", "!@#", etc.
/// let mut input1 = Cursor::new("abc123".as_bytes());
/// let mut source1 = Source::new(input1);
/// let result1 = parse(not_digits, &mut source1).unwrap();
/// assert_eq!(result1, "abc");
///
/// // Stops at first digit
/// let not_digits2 = Utf8Class::not_in("0123456789");
/// let mut input2 = Cursor::new("123abc".as_bytes());
/// let mut source2 = Source::new(input2);
/// let result2 = parse(not_digits2, &mut source2).unwrap();
/// assert_eq!(result2, ""); // matches empty string
/// ```
pub fn not_in(chars: &str) -> Utf8Class {
Utf8Class {
allowed: chars.chars().collect(),
predicate: None,
min_length: 0,
max_length: None,
negated: true,
}
}
/// Create a new Utf8Class parser that matches any character NOT in the given string
/// with a minimum required length.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let not_digits = Utf8Class::not_in_with_min("0123456789", 2);
///
/// // Must match at least 2 non-digit characters
/// let mut input = Cursor::new("abc123".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(not_digits, &mut source).unwrap();
/// assert_eq!(result, "abc");
/// ```
pub fn not_in_with_min(chars: &str, min_length: usize) -> Utf8Class {
Utf8Class {
allowed: chars.chars().collect(),
predicate: None,
min_length,
max_length: None,
negated: true,
}
}
/// Create a new Utf8Class parser with a minimum required length.
///
/// The parser must match at least `min_length` characters or it fails.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let digits = Utf8Class::with_min("0123456789", 2);
///
/// // Matches: "12", "999", "12345", etc.
/// let mut input1 = Cursor::new("123abc".as_bytes());
/// let mut source1 = Source::new(input1);
/// let result1 = parse(digits, &mut source1).unwrap();
/// assert_eq!(result1, "123");
///
/// // Fails on: "", "1"
/// let digits2 = Utf8Class::with_min("0123456789", 2);
/// let mut input2 = Cursor::new("1abc".as_bytes());
/// let mut source2 = Source::new(input2);
/// let result2 = parse(digits2, &mut source2);
/// assert!(result2.is_err()); // fails because only 1 digit
/// ```
pub fn with_min(chars: &str, min_length: usize) -> Self {
Self {
allowed: chars.chars().collect(),
predicate: None,
min_length,
max_length: None,
negated: false,
}
}
/// Create a new Utf8Class parser with a maximum length limit.
///
/// The parser will stop after matching `max_length` characters, even if more
/// matching characters are available.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let digits = Utf8Class::with_max("0123456789", 3);
///
/// // From "12345", matches "123" and stops
/// let mut input = Cursor::new("12345abc".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(digits, &mut source).unwrap();
/// assert_eq!(result, "123");
/// ```
pub fn with_max(chars: &str, max_length: usize) -> Self {
Self {
allowed: chars.chars().collect(),
predicate: None,
min_length: 0,
max_length: Some(max_length),
negated: false,
}
}
/// Create a new Utf8Class parser with both minimum and maximum length limits.
///
/// The parser must match at least `min_length` characters and will stop after
/// `max_length` characters.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let digits = Utf8Class::with_bounds("0123456789", 2, 4);
///
/// // Matches 2-4 digits: "12", "123", "1234"
/// let mut input1 = Cursor::new("123abc".as_bytes());
/// let mut source1 = Source::new(input1);
/// let result1 = parse(digits, &mut source1).unwrap();
/// assert_eq!(result1, "123");
///
/// // Stops at 4 even from "123456"
/// let digits2 = Utf8Class::with_bounds("0123456789", 2, 4);
/// let mut input2 = Cursor::new("123456abc".as_bytes());
/// let mut source2 = Source::new(input2);
/// let result2 = parse(digits2, &mut source2).unwrap();
/// assert_eq!(result2, "1234");
/// ```
pub fn with_bounds(chars: &str, min_length: usize, max_length: usize) -> Self {
Self {
allowed: chars.chars().collect(),
predicate: None,
min_length,
max_length: Some(max_length),
negated: false,
}
}
/// Create a Utf8Class parser for ASCII digits (0-9).
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let digits = Utf8Class::digits();
/// // Equivalent to: Utf8Class::with_min("0123456789", 1)
///
/// let mut input = Cursor::new("123abc".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(digits, &mut source).unwrap();
/// assert_eq!(result, "123");
/// ```
pub fn digits() -> Self {
Self::with_min("0123456789", 1)
}
/// Create a Utf8Class parser for ASCII alphabetic characters (a-z, A-Z).
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let alpha = Utf8Class::alpha();
///
/// // Matches: "abc", "XYZ", "Hello", etc.
/// let mut input = Cursor::new("Hello123".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(alpha, &mut source).unwrap();
/// assert_eq!(result, "Hello");
/// ```
pub fn alpha() -> Self {
Self::with_min("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", 1)
}
/// Create a Utf8Class parser for ASCII alphanumeric characters (a-z, A-Z, 0-9).
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let alnum = Utf8Class::alphanumeric();
///
/// // Matches: "abc123", "Hello42", etc.
/// let mut input = Cursor::new("Hello42_world".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(alnum, &mut source).unwrap();
/// assert_eq!(result, "Hello42");
/// ```
pub fn alphanumeric() -> Self {
Self::with_min(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
1,
)
}
/// Create a Utf8Class parser for ASCII whitespace characters.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let whitespace = Utf8Class::whitespace();
///
/// // Matches: " ", "\t\n", " ", etc.
/// let mut input = Cursor::new(" \t\r\nhello".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(whitespace, &mut source).unwrap();
/// assert_eq!(result, " \t\r\n");
/// ```
pub fn whitespace() -> Self {
Self::with_min(" \t\r\n\u{0b}\u{0c}", 1)
}
/// Create a Utf8Class parser for hexadecimal digits (0-9, a-f, A-F).
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let hex = Utf8Class::hex_digits();
///
/// // Matches: "1a2b", "DEADBEEF", "0xff", etc.
/// let mut input = Cursor::new("DEADBEEFghij".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(hex, &mut source).unwrap();
/// assert_eq!(result, "DEADBEEF");
/// ```
pub fn hex_digits() -> Self {
Self::with_min("0123456789abcdefABCDEF", 1)
}
/// Create a Utf8Class parser for Unicode letters (using char::is_alphabetic).
///
/// This matches Unicode letters including accented characters, Greek letters, etc.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let unicode_alpha = Utf8Class::unicode_alpha();
///
/// // Matches: "café", "αβγ", "こんにちは", etc.
/// let mut input = Cursor::new("café123".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(unicode_alpha, &mut source).unwrap();
/// assert_eq!(result, "café");
/// ```
pub fn unicode_alpha() -> Self {
Self::from_predicate_min(char::is_alphabetic, 1)
}
/// Create a Utf8Class parser for Unicode digits (using char::is_numeric).
///
/// This matches Unicode numeric characters including Arabic-Indic digits, etc.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let unicode_digits = Utf8Class::unicode_digits();
///
/// // Matches: "123", "۱۲۳", "１２３", etc.
/// let mut input = Cursor::new("１２３abc".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(unicode_digits, &mut source).unwrap();
/// assert_eq!(result, "１２３");
/// ```
pub fn unicode_digits() -> Self {
Self::from_predicate_min(char::is_numeric, 1)
}
/// Create a Utf8Class parser for Unicode whitespace (using char::is_whitespace).
///
/// This matches all Unicode whitespace characters including various space types.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let unicode_ws = Utf8Class::unicode_whitespace();
///
/// // Matches: " ", "\u{00A0}", "\u{2000}", etc.
/// let mut input = Cursor::new(" \u{00A0}\u{2000}hello".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(unicode_ws, &mut source).unwrap();
/// assert_eq!(result, " \u{00A0}\u{2000}");
/// ```
pub fn unicode_whitespace() -> Self {
Self::from_predicate_min(char::is_whitespace, 1)
}
}
impl<F> Utf8Class<F>
where
F: Fn(char) -> bool,
{
/// Create a new Utf8Class parser that uses a predicate function to test characters.
///
/// The predicate function will be called for each character to determine if it
/// should be matched. This allows for complex matching logic.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// // Match uppercase letters
/// let uppercase = Utf8Class::from_predicate(|c| c.is_uppercase());
/// let mut input1 = Cursor::new("HELLO world".as_bytes());
/// let mut source1 = Source::new(input1);
/// let result1 = parse(uppercase, &mut source1).unwrap();
/// assert_eq!(result1, "HELLO");
///
/// // Match vowels
/// let vowels = Utf8Class::from_predicate(|c| matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'A' | 'E' | 'I' | 'O' | 'U'));
/// let mut input2 = Cursor::new("aeiou123".as_bytes());
/// let mut source2 = Source::new(input2);
/// let result2 = parse(vowels, &mut source2).unwrap();
/// assert_eq!(result2, "aeiou");
/// ```
pub fn from_predicate(predicate: F) -> Self {
Self {
allowed: HashSet::new(), // Ignored when predicate is present
predicate: Some(predicate),
min_length: 0,
max_length: None,
negated: false,
}
}
/// Create a Utf8Class parser with a predicate and minimum length.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let uppercase = Utf8Class::from_predicate_min(|c| c.is_uppercase(), 2);
///
/// // Must match at least 2 uppercase letters
/// let mut input = Cursor::new("HELLO world".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(uppercase, &mut source).unwrap();
/// assert_eq!(result, "HELLO");
/// ```
pub fn from_predicate_min(predicate: F, min_length: usize) -> Self {
Self {
allowed: HashSet::new(),
predicate: Some(predicate),
min_length,
max_length: None,
negated: false,
}
}
/// Create a Utf8Class parser with a predicate and maximum length.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let uppercase = Utf8Class::from_predicate_max(|c| c.is_uppercase(), 5);
///
/// // Match at most 5 uppercase letters
/// let mut input = Cursor::new("HELLOWORLD123".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(uppercase, &mut source).unwrap();
/// assert_eq!(result, "HELLO"); // stops at 5 letters
/// ```
pub fn from_predicate_max(predicate: F, max_length: usize) -> Self {
Self {
allowed: HashSet::new(),
predicate: Some(predicate),
min_length: 0,
max_length: Some(max_length),
negated: false,
}
}
/// Create a Utf8Class parser with a predicate and both min/max length.
///
/// # Examples
///
/// ```rust
/// use neotoma::{utf8class::Utf8Class, parser::{parse, Source}};
/// use std::io::Cursor;
///
/// let uppercase = Utf8Class::from_predicate_bounds(|c| c.is_uppercase(), 2, 5);
///
/// // Match 2-5 uppercase letters
/// let mut input = Cursor::new("HELLOWORLD123".as_bytes());
/// let mut source = Source::new(input);
/// let result = parse(uppercase, &mut source).unwrap();
/// assert_eq!(result, "HELLO"); // matches 5 letters then stops
/// ```
pub fn from_predicate_bounds(predicate: F, min_length: usize, max_length: usize) -> Self {
Self {
allowed: HashSet::new(),
predicate: Some(predicate),
min_length,
max_length: Some(max_length),
negated: false,
}
}
}
impl<F> Utf8Class<F> {
/// Check if a character matches this class
fn char_matches(&self, c: char) -> bool
where
F: Fn(char) -> bool,
{
let matches = if let Some(ref predicate) = self.predicate {
predicate(c)
} else {
self.allowed.contains(&c)
};
if self.negated { !matches } else { matches }
}
}
impl<F, Ctx> Parser<Ctx> for Utf8Class<F>
where
F: Fn(char) -> bool + 'static,
{
type Output = String;
fn id(&self) -> u64 {
use std::any::TypeId;
use std::hash::{DefaultHasher, Hash, Hasher};
let mut hasher = DefaultHasher::new();
TypeId::of::<Self>().hash(&mut hasher);
// Hash the allowed character set for character set-based classes
// We need to hash the contents of the set in a deterministic way
let mut chars: Vec<char> = self.allowed.iter().copied().collect();
chars.sort(); // Ensure deterministic ordering
chars.hash(&mut hasher);
self.min_length.hash(&mut hasher);
self.max_length.hash(&mut hasher);
self.negated.hash(&mut hasher);
// Note: predicate functions can't be hashed directly, but the allowed set
// captures the essential state for non-predicate classes
hasher.finish()
}
fn read<S>(
&self,
source: &mut Source<S>,
_cache: &mut impl ParsingCache,
_context: &mut Ctx,
) -> ParseResult<Self::Output>
where
S: Parsable,
{
let mut result = String::new();
let mut char_count = 0;
loop {
// Check max length limit
if let Some(max_length) = self.max_length {
if char_count >= max_length {
break;
}
}
// Check if the next character matches before consuming it
source.push();
match read_utf8_char(source) {
Ok(c) => {
if self.char_matches(c) {
// Character is in our class, consume it
source.commit(); // Commit the read
result.push(c);
char_count += 1;
} else {
// Character not in class, backtrack and stop matching
source.pop(); // Backtrack
break;
}
}
Err(Error::NoMatch) => {
// End of input or invalid UTF-8, stop matching
source.pop(); // Backtrack
break;
}
Err(err) => {
source.pop(); // Backtrack
return Err(err);
}
}
}
// Check minimum length requirement
if char_count < self.min_length {
return Err(Error::NoMatch);
}
Ok(result)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::parse;
use std::io::Cursor;
#[test]
fn test_id_implementation_different_utf8class_parsers() {
// Test that Utf8Class implements proper id() method
// Different Utf8Class parsers should have different IDs to avoid cache conflicts
let class1 = Utf8Class::new("abc");
let class2 = Utf8Class::new("xyz");
// These should have different IDs because they have different character sets
// This test will FAIL if Utf8Class uses default id() implementation
assert_ne!(
<Utf8Class as crate::parser::Parser<()>>::id(&class1),
<Utf8Class as crate::parser::Parser<()>>::id(&class2),
"Different Utf8Class instances should have different IDs to avoid cache collisions"
);
}
#[test]
fn test_id_implementation_same_utf8class_parsers() {
// Test that identical Utf8Class parsers have the same ID
let class1 = Utf8Class::new("abc");
let class2 = Utf8Class::new("abc");
assert_eq!(
<Utf8Class as crate::parser::Parser<()>>::id(&class1),
<Utf8Class as crate::parser::Parser<()>>::id(&class2),
"Identical Utf8Class instances should have the same ID for cache efficiency"
);
}
#[test]
fn test_id_implementation_utf8class_different_bounds() {
// Test Utf8Class parsers with different bounds
let class1 = Utf8Class::with_min("abc", 1);
let class2 = Utf8Class::with_min("abc", 2);
// These should have different IDs because they have different minimum bounds
// This test will FAIL if Utf8Class uses default id() implementation
assert_ne!(
<Utf8Class as crate::parser::Parser<()>>::id(&class1),
<Utf8Class as crate::parser::Parser<()>>::id(&class2),
"Utf8Class instances with different bounds should have different IDs"
);
}
#[test]
fn test_id_implementation_utf8class_predicate_parsers() {
// Test Utf8Class parsers with predicates
let class1 = Utf8Class::from_predicate(|c| c.is_alphabetic());
let class2 = Utf8Class::from_predicate(|c| c.is_numeric());
// These should have different IDs because they have different predicates
// This test will FAIL if Utf8Class uses default id() implementation
// Note: This test is complex because function pointers might not be easily comparable
// Helper function to get ID with proper type inference
fn get_id<P: crate::parser::Parser<()>>(parser: &P) -> u64 {
parser.id()
}
assert_ne!(
get_id(&class1),
get_id(&class2),
"Utf8Class instances with different predicates should have different IDs"
);
}
#[test]
fn test_utf8class_basic_functionality() {
// Basic functionality test to ensure Utf8Class works correctly
let class = Utf8Class::new("abc");
let mut input = Cursor::new("aabbcc123".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
assert_eq!(result, "aabbcc");
}
#[test]
fn test_utf8class_unicode_characters() {
// Test with various Unicode character ranges
let emoji_class = Utf8Class::new("😀😁😂🤣😃");
let mut input = Cursor::new("😀😁😂abc".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(emoji_class, &mut source).unwrap();
assert_eq!(result, "😀😁😂");
// Test with mixed scripts
let mixed_class = Utf8Class::new("aαа世界");
let mut input2 = Cursor::new("aα世abc".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(mixed_class, &mut source2).unwrap();
// The class "aαа世界" should match all characters that are in the set
assert!(result2.contains('a') && result2.contains('α') && result2.contains('世'));
}
#[test]
fn test_utf8class_negated_functionality() {
// Test not_in functionality
let not_digits = Utf8Class::not_in("0123456789");
let mut input = Cursor::new("abc123".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(not_digits, &mut source).unwrap();
assert_eq!(result, "abc");
// Test not_in with Unicode
let not_emoji = Utf8Class::not_in("😀😁😂🤣");
let mut input2 = Cursor::new("Hello世界😀more".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(not_emoji, &mut source2).unwrap();
assert_eq!(result2, "Hello世界");
}
#[test]
fn test_utf8class_boundary_conditions() {
// Test min/max bounds with Unicode characters
let unicode_class = Utf8Class::with_bounds("世界测试", 2, 3);
// Should succeed with 3 characters
let mut input1 = Cursor::new("世界测试abc".as_bytes());
let mut source1 = crate::parser::Source::new(&mut input1);
let result1 = parse(unicode_class, &mut source1).unwrap();
assert_eq!(result1, "世界测"); // Stops at max of 3
// Test failure below minimum
let unicode_class2 = Utf8Class::with_min("世界", 3);
let mut input2 = Cursor::new("世界abc".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(unicode_class2, &mut source2);
assert!(result2.is_err()); // Only 2 characters, need 3
}
#[test]
fn test_utf8class_predicate_functionality() {
// Test various Unicode predicates
let alpha_class = Utf8Class::from_predicate(char::is_alphabetic);
let mut input1 = Cursor::new("Helloαβγ世界123".as_bytes());
let mut source1 = crate::parser::Source::new(&mut input1);
let result1 = parse(alpha_class, &mut source1).unwrap();
assert_eq!(result1, "Helloαβγ世界");
// Test numeric predicate
let numeric_class = Utf8Class::from_predicate(char::is_numeric);
let mut input2 = Cursor::new("１２３456abc".as_bytes()); // Unicode + ASCII digits
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(numeric_class, &mut source2).unwrap();
assert_eq!(result2, "１２３456");
// Test whitespace predicate
let ws_class = Utf8Class::from_predicate(char::is_whitespace);
let mut input3 = Cursor::new(" \t\u{00A0}\u{2000}abc".as_bytes());
let mut source3 = crate::parser::Source::new(&mut input3);
let result3 = parse(ws_class, &mut source3).unwrap();
assert_eq!(result3, " \t\u{00A0}\u{2000}");
}
#[test]
fn test_utf8class_convenience_constructors() {
// Test convenience constructors
let digits = Utf8Class::digits();
let mut input1 = Cursor::new("123abc".as_bytes());
let mut source1 = crate::parser::Source::new(&mut input1);
let result1 = parse(digits, &mut source1).unwrap();
assert_eq!(result1, "123");
// Test alpha constructor
let alpha = Utf8Class::alpha();
let mut input2 = Cursor::new("Hello123".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(alpha, &mut source2).unwrap();
assert_eq!(result2, "Hello");
// Test unicode_alpha constructor
let unicode_alpha = Utf8Class::unicode_alpha();
let mut input3 = Cursor::new("Helloαβγ世界123".as_bytes());
let mut source3 = crate::parser::Source::new(&mut input3);
let result3 = parse(unicode_alpha, &mut source3).unwrap();
assert_eq!(result3, "Helloαβγ世界");
// Test unicode_digits constructor
let unicode_digits = Utf8Class::unicode_digits();
let mut input4 = Cursor::new("১২৩123abc".as_bytes()); // Bengali + ASCII digits
let mut source4 = crate::parser::Source::new(&mut input4);
let result4 = parse(unicode_digits, &mut source4).unwrap();
assert_eq!(result4, "১২৩123");
}
#[test]
fn test_utf8class_empty_input() {
// Test with empty input
let class = Utf8Class::new("abc");
let mut input = Cursor::new("".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
assert_eq!(result, "");
// Test with minimum requirement on empty input
let class_min = Utf8Class::with_min("abc", 1);
let mut input2 = Cursor::new("".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(class_min, &mut source2);
assert!(result2.is_err());
}
#[test]
fn test_utf8class_position_tracking() {
// Verify position is correctly tracked with Unicode characters
let class = Utf8Class::new("世界");
let mut input = Cursor::new("世界测XYZ".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
assert_eq!(result, "世界");
// Position should be at '测'
let next_char = crate::utf8util::read_utf8_char(&mut source).unwrap();
assert_eq!(next_char, '测');
}
#[test]
fn test_utf8class_complex_unicode_scenarios() {
// Test with combining characters
let class = Utf8Class::new("eé\u{0301}"); // e, é, and combining acute
let mut input = Cursor::new("eée\u{0301}abc".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
// Note: This tests individual Unicode code points, not grapheme clusters
assert!(result.contains('e') && result.contains('é'));
// Test with surrogate pairs / 4-byte UTF-8
let emoji_class = Utf8Class::new("🌍🌎🌏");
let mut input2 = Cursor::new("🌍🌎abc".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(emoji_class, &mut source2).unwrap();
assert_eq!(result2, "🌍🌎");
}
#[test]
fn test_utf8class_large_character_set() {
// Test with a large Unicode character set
let large_set = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZαβγδεζηθικλμνξοπρστυφχψωабвгдежзийклмнопрстуфхцчшщъыьэюя世界测试日本語한국어Hello";
let class = Utf8Class::new(large_set);
let mut input = Cursor::new("Hello世界αβ한국123".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
// Should match all characters that are in the large set
assert!(!result.is_empty());
assert!(result.contains("Hello"));
assert!(result.contains("世界"));
}
#[test]
fn test_utf8class_invalid_utf8_handling() {
// Test behavior with invalid UTF-8 sequences
let class = Utf8Class::from_predicate(|_| true); // Accept any valid character
// Create input with invalid UTF-8 in the middle
let mut invalid_input = Vec::new();
invalid_input.extend_from_slice("Hello".as_bytes());
invalid_input.push(0xFF); // Invalid UTF-8 byte
invalid_input.extend_from_slice("World".as_bytes());
let mut input = Cursor::new(&invalid_input);
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
assert_eq!(result, "Hello"); // Should stop at invalid UTF-8
}
#[test]
fn test_utf8class_predicate_edge_cases() {
// Predicate that always returns true
let always_true = Utf8Class::from_predicate(|_| true);
let mut input1 = Cursor::new("Hello世界123!@#".as_bytes());
let mut source1 = crate::parser::Source::new(&mut input1);
let result1 = parse(always_true, &mut source1).unwrap();
assert_eq!(result1, "Hello世界123!@#");
// Predicate that always returns false
let always_false = Utf8Class::from_predicate(|_| false);
let mut input2 = Cursor::new("Hello".as_bytes());
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(always_false, &mut source2).unwrap();
assert_eq!(result2, "");
// Complex predicate: vowels (ASCII and some Unicode)
let vowels = Utf8Class::from_predicate(|c| {
matches!(
c,
'a' | 'e'
| 'i'
| 'o'
| 'u'
| 'A'
| 'E'
| 'I'
| 'O'
| 'U'
| 'α'
| 'ε'
| 'ι'
| 'ο'
| 'υ'
)
});
let mut input3 = Cursor::new("aeiouαεβγ123".as_bytes());
let mut source3 = crate::parser::Source::new(&mut input3);
let result3 = parse(vowels, &mut source3).unwrap();
assert_eq!(result3, "aeiouαε");
}
#[test]
fn test_utf8class_character_vs_byte_counting() {
// Verify character counting vs byte counting
let class = Utf8Class::with_max("🌍🌎🌏abc", 3);
// "🌍🌎🌏" are each 4 bytes but should count as 3 characters
let mut input = Cursor::new("🌍🌎🌏abc".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
assert_eq!(result, "🌍🌎🌏"); // Should get all 3 emoji characters
// Test with min requirement
let class2 = Utf8Class::with_min("🌍", 2);
let mut input2 = Cursor::new("🌍abc".as_bytes()); // Only 1 character
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse(class2, &mut source2);
assert!(result2.is_err()); // Should fail - only 1 character, need 2
}
#[test]
fn test_utf8class_performance_large_input() {
// Test with reasonably large Unicode input
let large_unicode = "世界".repeat(5000);
let class = Utf8Class::new("世界");
let mut input = Cursor::new(large_unicode.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(class, &mut source).unwrap();
assert_eq!(result.chars().count(), 10000); // 5000 * 2 characters
assert!(result.chars().all(|c| c == '世' || c == '界'));
}
#[test]
fn test_utf8class_mixed_ascii_unicode() {
// Test realistic mixed ASCII/Unicode scenarios
let mixed_class = Utf8Class::new("Hello世界-_123");
let mut input = Cursor::new("Hello-世界_123!@#".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(mixed_class, &mut source).unwrap();
assert_eq!(result, "Hello-世界_123");
// Remaining should be "!@#"
let remaining_bytes = source.peek(3).unwrap();
assert_eq!(remaining_bytes, "!@#".as_bytes());
}
}