grammar.rs
raw
//! # Grammar Syntax
//! - `"terminal"` - matches literal string (with `\"` for escaped quotes)
//! - `digits` - matches one or more decimal digits (0-9)
//! - `alpha` - matches one or more alphabetic characters (ASCII)
//! - `alphanumeric` - matches one or more alphanumeric characters (ASCII)
//! - `whitespace` - matches one or more whitespace characters (ASCII)
//! - `udigits` - matches one or more decimal digits (unicode)
//! - `ualpha` - matches one or more alphabetic characters (unicode)
//! - `ualphanumeric` - matches one or more alphanumeric characters (unicode)
//! - `uwhitespace` - matches one or more whitespace characters (unicode)
//! - `hexdigits` - matches one or more hexadecimal digits (0-9, a-f, A-F)
//! - `eof` - matches end of file (ensures complete input consumption)
//! - `[abc]` - matches any character in the set (custom character class)
//! - `[^abc]` - matches any character NOT in the set (negated character class)
//! - `(A B C)` - matches A followed by B followed by C (sequence)
//! - `(| A B C)` - matches either A or B or C (alternatives)
//! - `(* A)` - matches zero or more instances of A
//! - `(+ A)` - matches one or more instances of A
//! - `(? A)` - matches zero or one instances of A
//! - `(* A / B)` - matches zero or more A's separated by B (trailing B allowed but not required)
//! - `(+ A / B)` - matches one or more A's separated by B (trailing B allowed but not required)
//! - `name = rule` - Names a parsing rule, for use in other rules and/or recursive rule definitions
//! - `name` - References a named rule. The `digits`, `alpha` and so on are not valid rule names
//! - `@start name` - Identifies the starting rule for the grammar. If not provided, defaults to the last rule defined
use crate::{
either::Either, eof::EndOfFile, literal::Literal, parser::Parser, repeat::Repeat,
result::Error, sequence::Sequence, until::Utf8Until, utf8class::Utf8Class,
utf8util::read_utf8_char,
};
// Const literal parsers for grammar punctuation - avoiding runtime allocations
const OPEN_PAREN: Literal = Literal::from_str_const("(");
const CLOSE_PAREN: Literal = Literal::from_str_const(")");
const PIPE: Literal = Literal::from_str_const("|");
const ASTERISK: Literal = Literal::from_str_const("*");
const PLUS: Literal = Literal::from_str_const("+");
const QUESTION: Literal = Literal::from_str_const("?");
const LESS_THAN: Literal = Literal::from_str_const("<");
const SLASH: Literal = Literal::from_str_const("/");
const OPEN_BRACKET: Literal = Literal::from_str_const("[");
//const CLOSE_BRACKET: Literal = Literal::from_str_const("]");
const CARET: Literal = Literal::from_str_const("^");
const QUOTE: Literal = Literal::from_str_const("\"");
const AT_SYMBOL: Literal = Literal::from_str_const("@");
const EQUALS: Literal = Literal::from_str_const("=");
// Const literal parsers for new grammar keywords
const START_KEYWORD: Literal = Literal::from_str_const("start");
// Const literal parsers for grammar keywords - avoiding runtime allocations
const DIGITS_KEYWORD: Literal = Literal::from_str_const("digits");
const ALPHA_KEYWORD: Literal = Literal::from_str_const("alpha");
const ALPHANUMERIC_KEYWORD: Literal = Literal::from_str_const("alphanumeric");
const WHITESPACE_KEYWORD: Literal = Literal::from_str_const("whitespace");
const HEXDIGITS_KEYWORD: Literal = Literal::from_str_const("hexdigits");
const UDIGITS_KEYWORD: Literal = Literal::from_str_const("udigits");
const UALPHA_KEYWORD: Literal = Literal::from_str_const("ualpha");
const UALPHANUMERIC_KEYWORD: Literal = Literal::from_str_const("ualphanumeric");
const UWHITESPACE_KEYWORD: Literal = Literal::from_str_const("uwhitespace");
const EOF_KEYWORD: Literal = Literal::from_str_const("eof");
/// Context type used internally by grammar parsers.
///
/// This context allows grammar parsers to maintain state during parsing,
/// such as tracking recursion depth, variable bindings, or other grammar-specific information.
#[derive(Debug, Clone, Default)]
struct GrammarContext {
/// Map of rule names to their grammar definitions
rules: std::collections::HashMap<String, GrammarNode>,
/// Starting rule name for parsing
start_rule: Option<String>,
}
/// Parser for grammar expressions
///
/// This type is responsible for parsing grammar syntax and producing
/// a Grammar which implements a parser for the described language.
#[derive(Clone)]
pub struct GrammarParser;
impl Default for GrammarParser {
fn default() -> Self {
Self::new()
}
}
impl GrammarParser {
/// Create a new GrammarParser instance.
///
/// # Examples
///
/// ```rust
/// use neotoma::grammar::GrammarParser;
/// let parser = GrammarParser::new();
/// ```
pub fn new() -> Self {
Self
}
}
// Implementation for external calls (no context)
impl Parser<()> for GrammarParser {
type Output = Grammar;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
_context: &mut (),
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Create a new GrammarContext for accumulating rules and directives
let mut grammar_context = GrammarContext::default();
let mut last_node = None;
let ws = crate::utf8class::Utf8Class::whitespace();
// Parse multiple grammar constructs until end of input
loop {
// Skip any whitespace between constructs
let _ = ws.parse(source, cache, &mut grammar_context);
// Try to parse another grammar construct
match self.read(source, cache, &mut grammar_context) {
Ok(node) => {
last_node = Some(node);
// Continue parsing more constructs
}
Err(Error::NoMatch) => {
// No more constructs to parse, we're done
break;
}
Err(other_error) => {
// Actual parsing error, propagate it
return Err(other_error);
}
}
}
// After parsing all constructs, validate and build the final grammar
if let Some(start) = grammar_context.start_rule.as_deref() {
if let Some(rule) = grammar_context.rules.get(start).cloned() {
Ok(Grammar::new(rule, grammar_context))
} else {
// The grammar syntax is invalid, due to referencing a
// nonexistent start rule
Err(Error::NoMatch)
}
} else if let Some(last_node) = last_node {
// Use the last parsed node if no start rule specified
Ok(Grammar::new(last_node, grammar_context))
} else {
// No constructs parsed at all
Err(Error::NoMatch)
}
}
}
// Implementation for internal recursive calls (with GrammarContext)
impl Parser<GrammarContext> for GrammarParser {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Try each possible expression type in order
// First, try meta-constructs that update context
// Try @start directive
if let Ok(start_rule) = StartDirective.parse(source, cache, context) {
context.start_rule = Some(start_rule);
return Ok(GrammarNode::Empty);
}
// Try rule definition
if let Ok((name, node)) = RuleDefinition.parse(source, cache, context) {
context.rules.insert(name, node);
return Ok(GrammarNode::Empty);
}
// Then try terminals (quoted strings)
if let Ok(literal) = Terminal.parse(source, cache, context) {
return Ok(GrammarNode::Terminal(literal));
}
// Try built-in keyword types
if let Ok(digits) = Digits.parse(source, cache, context) {
return Ok(GrammarNode::Digits(digits));
}
if let Ok(alphanumeric) = Alphanumeric.parse(source, cache, context) {
return Ok(GrammarNode::Alphanumeric(alphanumeric));
}
if let Ok(alpha) = Alpha.parse(source, cache, context) {
return Ok(GrammarNode::Alpha(alpha));
}
if let Ok(whitespace) = Whitespace.parse(source, cache, context) {
return Ok(GrammarNode::Whitespace(whitespace));
}
if let Ok(udigits) = UDigits.parse(source, cache, context) {
return Ok(GrammarNode::UDigits(udigits));
}
if let Ok(ualphanumeric) = UAlphanumeric.parse(source, cache, context) {
return Ok(GrammarNode::UAlphanumeric(ualphanumeric));
}
if let Ok(ualpha) = UAlpha.parse(source, cache, context) {
return Ok(GrammarNode::UAlpha(ualpha));
}
if let Ok(uwhitespace) = UWhitespace.parse(source, cache, context) {
return Ok(GrammarNode::UWhitespace(uwhitespace));
}
if let Ok(hexdigits) = HexDigits.parse(source, cache, context) {
return Ok(GrammarNode::HexDigits(hexdigits));
}
if let Ok(eof) = EndOfFileParser.parse(source, cache, context) {
return Ok(GrammarNode::EndOfFile(eof));
}
// Try character classes [abc] and [^abc]
if let Ok(inclass) = InClass.parse(source, cache, context) {
return Ok(GrammarNode::InClass(inclass));
}
if let Ok(notinclass) = NotInClass.parse(source, cache, context) {
return Ok(GrammarNode::NotInClass(notinclass));
}
// Try alternatives syntax: (| A B C)
if let Ok(expr) = Alternatives.parse(source, cache, context) {
return Ok(expr);
}
// Try separated repetitions first (longer patterns)
// Try zero-or-more separated syntax: (* A / B)
if let Ok(expr) = ZeroOrMoreSeparated.parse(source, cache, context) {
return Ok(expr);
}
// Try one-or-more separated syntax: (+ A / B)
if let Ok(expr) = OneOrMoreSeparated.parse(source, cache, context) {
return Ok(expr);
}
// Try non-separated repetitions (shorter patterns)
// Try zero-or-more syntax: (* A)
if let Ok(expr) = ZeroOrMore.parse(source, cache, context) {
return Ok(expr);
}
// Try one-or-more syntax: (+ A)
if let Ok(expr) = OneOrMore.parse(source, cache, context) {
return Ok(expr);
}
// Try zero-or-one syntax: (? A)
if let Ok(expr) = ZeroOrOne.parse(source, cache, context) {
return Ok(expr);
}
// Try read-until-parse syntax: (< A B)
if let Ok(expr) = ReadUntilAndParse.parse(source, cache, context) {
return Ok(expr);
}
// Try read-until syntax: (< A)
if let Ok(expr) = ReadUntil.parse(source, cache, context) {
return Ok(expr);
}
// Try rule reference: identifier
if let Ok(expr) = RuleReferenceParser.parse(source, cache, context) {
return Ok(expr);
}
// Try sequential syntax last: (A B C)
// This must be last because it's the most general case
if let Ok(expr) = Sequential.parse(source, cache, context) {
return Ok(expr);
}
Err(Error::NoMatch)
}
}
/// Result type for Grammar parsing
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum GrammarResult {
/// Literal string result (from terminals)
Literal(Vec<u8>),
/// UTF-8 string result (from unicode character classes)
Unicode(String),
/// Byte sequence result (from ASCII character classes)
Bytes(Vec<u8>),
/// Sequence result containing multiple sub-results
Sequence(Vec<GrammarResult>),
/// Alternative result - exactly one of the options matched
Alternative(Box<GrammarResult>),
/// Repetition result - zero or more matches
Repetition(Vec<GrammarResult>),
/// Optional result - zero or one match
Optional(Option<Box<GrammarResult>>),
/// Empty result (for () or empty matches)
Empty,
}
impl GrammarResult {
/// Convert this result to bytes for situations where byte representation is needed
pub fn to_bytes(&self) -> Vec<u8> {
match self {
GrammarResult::Literal(bytes) => bytes.clone(),
GrammarResult::Unicode(string) => string.as_bytes().to_vec(),
GrammarResult::Bytes(bytes) => bytes.clone(),
GrammarResult::Sequence(results) => {
let mut combined = Vec::new();
for result in results {
combined.extend(result.to_bytes());
}
combined
}
GrammarResult::Alternative(result) => result.to_bytes(),
GrammarResult::Repetition(results) => {
let mut combined = Vec::new();
for result in results {
combined.extend(result.to_bytes());
}
combined
}
GrammarResult::Optional(Some(result)) => result.to_bytes(),
GrammarResult::Optional(None) | GrammarResult::Empty => Vec::new(),
}
}
/// Check if this result represents an empty match
pub fn is_empty(&self) -> bool {
matches!(self, GrammarResult::Empty | GrammarResult::Optional(None))
}
}
/// Internal AST node for grammar parsing (private)
#[derive(Debug, Clone, PartialEq, Eq)]
enum GrammarNode {
RuleReference(String),
Empty,
Terminal(Literal),
Digits(Utf8Class),
Alpha(Utf8Class),
Alphanumeric(Utf8Class),
Whitespace(Utf8Class),
UDigits(Utf8Class),
UAlpha(Utf8Class),
UAlphanumeric(Utf8Class),
UWhitespace(Utf8Class),
HexDigits(Utf8Class),
EndOfFile(EndOfFile),
InClass(Utf8Class),
NotInClass(Utf8Class),
Sequential(Sequence<Box<GrammarNode>, Box<GrammarNode>>),
SequentialEnd(Sequence<Box<GrammarNode>, ()>),
Alternatives(Either<Box<GrammarNode>, Box<GrammarNode>>),
ZeroOrMore(Repeat<Box<GrammarNode>, ()>),
OneOrMore(Repeat<Box<GrammarNode>, ()>),
ZeroOrOne(Repeat<Box<GrammarNode>>),
ZeroOrMoreSeparated(Repeat<Box<GrammarNode>, Box<GrammarNode>>),
OneOrMoreSeparated(Repeat<Box<GrammarNode>, Box<GrammarNode>>),
ReadUntil(Utf8Until<Box<GrammarNode>>),
ReadUntilParse(Utf8Until<Box<GrammarNode>>, Box<GrammarNode>),
}
/// Grammar parser with context for named rules and recursion
#[derive(Debug, Clone)]
pub struct Grammar {
node: GrammarNode,
context: GrammarContext,
}
impl Grammar {
/// Create a new Grammar with the given node and context (private)
fn new(node: GrammarNode, context: GrammarContext) -> Self {
Self { node, context }
}
/// Check if a rule exists in this grammar's context
pub fn rule_exists(&self, name: &str) -> bool {
self.context.rules.contains_key(name)
}
/// Get the start rule name, if any
pub fn start_rule(&self) -> Option<&str> {
self.context.start_rule.as_deref()
}
}
// Implementation for Grammar struct for external API (uses embedded context)
impl Parser<()> for Grammar {
type Output = GrammarResult;
fn id(&self) -> u64 {
use std::any::TypeId;
use std::hash::{DefaultHasher, Hash, Hasher};
let mut hasher = DefaultHasher::new();
TypeId::of::<Self>().hash(&mut hasher);
// Hash the node
self.node.id().hash(&mut hasher);
// Hash the context fields
self.context.rules.len().hash(&mut hasher);
if let Some(ref start_rule) = self.context.start_rule {
start_rule.hash(&mut hasher);
}
hasher.finish()
}
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
_context: &mut (),
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Use our embedded context for parsing
let mut grammar_context = self.context.clone();
// Delegate to the internal node with our context
self.node.read(source, cache, &mut grammar_context)
}
}
// Implementation for Grammar struct with GrammarContext - delegates to internal node
impl Parser<GrammarContext> for Grammar {
type Output = GrammarResult;
fn id(&self) -> u64 {
use std::any::TypeId;
use std::hash::{DefaultHasher, Hash, Hasher};
let mut hasher = DefaultHasher::new();
TypeId::of::<Self>().hash(&mut hasher);
// Hash the node
self.node.id().hash(&mut hasher);
// Hash the context fields
self.context.rules.len().hash(&mut hasher);
if let Some(ref start_rule) = self.context.start_rule {
start_rule.hash(&mut hasher);
}
hasher.finish()
}
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Delegate to the internal node
self.node.read(source, cache, context)
}
}
// Implementation for GrammarNode enum - handles the actual parsing logic
impl Parser<GrammarContext> for GrammarNode {
type Output = GrammarResult;
fn id(&self) -> u64 {
use std::any::TypeId;
use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem;
let mut hasher = DefaultHasher::new();
TypeId::of::<Self>().hash(&mut hasher);
// Hash the enum discriminant
mem::discriminant(self).hash(&mut hasher);
// Hash the contents based on variant
match self {
GrammarNode::RuleReference(name) => {
name.hash(&mut hasher);
}
GrammarNode::Empty => {
// Nothing to hash for empty
}
GrammarNode::Terminal(literal) => {
<Literal as Parser<GrammarContext>>::id(literal).hash(&mut hasher);
}
GrammarNode::Digits(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::Alpha(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::Alphanumeric(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::Whitespace(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::UDigits(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::UAlpha(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::UAlphanumeric(utf8_class) => {
<Utf8Class<fn(char) -> bool> as Parser<GrammarContext>>::id(utf8_class)
.hash(&mut hasher);
}
GrammarNode::UWhitespace(utf8_class) => {
<Utf8Class<fn(char) -> bool> as Parser<GrammarContext>>::id(utf8_class)
.hash(&mut hasher);
}
GrammarNode::HexDigits(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::EndOfFile(eof) => {
<EndOfFile as Parser<GrammarContext>>::id(eof).hash(&mut hasher);
}
GrammarNode::InClass(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::NotInClass(utf8_class) => {
<Utf8Class as Parser<GrammarContext>>::id(utf8_class).hash(&mut hasher);
}
GrammarNode::Sequential(sequence) => {
<Sequence<Box<GrammarNode>, Box<GrammarNode>> as Parser<GrammarContext>>::id(
sequence,
)
.hash(&mut hasher);
}
GrammarNode::SequentialEnd(sequence) => {
<Sequence<Box<GrammarNode>, ()> as Parser<GrammarContext>>::id(sequence)
.hash(&mut hasher);
}
GrammarNode::Alternatives(either) => {
<Either<Box<GrammarNode>, Box<GrammarNode>> as Parser<GrammarContext>>::id(either)
.hash(&mut hasher);
}
GrammarNode::ZeroOrMore(repeat) => {
<Repeat<Box<GrammarNode>, ()> as Parser<GrammarContext>>::id(repeat)
.hash(&mut hasher);
}
GrammarNode::OneOrMore(repeat) => {
<Repeat<Box<GrammarNode>, ()> as Parser<GrammarContext>>::id(repeat)
.hash(&mut hasher);
}
GrammarNode::ZeroOrOne(repeat) => {
<Repeat<Box<GrammarNode>> as Parser<GrammarContext>>::id(repeat).hash(&mut hasher);
}
GrammarNode::ZeroOrMoreSeparated(repeat) => {
<Repeat<Box<GrammarNode>, Box<GrammarNode>> as Parser<GrammarContext>>::id(repeat)
.hash(&mut hasher);
}
GrammarNode::OneOrMoreSeparated(repeat) => {
<Repeat<Box<GrammarNode>, Box<GrammarNode>> as Parser<GrammarContext>>::id(repeat)
.hash(&mut hasher);
}
GrammarNode::ReadUntil(until) => {
<Utf8Until<Box<GrammarNode>> as Parser<GrammarContext>>::id(until)
.hash(&mut hasher);
}
GrammarNode::ReadUntilParse(until, content_parser) => {
<Utf8Until<Box<GrammarNode>> as Parser<GrammarContext>>::id(until)
.hash(&mut hasher);
<Box<GrammarNode> as Parser<GrammarContext>>::id(content_parser).hash(&mut hasher);
}
}
hasher.finish()
}
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
match self {
GrammarNode::RuleReference(name) => {
// Look up the rule in the context
if let Some(rule_node) = context.rules.get(name).cloned() {
// Recursive call to parse the referenced rule
rule_node.parse(source, cache, context)
} else {
Err(Error::NoMatch)
}
}
GrammarNode::Empty => {
// Empty variant returns empty result
Ok(GrammarResult::Empty)
}
GrammarNode::Terminal(literal) => {
let result = literal.parse(source, cache, context)?;
Ok(GrammarResult::Literal(result.into()))
}
GrammarNode::Digits(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::Alpha(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::Alphanumeric(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::Whitespace(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::UDigits(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::UAlpha(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::UAlphanumeric(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::UWhitespace(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::HexDigits(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::EndOfFile(eof) => {
eof.parse(source, cache, context)?;
Ok(GrammarResult::Empty)
}
GrammarNode::InClass(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::NotInClass(utf8_class) => {
let result = utf8_class.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::Sequential(sequence) => {
let result = sequence.parse(source, cache, context)?;
// Create a sequence result containing both sub-results
Ok(GrammarResult::Sequence(vec![result.0, result.1]))
}
GrammarNode::SequentialEnd(sequence) => {
let result = sequence.parse(source, cache, context)?;
// Only return the result from the expression side (ignore the () side)
Ok(result.0)
}
GrammarNode::Alternatives(either) => {
let result = either.parse(source, cache, context)?;
// Return whichever alternative matched
match result {
(Some(left), None) => Ok(GrammarResult::Alternative(Box::new(left))),
(None, Some(right)) => Ok(GrammarResult::Alternative(Box::new(right))),
_ => unreachable!("Either should return exactly one result"),
}
}
GrammarNode::ZeroOrMore(repeat) => {
let results = repeat.parse(source, cache, context)?;
// Convert all results to GrammarResult
Ok(GrammarResult::Repetition(results))
}
GrammarNode::OneOrMore(repeat) => {
let results = repeat.parse(source, cache, context)?;
// Convert all results to GrammarResult
Ok(GrammarResult::Repetition(results))
}
GrammarNode::ZeroOrOne(option) => {
let result = option.parse(source, cache, context)?;
if result.is_empty() {
Ok(GrammarResult::Optional(None))
} else {
Ok(GrammarResult::Optional(
result.into_iter().next().map(Box::new),
))
}
}
GrammarNode::ZeroOrMoreSeparated(repeat) => {
let results = repeat.parse(source, cache, context)?;
// Convert all results to GrammarResult
Ok(GrammarResult::Repetition(results))
}
GrammarNode::OneOrMoreSeparated(repeat) => {
let results = repeat.parse(source, cache, context)?;
// Convert all results to GrammarResult
Ok(GrammarResult::Repetition(results))
}
GrammarNode::ReadUntil(until) => {
let result = until.parse(source, cache, context)?;
Ok(GrammarResult::Unicode(result))
}
GrammarNode::ReadUntilParse(until, content_parser) => {
let captured = until.parse(source, cache, context)?;
// Parse the captured content with the content parser
let mut captured_input = std::io::Cursor::new(captured.as_bytes());
let mut captured_source = crate::parser::Source::new(&mut captured_input);
let content_result = content_parser.parse(&mut captured_source, cache, context)?;
Ok(content_result)
}
}
}
}
struct Terminal;
impl Parser<GrammarContext> for Terminal {
type Output = Literal;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
QUOTE.parse(source, cache, context)?;
let mut bytes = Vec::new();
let mut escaped = false;
let mut reading = true;
while reading {
let byte = source.peek1()?;
if escaped {
bytes.push(byte);
escaped = false;
source.advance(1);
} else if byte == b'\\' {
escaped = true;
source.advance(1);
} else if byte == b'"' {
reading = false;
} else {
bytes.push(byte);
source.advance(1);
}
}
QUOTE.parse(source, cache, context)?;
Ok(Literal::from_bytes(&bytes))
}
}
struct Digits;
impl Parser<GrammarContext> for Digits {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
DIGITS_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::digits())
}
}
struct Alpha;
impl Parser<GrammarContext> for Alpha {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
ALPHA_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::alpha())
}
}
struct Alphanumeric;
impl Parser<GrammarContext> for Alphanumeric {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
ALPHANUMERIC_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::alphanumeric())
}
}
struct Whitespace;
impl Parser<GrammarContext> for Whitespace {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
WHITESPACE_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::whitespace())
}
}
struct HexDigits;
impl Parser<GrammarContext> for HexDigits {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
HEXDIGITS_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::hex_digits())
}
}
struct UDigits;
impl Parser<GrammarContext> for UDigits {
type Output = Utf8Class<fn(char) -> bool>;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
UDIGITS_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::unicode_digits())
}
}
struct UAlpha;
impl Parser<GrammarContext> for UAlpha {
type Output = Utf8Class<fn(char) -> bool>;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
UALPHA_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::unicode_alpha())
}
}
struct UAlphanumeric;
impl Parser<GrammarContext> for UAlphanumeric {
type Output = Utf8Class<fn(char) -> bool>;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
UALPHANUMERIC_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::from_predicate_min(|c| c.is_alphanumeric(), 1))
}
}
struct UWhitespace;
impl Parser<GrammarContext> for UWhitespace {
type Output = Utf8Class<fn(char) -> bool>;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
UWHITESPACE_KEYWORD.parse(source, cache, context)?;
Ok(Utf8Class::unicode_whitespace())
}
}
struct EndOfFileParser;
impl Parser<GrammarContext> for EndOfFileParser {
type Output = EndOfFile;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
EOF_KEYWORD.parse(source, cache, context)?;
Ok(EndOfFile::new())
}
}
struct Identifier;
impl Parser<GrammarContext> for Identifier {
type Output = String;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse identifier: letter followed by letters/digits/underscores
let first_char = Utf8Class::alpha().parse(source, cache, context)?;
let rest_chars = Utf8Class::from_predicate_min(|c| c.is_alphanumeric() || c == '_', 0)
.parse(source, cache, context)
.unwrap_or_default();
Ok(format!("{first_char}{rest_chars}"))
}
}
struct InClass;
impl Parser<GrammarContext> for InClass {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Match opening bracket '['
OPEN_BRACKET.parse(source, cache, context)?;
if source.peek1()? == b'^' {
return Err(Error::NoMatch);
}
let mut chars = String::new();
let mut escaped = false;
loop {
// Read a UTF-8 character
let ch = read_utf8_char(source)?;
if escaped {
// Add escaped character
chars.push(ch);
escaped = false;
} else if ch == '\\' {
// Next character is escaped
escaped = true;
} else if ch == ']' {
// End of character class
break;
} else {
// Regular character - add to class
chars.push(ch);
}
}
// We don't actually call this, because read_utf8_char has
// already consumed it from the source
//CLOSE_BRACKET.parse(source, cache, context)?;
// Create UTF-8 character class from the collected characters
// Use with_min(1) to require at least one character match
Ok(Utf8Class::with_min(&chars, 1))
}
}
struct NotInClass;
impl Parser<GrammarContext> for NotInClass {
type Output = Utf8Class;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Match opening bracket '['
OPEN_BRACKET.parse(source, cache, context)?;
// Must have '^' as the next character for negated class
CARET.parse(source, cache, context)?;
let mut chars = String::new();
let mut escaped = false;
loop {
// Read a UTF-8 character
let ch = read_utf8_char(source)?;
if escaped {
// Add escaped character
chars.push(ch);
escaped = false;
} else if ch == '\\' {
// Next character is escaped
escaped = true;
} else if ch == ']' {
// End of character class
break;
} else {
// Regular character - add to class
chars.push(ch);
}
}
// We don't actually call this, because read_utf8_char has
// already consumed it from the source
//CLOSE_BRACKET.parse(source, cache, context)?;
// Create negated UTF-8 character class from the collected characters
// Use not_in_with_min(1) to require at least one character match and negate the set
Ok(Utf8Class::not_in_with_min(&chars, 1))
}
}
struct Sequential;
impl Parser<GrammarContext> for Sequential {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Skip optional whitespace after (
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse sequence of expressions
let mut expressions: Vec<GrammarNode> = Vec::new();
// Parse expressions until closing paren
while source.peek1().unwrap_or(0) != b')' {
// Skip whitespace before expression
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Check if we've reached the closing paren after skipping whitespace
if source.peek1().unwrap_or(0) == b')' {
break;
}
// Try to parse a sub-expression
match <GrammarParser as Parser<GrammarContext>>::read(
&GrammarParser,
source,
cache,
context,
) {
Ok(expr) => {
expressions.push(expr);
}
Err(_) => {
// If we can't parse a sub-expression, return NoMatch
return Err(Error::NoMatch);
}
}
// Skip whitespace after expression
let _ = Utf8Class::whitespace().parse(source, cache, context);
}
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the sequence from parsed expressions
if expressions.is_empty() {
return Err(Error::NoMatch);
}
if expressions.len() == 1 {
return Ok(expressions.into_iter().next().unwrap());
}
// Build right-associative sequence ending with ()
let mut expressions = expressions;
expressions.reverse(); // Process from right to left: [C, B, A]
// Start with the rightmost element and wrap it as Sequence<C, ()>
let last_expr = expressions.remove(0); // Remove C
let mut result = GrammarNode::SequentialEnd(Sequence::new(Box::new(last_expr), ()));
// Build the chain: Sequence<B, Sequence<C, ()>> -> Sequence<A, Sequence<B, Sequence<C, ()>>>
for expr in expressions {
let sequence = Sequence::new(Box::new(expr), Box::new(result));
result = GrammarNode::Sequential(sequence);
}
Ok(result)
}
}
struct Alternatives;
impl Parser<GrammarContext> for Alternatives {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse pipe symbol
PIPE.parse(source, cache, context)?;
// Skip optional whitespace after |
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse alternative expressions
let mut alternatives: Vec<GrammarNode> = Vec::new();
// Parse expressions until closing paren
while source.peek1().unwrap_or(0) != b')' {
// Skip whitespace before expression
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Check if we've reached the closing paren after skipping whitespace
if source.peek1().unwrap_or(0) == b')' {
break;
}
// Try to parse a sub-expression
match <GrammarParser as Parser<GrammarContext>>::read(
&GrammarParser,
source,
cache,
context,
) {
Ok(expr) => {
alternatives.push(expr);
}
Err(_) => {
// If we can't parse a sub-expression, return NoMatch
return Err(Error::NoMatch);
}
}
// Skip whitespace after expression
let _ = Utf8Class::whitespace().parse(source, cache, context);
}
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the alternatives from parsed expressions
if alternatives.is_empty() {
return Err(Error::NoMatch);
}
if alternatives.len() == 1 {
return Ok(alternatives.into_iter().next().unwrap());
}
// Build right-associative alternatives without () termination
let mut alternatives = alternatives;
alternatives.reverse(); // Process from right to left: [C, B, A]
// Start with the rightmost element
let mut result = alternatives.remove(0); // Remove C
// Build the chain: Either<B, C> -> Either<A, Either<B, C>>
for expr in alternatives {
let either = Either::new(Box::new(expr), Box::new(result));
result = GrammarNode::Alternatives(either);
}
Ok(result)
}
}
struct ZeroOrMore;
impl Parser<GrammarContext> for ZeroOrMore {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse asterisk symbol
ASTERISK.parse(source, cache, context)?;
// Skip optional whitespace after *
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the inner expression
let inner_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the zero-or-more expression
let repeat = Repeat::new(Box::new(inner_expr));
Ok(GrammarNode::ZeroOrMore(repeat))
}
}
struct OneOrMore;
impl Parser<GrammarContext> for OneOrMore {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse plus symbol
PLUS.parse(source, cache, context)?;
// Skip optional whitespace after +
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the inner expression
let inner_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the one-or-more expression
let repeat = Repeat::with_min(Box::new(inner_expr), 1);
Ok(GrammarNode::OneOrMore(repeat))
}
}
struct ZeroOrMoreSeparated;
impl Parser<GrammarContext> for ZeroOrMoreSeparated {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse asterisk symbol
ASTERISK.parse(source, cache, context)?;
// Skip optional whitespace after *
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the inner expression
let inner_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the separator: /
SLASH.parse(source, cache, context)?;
// Skip optional whitespace after /
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the separator expression
let separator_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the zero-or-more separated expression
let repeat = Repeat::with_joint(Box::new(inner_expr), Box::new(separator_expr));
Ok(GrammarNode::ZeroOrMoreSeparated(repeat))
}
}
struct OneOrMoreSeparated;
impl Parser<GrammarContext> for OneOrMoreSeparated {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse plus symbol
PLUS.parse(source, cache, context)?;
// Skip optional whitespace after +
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the inner expression
let inner_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the separator: /
SLASH.parse(source, cache, context)?;
// Skip optional whitespace after /
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the separator expression
let separator_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the one-or-more separated expression
let repeat = Repeat::with_joint_min(Box::new(inner_expr), Box::new(separator_expr), 1);
Ok(GrammarNode::OneOrMoreSeparated(repeat))
}
}
struct ZeroOrOne;
impl Parser<GrammarContext> for ZeroOrOne {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse question mark symbol
QUESTION.parse(source, cache, context)?;
// Skip optional whitespace after ?
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the inner expression
let inner_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the zero-or-one expression
let repeat = Repeat::with_max(Box::new(inner_expr), 1);
Ok(GrammarNode::ZeroOrOne(repeat))
}
}
struct ReadUntil;
impl Parser<GrammarContext> for ReadUntil {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse less-than symbol
LESS_THAN.parse(source, cache, context)?;
// Skip optional whitespace after <
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the end condition expression
let end_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the read-until expression
let utf8_until = Utf8Until::new(Box::new(end_expr));
Ok(GrammarNode::ReadUntil(utf8_until))
}
}
struct ReadUntilAndParse;
impl Parser<GrammarContext> for ReadUntilAndParse {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse opening parenthesis
OPEN_PAREN.parse(source, cache, context)?;
// Parse less-than symbol
LESS_THAN.parse(source, cache, context)?;
// Skip optional whitespace after <
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the end condition expression
let end_expr = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse the content parser expression
let content_parser = GrammarParser.parse(source, cache, context)?;
// Skip optional whitespace before closing paren
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse closing parenthesis
CLOSE_PAREN.parse(source, cache, context)?;
// Build the read-until-parse expression
let utf8_until = Utf8Until::new(Box::new(end_expr));
Ok(GrammarNode::ReadUntilParse(
utf8_until,
Box::new(content_parser),
))
}
}
// Parser for @start directive
struct StartDirective;
impl Parser<GrammarContext> for StartDirective {
type Output = String;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse @start keyword
AT_SYMBOL.parse(source, cache, context)?;
START_KEYWORD.parse(source, cache, context)?;
// Skip whitespace (including newlines)
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse rule name (identifier: letters, digits, underscores)
let name = Identifier.parse(source, cache, context)?;
Ok(name)
}
}
// Parser for rule definition: name = expression
struct RuleDefinition;
impl Parser<GrammarContext> for RuleDefinition {
type Output = (String, GrammarNode);
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse rule name (identifier: letters, digits, underscores)
let name = Identifier.parse(source, cache, context)?;
// Skip whitespace (including newlines)
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse equals sign
EQUALS.parse(source, cache, context)?;
// Skip whitespace (including newlines)
let _ = Utf8Class::whitespace().parse(source, cache, context);
// Parse expression
let expression = <GrammarParser as Parser<GrammarContext>>::read(
&GrammarParser,
source,
cache,
context,
)?;
Ok((name, expression))
}
}
// Parser for rule reference: just an identifier
struct RuleReferenceParser;
impl Parser<GrammarContext> for RuleReferenceParser {
type Output = GrammarNode;
fn read<S>(
&self,
source: &mut crate::parser::Source<S>,
cache: &mut impl crate::cache::ParsingCache,
context: &mut GrammarContext,
) -> crate::result::ParseResult<Self::Output>
where
S: crate::parser::Parsable,
{
// Parse identifier (letters, digits, underscores)
let name = Identifier.parse(source, cache, context)?;
// Check if this is a reserved keyword
if matches!(
name.as_str(),
"digits"
| "alpha"
| "alphanumeric"
| "whitespace"
| "hexdigits"
| "udigits"
| "ualpha"
| "ualphanumeric"
| "uwhitespace"
| "eof"
) {
return Err(Error::NoMatch);
}
// Always create a rule reference - we'll resolve it later
Ok(GrammarNode::RuleReference(name))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::{parse, parse_with_context};
use std::io::Cursor;
#[test]
fn test_inclass_basic() {
let parser = InClass;
let mut input = Cursor::new(b"[abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
// Test that the character class was created correctly
// We can't easily test the internal structure, but we can verify it was created
let _utf8_class = result;
}
#[test]
fn test_inclass_with_utf8() {
let parser = InClass;
// Test with UTF-8 characters
let mut input = Cursor::new("[αβγ世界]".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_inclass_with_escapes() {
let parser = InClass;
// Test with escaped brackets
let mut input = Cursor::new(b"[\\[\\]]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_inclass_empty() {
let parser = InClass;
// Test empty character class
let mut input = Cursor::new(b"[]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_inclass_missing_closing_bracket() {
let parser = InClass;
// Test malformed input - should fail
let mut input = Cursor::new(b"[abc");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err());
}
#[test]
fn test_inclass_functional() {
// Test that the created character class actually works for parsing
let parser = InClass;
// Parse the character class definition
let mut input = Cursor::new(b"[abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let char_class = parse_with_context(parser, &mut source, &mut context).unwrap();
// Test the character class against actual input
let mut test_input1 = Cursor::new(b"a");
let mut test_source1 = crate::parser::Source::new(&mut test_input1);
let result1 = parse(char_class.clone(), &mut test_source1);
assert!(result1.is_ok()); // Should match 'a'
if let Ok(matched) = result1 {
assert_eq!(matched, "a"); // Should match exactly 'a'
}
let mut test_input2 = Cursor::new(b"d");
let mut test_source2 = crate::parser::Source::new(&mut test_input2);
let result2 = parse(char_class, &mut test_source2);
// With min_length 1, it should fail to match 'd' since it's not in [abc]
assert!(result2.is_err()); // Should NOT match 'd'
}
#[test]
fn test_not_inclass_basic() {
let parser = NotInClass;
let mut input = Cursor::new(b"[^abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_not_inclass_with_utf8() {
let parser = NotInClass;
// Test with UTF-8 characters
let mut input = Cursor::new("[^αβγ世界]".as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_not_inclass_with_escapes() {
let parser = NotInClass;
// Test with escaped brackets
let mut input = Cursor::new(b"[^\\[\\]]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_not_inclass_empty() {
let parser = NotInClass;
// Test empty negated character class - should match any character
let mut input = Cursor::new(b"[^]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
let _utf8_class = result;
}
#[test]
fn test_not_inclass_missing_closing_bracket() {
let parser = NotInClass;
// Test malformed input - should fail
let mut input = Cursor::new(b"[^abc");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err());
}
#[test]
fn test_not_inclass_missing_caret() {
let parser = NotInClass;
// Test input without caret - should fail
let mut input = Cursor::new(b"[abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err());
}
#[test]
fn test_not_inclass_functional() {
// Test that the created negated character class actually works for parsing
let parser = NotInClass;
// Parse the negated character class definition
let mut input = Cursor::new(b"[^abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let char_class = parse_with_context(parser, &mut source, &mut context).unwrap();
// Test the character class against actual input
// Should match 'd' (not in [abc])
let mut test_input1 = Cursor::new(b"d");
let mut test_source1 = crate::parser::Source::new(&mut test_input1);
let result1 = parse(char_class.clone(), &mut test_source1);
assert!(result1.is_ok()); // Should match 'd'
if let Ok(matched) = result1 {
assert_eq!(matched, "d"); // Should match exactly 'd'
}
// Should NOT match 'a' (in [abc])
let mut test_input2 = Cursor::new(b"a");
let mut test_source2 = crate::parser::Source::new(&mut test_input2);
let result2 = parse(char_class, &mut test_source2);
assert!(result2.is_err()); // Should NOT match 'a'
}
#[test]
fn test_grammar_expression_terminal() {
// Test that GrammarParser::new() can parse a terminal
let parser = GrammarParser::new();
let mut input = Cursor::new(b"\"hello\"");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
match parse_with_context(parser, &mut source, &mut context) {
Ok(GrammarNode::Terminal(literal)) => {
// Verify we got a literal
assert_eq!(literal, b"hello".as_slice().into());
}
_ => panic!("Expected Terminal expression"),
}
}
#[test]
fn test_grammar_expression_digits() {
// Test that GrammarParser::new() can parse digits keyword
let parser = GrammarParser::new();
let mut input = Cursor::new(b"digits");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
match parse_with_context(parser, &mut source, &mut context) {
Ok(GrammarNode::Digits(_)) => {
// Success - we got a digits expression
}
_ => panic!("Expected Digits expression"),
}
}
#[test]
fn test_grammar_expression_alpha() {
// Test that GrammarParser::new() can parse alpha keyword
let parser = GrammarParser::new();
let mut input = Cursor::new(b"alpha");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
match parse_with_context(parser, &mut source, &mut context) {
Ok(GrammarNode::Alpha(_)) => {
// Success - we got an alpha expression
}
_ => panic!("Expected Alpha expression"),
}
}
#[test]
fn test_grammar_expression_inclass() {
// Test that GrammarParser::new() can parse character class
let parser = GrammarParser::new();
let mut input = Cursor::new(b"[abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
match parse_with_context(parser, &mut source, &mut context) {
Ok(GrammarNode::InClass(_)) => {
// Success - we got an InClass expression
}
_ => panic!("Expected InClass expression"),
}
}
#[test]
fn test_grammar_expression_not_inclass() {
// Test that GrammarParser::new() can parse negated character class
let parser = GrammarParser::new();
let mut input = Cursor::new(b"[^abc]");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
match parse_with_context(parser, &mut source, &mut context) {
Ok(GrammarNode::NotInClass(_)) => {
// Success - we got a NotInClass expression
}
_ => panic!("Expected NotInClass expression"),
}
}
#[test]
fn test_grammar_expression_no_match() {
// Test that GrammarParser::new() fails on invalid input
let parser = GrammarParser::new();
// This should not match since it's not a valid grammar expression
// Use something that can't be parsed as any grammar construct
let mut input = Cursor::new(b"@#$%");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err());
}
#[test]
fn test_grammar_expression_parenthesized_sequence() {
// Test that GrammarParser::new() can parse parenthesized sequences
let parser = GrammarParser::new();
// Test a simple sequence - should be successfully parsed
let mut input = Cursor::new(b"(\"hello\" \"world\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
// Should succeed because parenthesized expressions are now implemented
assert!(result.is_ok());
}
#[test]
fn test_grammar_expression_alternatives() {
// Test that GrammarParser::new() can parse alternatives syntax
let parser = GrammarParser::new();
// Test alternatives - should be successfully parsed
let mut input = Cursor::new(b"(| \"hello\" \"world\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
// Should succeed because alternatives are now implemented
assert!(result.is_ok());
}
#[test]
fn test_alternatives_parser_direct() {
// Test the Alternatives parser directly
let mut input = Cursor::new(b"(| \"hello\" \"world\")");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(Alternatives, &mut source, &mut GrammarContext::default());
println!("Direct Alternatives parser result: {result:?}");
assert!(result.is_ok());
}
#[test]
fn test_grammar_expression_repetition() {
// Test that GrammarParser::new() can parse repetition syntax
let parser = GrammarParser::new();
// Test zero or more - should be successfully parsed
let mut input = Cursor::new(b"(* \"hello\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
// Should succeed because repetitions are now implemented
assert!(result.is_ok());
}
#[test]
fn test_grammar_expression_one_or_more() {
// Test one-or-more repetition
let parser = GrammarParser::new();
let mut input = Cursor::new(b"(+ \"hello\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
}
#[test]
fn test_grammar_expression_zero_or_one() {
// Test zero-or-one repetition
let parser = GrammarParser::new();
let mut input = Cursor::new(b"(? \"hello\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
}
#[test]
fn test_grammar_expression_nested_sequences() {
// Test nested parenthesized expressions
let parser = GrammarParser::new();
let mut input = Cursor::new(b"(\"hello\" digits \"world\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
}
#[test]
fn test_grammar_expression_sequence_structure() {
// Test that sequences actually create the proper recursive structure
let parser = GrammarParser::new();
let mut input = Cursor::new(b"(\"hello\" \"world\" \"test\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
// Verify we got a Sequential expression
if let Ok(GrammarNode::Sequential(_sequence)) = result {
// The sequence should be: Sequence<"hello", Sequence<"world", "test">>
// This demonstrates that our recursive structure building works
} else {
panic!("Expected Sequential expression");
}
}
#[test]
fn test_grammar_expression_alternatives_structure() {
// Test that alternatives create the proper recursive structure
let parser = GrammarParser::new();
let mut input = Cursor::new(b"(| \"hello\" \"world\" \"test\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
// Verify we got an Alternatives expression
if let Ok(GrammarNode::Alternatives(_either)) = result {
// The alternatives should be: Either<"hello", Either<"world", "test">>
// This demonstrates that our recursive structure building works
} else {
panic!("Expected Alternatives expression");
}
}
#[test]
fn test_grammar_expression_repetition_structure() {
// Test that repetitions create the proper structure
let parser = GrammarParser::new();
let mut input = Cursor::new(b"(* \"hello\")");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
// Verify we got a ZeroOrMore expression
if let Ok(GrammarNode::ZeroOrMore(_repeat)) = result {
// The repetition properly wraps the inner expression
} else {
panic!("Expected ZeroOrMore expression");
}
}
// Tests for parsing functionality
#[test]
fn test_sequence_parsing_basic() {
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"("hello" "world")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "Sequence parsing should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::Sequential(_) => {
// Expected: this is the correct structure for sequences
}
_ => panic!("Expected Sequential expression, got {expr:?}"),
}
}
}
#[test]
fn test_alternatives_parsing_basic() {
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(| "hello" "world")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "Alternatives parsing should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::Alternatives(_) => {
// Expected: this is the correct structure for alternatives
}
_ => panic!("Expected Alternatives expression, got {expr:?}"),
}
}
}
// Tests for all character class keywords
#[test]
#[allow(clippy::type_complexity)]
fn test_all_character_class_keywords() {
let test_cases: Vec<(&'static str, Box<dyn Fn(&GrammarNode) -> bool>)> = vec![
(
"digits",
Box::new(|expr| matches!(expr, GrammarNode::Digits(_))),
),
(
"alpha",
Box::new(|expr| matches!(expr, GrammarNode::Alpha(_))),
),
(
"alphanumeric",
Box::new(|expr| matches!(expr, GrammarNode::Alphanumeric(_))),
),
(
"whitespace",
Box::new(|expr| matches!(expr, GrammarNode::Whitespace(_))),
),
(
"hexdigits",
Box::new(|expr| matches!(expr, GrammarNode::HexDigits(_))),
),
(
"udigits",
Box::new(|expr| matches!(expr, GrammarNode::UDigits(_))),
),
(
"ualpha",
Box::new(|expr| matches!(expr, GrammarNode::UAlpha(_))),
),
(
"ualphanumeric",
Box::new(|expr| matches!(expr, GrammarNode::UAlphanumeric(_))),
),
(
"uwhitespace",
Box::new(|expr| matches!(expr, GrammarNode::UWhitespace(_))),
),
];
for (keyword, matcher) in test_cases {
let parser = GrammarParser::new();
let mut input = Cursor::new(keyword.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Failed to parse keyword: {keyword}");
if let Ok(expr) = result {
assert!(
matcher(&expr),
"Wrong expression type for keyword {keyword}: {expr:?}",
);
}
}
}
// Tests for terminal parsing with various escape sequences
#[test]
fn test_terminal_with_escape_sequences() {
let test_cases = vec![
(r#""hello""#, "hello"),
(r#""hello world""#, "hello world"),
(r#""line\nbreak""#, "linenbreak"), // \n becomes literal n
(r#""quote\"inside""#, r#"quote"inside"#), // \" becomes literal "
(r#""backslash\\here""#, r#"backslash\here"#), // \\ becomes literal \
(r#""tab\there""#, "tabthere"), // \t becomes literal t
];
for (input, expected) in test_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Failed to parse terminal: {input}");
if let Ok(GrammarNode::Terminal(literal)) = result {
assert_eq!(
literal,
expected.as_bytes().into(),
"Wrong literal value for: {input}",
);
} else {
panic!("Expected Terminal expression for: {input}");
}
}
}
// Tests for character class parsing
#[test]
fn test_character_class_parsing() {
let test_cases = vec![
("[abc]", true), // basic character class
("[^abc]", true), // negated character class
("[a-z]", true), // range (should be parsed as individual chars for now)
("[αβγ]", true), // unicode characters
("[]", true), // empty character class
("[^]", true), // empty negated character class
];
for (input, should_succeed) in test_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
if should_succeed {
assert!(result.is_ok(), "Failed to parse character class: {input}");
match result {
Ok(GrammarNode::InClass(_)) | Ok(GrammarNode::NotInClass(_)) => {
// Success - got the expected character class type
}
_ => panic!("Expected character class expression for: {input}"),
}
} else {
assert!(result.is_err(), "Should have failed to parse: {input}");
}
}
}
// Tests for repetition operators
#[test]
#[allow(clippy::type_complexity)]
fn test_repetition_operators() {
let test_cases: Vec<(&'static str, Box<dyn Fn(&GrammarNode) -> bool>)> = vec![
(
"(* \"hello\")",
Box::new(|expr| matches!(expr, GrammarNode::ZeroOrMore(_))),
),
(
"(+ \"hello\")",
Box::new(|expr| matches!(expr, GrammarNode::OneOrMore(_))),
),
(
"(? \"hello\")",
Box::new(|expr| matches!(expr, GrammarNode::ZeroOrOne(_))),
),
];
for (input, matcher) in test_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Failed to parse repetition: {input}");
if let Ok(expr) = result {
assert!(
matcher(&expr),
"Wrong expression type for repetition: {input}",
);
}
}
}
// Tests for separated repetitions
#[test]
#[allow(clippy::type_complexity)]
fn test_separated_repetitions() {
let test_cases: Vec<(&'static str, Box<dyn Fn(&GrammarNode) -> bool>)> = vec![
(
"(*\"item\" / \",\")",
Box::new(|expr| matches!(expr, GrammarNode::ZeroOrMoreSeparated(_))),
),
(
"(+\"item\" / \",\")",
Box::new(|expr| matches!(expr, GrammarNode::OneOrMoreSeparated(_))),
),
];
for (input, matcher) in test_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(
result.is_ok(),
"Failed to parse separated repetition: {input}",
);
if let Ok(expr) = result {
assert!(
matcher(&expr),
"Wrong expression type for separated repetition: {input}",
);
}
}
}
// Tests for deeply nested expressions
#[test]
fn test_deeply_nested_sequences() {
let input = "(\"a\" \"b\" \"c\" \"d\" \"e\" \"f\")";
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Failed to parse deeply nested sequence");
if let Ok(GrammarNode::Sequential(_)) = result {
// Success - deeply nested sequences should work
} else {
panic!("Expected Sequential expression for deeply nested sequence");
}
}
#[test]
fn test_deeply_nested_alternatives() {
let input = "(| \"a\" \"b\" \"c\" \"d\" \"e\" \"f\")";
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Failed to parse deeply nested alternatives");
if let Ok(GrammarNode::Alternatives(_)) = result {
// Success - deeply nested alternatives should work
} else {
panic!("Expected Alternatives expression for deeply nested alternatives");
}
}
// Tests for mixed sequences and alternatives
#[test]
fn test_nested_mixed_expressions() {
let input = "(\"start\" (| \"option1\" \"option2\") \"end\")";
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Failed to parse nested mixed expressions");
if let Ok(GrammarNode::Sequential(_)) = result {
// Success - nested mixed expressions should work
} else {
panic!("Expected Sequential expression for nested mixed expressions");
}
}
// Tests for error cases
#[test]
fn test_malformed_parentheses() {
// Test cases that should fail with NoMatch
let error_cases = vec!["(\"unclosed", "(\"missing_close\""];
for input in error_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(matches!(result, Err(crate::result::Error::NoMatch)));
}
// Test cases that should succeed (parser ignores trailing characters)
let success_cases = vec!["\"unopened\")", "\"missing_open\")"];
for input in success_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
}
// Test case that should succeed with nested parentheses
let nested_case = "((\"double\"))";
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(nested_case.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
}
#[test]
fn test_malformed_character_classes() {
let error_cases = vec!["[unclosed", "[^unclosed"];
for input in error_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(
result.is_err(),
"Should have failed for malformed character class: {input}",
);
}
}
#[test]
fn test_malformed_terminals() {
let error_cases = vec!["\"unclosed", "\"unterminated\\"];
for input in error_cases {
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(
dbg!(result).is_err(),
"Should have failed for malformed terminal: {input}",
);
}
}
// Performance and edge case tests
#[test]
fn test_empty_input() {
let parser = GrammarParser::new();
let mut input = Cursor::new(b"");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err(), "Should fail on empty input");
}
#[test]
fn test_whitespace_only() {
let parser = GrammarParser::new();
let mut input = Cursor::new(b" \t\n ");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err(), "Should fail on whitespace-only input");
}
#[test]
fn test_very_long_terminal() {
let long_content = "a".repeat(1000);
let input = format!("\"{long_content}\"");
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Should handle very long terminals");
if let Ok(GrammarNode::Terminal(literal)) = result {
assert_eq!(literal, long_content.as_bytes().into());
} else {
panic!("Expected Terminal expression for very long terminal");
}
}
#[test]
fn test_very_long_character_class() {
let long_chars =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".repeat(10);
let input = format!("[{long_chars}]");
let parser = GrammarParser::new();
let mut input_cursor = Cursor::new(input.as_bytes());
let mut source = crate::parser::Source::new(&mut input_cursor);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok(), "Should handle very long character classes");
if let Ok(GrammarNode::InClass(_)) = result {
// Success - very long character class handled
} else {
panic!("Expected InClass expression for very long character class");
}
}
// Tests for Expression Parser implementation
#[test]
fn test_expression_parser_terminal() {
let expr = GrammarNode::Terminal(Literal::from_str("hello"));
let mut input = Cursor::new(b"hello");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(result.is_ok(), "Expression parser should work for Terminal");
if let Ok(GrammarResult::Literal(bytes)) = result {
assert_eq!(bytes, b"hello".to_vec());
} else {
panic!("Expected Literal result");
}
}
#[test]
fn test_expression_parser_digits() {
let expr = GrammarNode::Digits(Utf8Class::digits());
let mut input = Cursor::new(b"12345");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(result.is_ok(), "Expression parser should work for Digits");
if let Ok(GrammarResult::Unicode(chars)) = result {
assert_eq!(chars, "12345");
} else {
panic!("Expected Bytes result");
}
}
#[test]
fn test_expression_parser_sequence() {
// Create a sequence: "hello" followed by "world"
let hello_expr = GrammarNode::Terminal(Literal::from_str("hello"));
let world_expr = GrammarNode::Terminal(Literal::from_str("world"));
let sequence = Sequence::new(Box::new(hello_expr), Box::new(world_expr));
let expr = GrammarNode::Sequential(sequence);
let mut input = Cursor::new(b"helloworld");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(
result.is_ok(),
"Expression parser should work for Sequential"
);
if let Ok(GrammarResult::Sequence(results)) = result {
assert_eq!(results.len(), 2);
// Convert to bytes to check the combined result
let combined_bytes = GrammarResult::Sequence(results).to_bytes();
assert_eq!(combined_bytes, b"helloworld".to_vec());
} else {
panic!("Expected Sequence result");
}
}
#[test]
fn test_expression_parser_alternatives() {
// Create alternatives: "hello" OR "world"
let hello_expr = GrammarNode::Terminal(Literal::from_str("hello"));
let world_expr = GrammarNode::Terminal(Literal::from_str("world"));
let either = Either::new(Box::new(hello_expr), Box::new(world_expr));
let expr = GrammarNode::Alternatives(either);
// Test first alternative
let mut input1 = Cursor::new(b"hello");
let mut source1 = crate::parser::Source::new(&mut input1);
let result1 =
parse_with_context(expr.clone(), &mut source1, &mut GrammarContext::default());
assert!(
result1.is_ok(),
"Expression parser should work for first alternative"
);
if let Ok(GrammarResult::Alternative(boxed_result)) = result1 {
assert_eq!(boxed_result.to_bytes(), b"hello".to_vec());
} else {
panic!("Expected Alternative result");
}
// Test second alternative
let mut input2 = Cursor::new(b"world");
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse_with_context(expr, &mut source2, &mut GrammarContext::default());
assert!(
result2.is_ok(),
"Expression parser should work for second alternative"
);
if let Ok(GrammarResult::Alternative(boxed_result)) = result2 {
assert_eq!(boxed_result.to_bytes(), b"world".to_vec());
} else {
panic!("Expected Alternative result");
}
}
#[test]
fn test_expression_parser_zero_or_more() {
// Create zero-or-more: (* "a")
let a_expr = GrammarNode::Terminal(Literal::from_str("a"));
let repeat = Repeat::new(Box::new(a_expr));
let expr = GrammarNode::ZeroOrMore(repeat);
let mut input = Cursor::new(b"aaab");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(
result.is_ok(),
"Expression parser should work for ZeroOrMore"
);
if let Ok(GrammarResult::Repetition(results)) = result {
let combined_bytes = GrammarResult::Repetition(results).to_bytes();
assert_eq!(combined_bytes, b"aaa".to_vec());
} else {
panic!("Expected Repetition result");
}
}
#[test]
fn test_expression_parser_zero_or_one() {
// Create zero-or-one: (? "maybe")
let maybe_expr = GrammarNode::Terminal(Literal::from_str("maybe"));
let option = Repeat::with_max(Box::new(maybe_expr), 1);
let expr = GrammarNode::ZeroOrOne(option);
let mut input = Cursor::new(b"maybe");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(
result.is_ok(),
"Expression parser should work for ZeroOrOne"
);
if let Ok(GrammarResult::Optional(Some(boxed_result))) = result {
assert_eq!(boxed_result.to_bytes(), b"maybe".to_vec());
} else {
panic!("Expected Optional(Some) result");
}
}
#[test]
fn test_expression_parser_sequential_end() {
// Create sequence ending: Sequence<"hello", ()>
let hello_expr = GrammarNode::Terminal(Literal::from_str("hello"));
let sequence = Sequence::new(Box::new(hello_expr), ());
let expr = GrammarNode::SequentialEnd(sequence);
let mut input = Cursor::new(b"hello");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(
result.is_ok(),
"Expression parser should work for SequentialEnd"
);
if let Ok(GrammarResult::Literal(bytes)) = result {
assert_eq!(bytes, b"hello".to_vec());
} else {
panic!("Expected Literal result from SequentialEnd");
}
}
#[test]
fn test_expression_parser_complex_nested() {
// Test a complex nested expression with the actual Expression Parser
// This verifies that the delegation works correctly for complex structures
// Build manually: Sequence<"start", Sequence<Digits, ()>>
let start_expr = GrammarNode::Terminal(Literal::from_str("start"));
let digits_expr = GrammarNode::Digits(Utf8Class::digits());
let inner_seq = Sequence::new(Box::new(digits_expr), ());
let inner_expr = GrammarNode::SequentialEnd(inner_seq);
let outer_seq = Sequence::new(Box::new(start_expr), Box::new(inner_expr));
let expr = GrammarNode::Sequential(outer_seq);
let mut input = Cursor::new(b"start123");
let mut source = crate::parser::Source::new(&mut input);
let result = parse_with_context(expr, &mut source, &mut GrammarContext::default());
assert!(
result.is_ok(),
"Expression parser should work for complex nested structures"
);
if let Ok(GrammarResult::Sequence(results)) = result {
let combined_bytes = GrammarResult::Sequence(results).to_bytes();
assert_eq!(combined_bytes, b"start123".to_vec());
} else {
panic!("Expected Sequence result from complex nested structure");
}
}
#[test]
fn test_expression_result_to_bytes() {
// Test the to_bytes() method on various GrammarResult types
// Test Literal
let literal_result = GrammarResult::Literal(b"hello".to_vec());
assert_eq!(literal_result.to_bytes(), b"hello".to_vec());
// Test Unicode
let unicode_result = GrammarResult::Unicode("world".to_string());
assert_eq!(unicode_result.to_bytes(), b"world".to_vec());
// Test Bytes
let bytes_result = GrammarResult::Bytes(b"test".to_vec());
assert_eq!(bytes_result.to_bytes(), b"test".to_vec());
// Test Sequence
let seq_result = GrammarResult::Sequence(vec![
GrammarResult::Literal(b"hello".to_vec()),
GrammarResult::Literal(b"world".to_vec()),
]);
assert_eq!(seq_result.to_bytes(), b"helloworld".to_vec());
// Test Alternative
let alt_result =
GrammarResult::Alternative(Box::new(GrammarResult::Literal(b"choice".to_vec())));
assert_eq!(alt_result.to_bytes(), b"choice".to_vec());
// Test Repetition
let rep_result = GrammarResult::Repetition(vec![
GrammarResult::Literal(b"a".to_vec()),
GrammarResult::Literal(b"b".to_vec()),
GrammarResult::Literal(b"c".to_vec()),
]);
assert_eq!(rep_result.to_bytes(), b"abc".to_vec());
// Test Optional (Some)
let opt_some_result =
GrammarResult::Optional(Some(Box::new(GrammarResult::Literal(b"maybe".to_vec()))));
assert_eq!(opt_some_result.to_bytes(), b"maybe".to_vec());
// Test Optional (None)
let opt_none_result = GrammarResult::Optional(None);
assert_eq!(opt_none_result.to_bytes(), Vec::<u8>::new());
// Test Empty
let empty_result = GrammarResult::Empty;
assert_eq!(empty_result.to_bytes(), Vec::<u8>::new());
}
#[test]
fn test_expression_result_is_empty() {
// Test the is_empty() method
assert!(!GrammarResult::Literal(b"hello".to_vec()).is_empty());
assert!(!GrammarResult::Unicode("world".to_string()).is_empty());
assert!(!GrammarResult::Bytes(b"test".to_vec()).is_empty());
assert!(!GrammarResult::Sequence(vec![]).is_empty()); // Empty sequence is not considered "empty"
assert!(
!GrammarResult::Alternative(Box::new(GrammarResult::Literal(b"a".to_vec()))).is_empty()
);
assert!(!GrammarResult::Repetition(vec![]).is_empty()); // Empty repetition is not considered "empty"
assert!(
!GrammarResult::Optional(Some(Box::new(GrammarResult::Literal(b"a".to_vec()))))
.is_empty()
);
assert!(GrammarResult::Optional(None).is_empty());
assert!(GrammarResult::Empty.is_empty());
}
#[test]
fn test_id_implementation_grammar_expression() {
// GrammarParser::new() is a unit struct with no parameters, so default id() is correct
let grammar1 = GrammarParser::new();
let grammar2 = GrammarParser::new();
// These should have the same ID since GrammarParser::new() has no parameters
assert_eq!(
<GrammarParser as crate::parser::Parser<()>>::id(&grammar1),
<GrammarParser as crate::parser::Parser<()>>::id(&grammar2),
"GrammarParser::new() instances should have same ID since they have no parameters"
);
}
#[test]
fn test_id_implementation_different_expressions() {
// This test checks that Expression enum implements proper id() method
// Different Expression variants with different data should have different IDs
let expr1 = GrammarNode::Terminal(Literal::from_str("hello"));
let expr2 = GrammarNode::Terminal(Literal::from_str("world"));
let expr3 = GrammarNode::Digits(Utf8Class::digits());
// These should have different IDs because they represent different parsing behavior
// This test will FAIL if Expression uses default id() implementation
assert_ne!(
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr1),
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr2),
"Different Expression instances should have different IDs to avoid cache collisions"
);
assert_ne!(
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr1),
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr3),
"Different Expression variants should have different IDs to avoid cache collisions"
);
assert_ne!(
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr2),
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr3),
"Different Expression variants should have different IDs to avoid cache collisions"
);
}
#[test]
fn test_id_implementation_same_expressions() {
// Test that identical expressions have the same ID
let expr1 = GrammarNode::Terminal(Literal::from_str("hello"));
let expr2 = GrammarNode::Terminal(Literal::from_str("hello"));
assert_eq!(
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr1),
<GrammarNode as crate::parser::Parser<GrammarContext>>::id(&expr2),
"Identical Expression instances should have the same ID for cache efficiency"
);
}
#[test]
fn test_id_implementation_expression_cache_correctness() {
use std::io::Cursor;
// This test verifies that cache works correctly without collisions
// when Expression implements proper id() method
let expr1 = GrammarNode::Terminal(Literal::from_str("hello"));
let expr2 = GrammarNode::Terminal(Literal::from_str("world"));
// Test parsing "hello" with expr1
let mut input1 = Cursor::new(b"hello");
let mut source1 = crate::parser::Source::new(&mut input1);
let result1 = parse_with_context(expr1, &mut source1, &mut GrammarContext::default());
assert!(result1.is_ok(), "First parse should succeed");
// Parse "world" with expr2 - this should succeed and not use cached result from expr1
let mut input2 = Cursor::new(b"world");
let mut source2 = crate::parser::Source::new(&mut input2);
let result2 = parse_with_context(expr2, &mut source2, &mut GrammarContext::default());
assert!(result2.is_ok(), "Second parse should succeed");
// Verify the results are different
if let (Ok(GrammarResult::Literal(bytes1)), Ok(GrammarResult::Literal(bytes2))) =
(result1, result2)
{
assert_ne!(
bytes1, bytes2,
"Results should be different - no cache collision should occur"
);
assert_eq!(bytes1, b"hello".to_vec());
assert_eq!(bytes2, b"world".to_vec());
} else {
panic!("Expected literal results");
}
}
#[test]
fn test_id_implementation_box_expression() {
// Test that Box<Grammar> properly implements id() method
let boxed_expr1 = Box::new(GrammarNode::Terminal(Literal::from_str("test1")));
let boxed_expr2 = Box::new(GrammarNode::Terminal(Literal::from_str("test2")));
// These should have different IDs since they contain different expressions
// This test will FAIL if Box<Grammar> uses default id() implementation
let id1 = <Box<GrammarNode> as crate::parser::Parser<GrammarContext>>::id(&boxed_expr1);
let id2 = <Box<GrammarNode> as crate::parser::Parser<GrammarContext>>::id(&boxed_expr2);
assert_ne!(
id1, id2,
"Different Box<Grammar> instances should have different IDs to avoid cache collisions"
);
}
#[test]
fn test_id_implementation_same_box_expression() {
// Test that identical boxed expressions have the same ID
let boxed_expr1 = Box::new(GrammarNode::Terminal(Literal::from_str("test")));
let boxed_expr2 = Box::new(GrammarNode::Terminal(Literal::from_str("test")));
assert_eq!(
<Box<GrammarNode> as crate::parser::Parser<GrammarContext>>::id(&boxed_expr1),
<Box<GrammarNode> as crate::parser::Parser<GrammarContext>>::id(&boxed_expr2),
"Identical Box<GrammarNode> instances should have the same ID for cache efficiency"
);
}
#[test]
fn test_id_implementation_different_keyword_parsers() {
// Test that different keyword parsers have different IDs
// These parsers are unit structs so default id() implementation is correct
let digits_parser = Digits;
let alpha_parser = Alpha;
let whitespace_parser = Whitespace;
// These should have different IDs since they're different types
assert_ne!(
<Digits as crate::parser::Parser<GrammarContext>>::id(&digits_parser),
<Alpha as crate::parser::Parser<GrammarContext>>::id(&alpha_parser),
"Different parser types should have different IDs"
);
assert_ne!(
<Alpha as crate::parser::Parser<GrammarContext>>::id(&alpha_parser),
<Whitespace as crate::parser::Parser<GrammarContext>>::id(&whitespace_parser),
"Different parser types should have different IDs"
);
assert_ne!(
<Digits as crate::parser::Parser<GrammarContext>>::id(&digits_parser),
<Whitespace as crate::parser::Parser<GrammarContext>>::id(&whitespace_parser),
"Different parser types should have different IDs"
);
// Same type instances should have same ID
let digits_parser2 = Digits;
assert_eq!(
<Digits as crate::parser::Parser<GrammarContext>>::id(&digits_parser),
<Digits as crate::parser::Parser<GrammarContext>>::id(&digits_parser2),
"Same parser type instances should have same ID"
);
}
#[test]
fn test_read_until_parse_functionality() {
// Test that ReadUntilParse actually parses captured content
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" "captured")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "ReadUntilParse parsing should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntilParse(_, _) => {
// Expected: this should create a ReadUntilParse expression
// The actual functionality will be tested when the expression is executed
}
_ => panic!("Expected ReadUntilParse expression, got {expr:?}"),
}
}
}
// Comprehensive tests for ReadUntil and ReadUntilParse
#[test]
fn test_read_until_basic_parsing() {
// Test basic ReadUntil syntax parsing
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "ReadUntil parsing should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: this should create a ReadUntil expression
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_complex_end_condition() {
// Test ReadUntil with complex end condition
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< (| "end" "stop"))"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntil with complex end condition should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: should handle complex end conditions
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_parse_complex_parser() {
// Test ReadUntilParse with complex content parser
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" (| "hello" "world"))"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntilParse with complex parser should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntilParse(_, _) => {
// Expected: should handle complex content parsers
}
_ => panic!("Expected ReadUntilParse expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_with_whitespace() {
// Test ReadUntil parsing with whitespace in syntax
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" )"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "ReadUntil with whitespace should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: whitespace should be handled properly
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_parse_with_whitespace() {
// Test ReadUntilParse parsing with whitespace in syntax
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" "content" )"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntilParse with whitespace should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntilParse(_, _) => {
// Expected: whitespace should be handled properly
}
_ => panic!("Expected ReadUntilParse expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_nested() {
// Test ReadUntil nested inside other expressions
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(* (< "end"))"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "Nested ReadUntil should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::ZeroOrMore(_) => {
// Expected: should create a repetition of ReadUntil
}
_ => panic!("Expected ZeroOrMore expression containing ReadUntil, got {expr:?}",),
}
}
}
#[test]
fn test_read_until_parse_nested() {
// Test ReadUntilParse nested inside other expressions
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(+ (< "end" "content"))"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "Nested ReadUntilParse should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::OneOrMore(_) => {
// Expected: should create a repetition of ReadUntilParse
}
_ => {
panic!("Expected OneOrMore expression containing ReadUntilParse, got {expr:?}",)
}
}
}
}
#[test]
fn test_read_until_in_sequence() {
// Test ReadUntil as part of a sequence
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"("start" (< "end") "finish")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "ReadUntil in sequence should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::Sequential(_) => {
// Expected: should create a sequence containing ReadUntil
}
_ => panic!("Expected Sequential expression containing ReadUntil, got {expr:?}",),
}
}
}
#[test]
fn test_read_until_parse_in_alternatives() {
// Test ReadUntilParse as part of alternatives
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(| (< "end" "content") "fallback")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntilParse in alternatives should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::Alternatives(_) => {
// Expected: should create alternatives containing ReadUntilParse
}
_ => panic!(
"Expected Alternatives expression containing ReadUntilParse, got {expr:?}",
),
}
}
}
#[test]
fn test_read_until_with_keyword_end() {
// Test ReadUntil with keyword end condition
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< digits)"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_ok(), "ReadUntil with keyword end should succeed");
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: should handle keyword end conditions
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_parse_with_keyword_parser() {
// Test ReadUntilParse with keyword content parser
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" alpha)"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntilParse with keyword parser should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntilParse(_, _) => {
// Expected: should handle keyword content parsers
}
_ => panic!("Expected ReadUntilParse expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_with_character_class() {
// Test ReadUntil with character class end condition
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< [abc])"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntil with character class should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: should handle character class end conditions
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_parse_with_character_class_parser() {
// Test ReadUntilParse with character class content parser
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" [0-9])"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntilParse with character class parser should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntilParse(_, _) => {
// Expected: should handle character class content parsers
}
_ => panic!("Expected ReadUntilParse expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_malformed_missing_closing_paren() {
// Test error handling for malformed ReadUntil syntax
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end""#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_err(), "Malformed ReadUntil should fail");
}
#[test]
fn test_read_until_parse_malformed_missing_content_parser() {
// Test error handling for ReadUntilParse missing content parser
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end")"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
// This should parse as ReadUntil, not ReadUntilParse
assert!(
result.is_ok(),
"ReadUntil (not ReadUntilParse) should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: should parse as ReadUntil when only one argument
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_parse_malformed_missing_closing_paren() {
// Test error handling for malformed ReadUntilParse syntax
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" "content""#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(result.is_err(), "Malformed ReadUntilParse should fail");
}
#[test]
fn test_read_until_deeply_nested_expressions() {
// Test ReadUntil with deeply nested end conditions
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< ("prefix" (| "end1" "end2")))"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntil with deeply nested end should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntil(_) => {
// Expected: should handle deeply nested expressions
}
_ => panic!("Expected ReadUntil expression, got {expr:?}"),
}
}
}
#[test]
fn test_read_until_parse_deeply_nested_content_parser() {
// Test ReadUntilParse with deeply nested content parser
let grammar = GrammarParser::new();
let mut input = Cursor::new(r#"(< "end" (* (| "hello" digits)))"#.as_bytes());
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar, &mut source);
assert!(
result.is_ok(),
"ReadUntilParse with deeply nested content parser should succeed"
);
if let Ok(expr) = result {
match expr.node {
GrammarNode::ReadUntilParse(_, _) => {
// Expected: should handle deeply nested content parsers
}
_ => panic!("Expected ReadUntilParse expression, got {expr:?}"),
}
}
}
// Tests for new grammar features
#[test]
fn test_start_directive_basic() {
let parser = StartDirective;
let mut input = Cursor::new(b"@start expr");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
assert_eq!(result, "expr".to_string());
}
#[test]
fn test_start_directive_with_whitespace() {
let parser = StartDirective;
let mut input = Cursor::new(b"@start main");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
assert_eq!(result, "main".to_string());
}
#[test]
fn test_start_directive_invalid() {
let parser = StartDirective;
let mut input = Cursor::new(b"start expr");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_err());
}
#[test]
fn test_rule_definition_basic() {
let parser = RuleDefinition;
let mut input = Cursor::new(b"expr = digits");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
assert_eq!(result.0, "expr".to_string());
// The result.1 should be a GrammarNode::Digits variant
match result.1 {
GrammarNode::Digits(_) => {}
_ => panic!("Expected Digits node, got {:?}", result.1),
}
}
#[test]
fn test_rule_definition_with_whitespace() {
let parser = RuleDefinition;
let mut input = Cursor::new(b"term = alpha");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
assert_eq!(result.0, "term".to_string());
match result.1 {
GrammarNode::Alpha(_) => {}
_ => panic!("Expected Alpha node, got {:?}", result.1),
}
}
#[test]
fn test_rule_reference_parser_with_known_rule() {
let parser = RuleReferenceParser;
let mut input = Cursor::new(b"expr");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
// Add a rule to the context so it's "known"
context.rules.insert(
"expr".to_string(),
GrammarNode::Digits(crate::utf8class::Utf8Class::digits()),
);
let result = parse_with_context(parser, &mut source, &mut context).unwrap();
match result {
GrammarNode::RuleReference(name) => assert_eq!(name, "expr".to_string()),
_ => panic!("Expected RuleReference, got {result:?}"),
}
}
#[test]
fn test_rule_reference_parser_with_unknown_rule() {
let parser = RuleReferenceParser;
let mut input = Cursor::new(b"unknown");
let mut source = crate::parser::Source::new(&mut input);
let mut context = GrammarContext::default();
// RuleReferenceParser should succeed in parsing the identifier
// but the rule reference will fail when executed (not during parsing)
let result = parse_with_context(parser, &mut source, &mut context);
assert!(result.is_ok());
if let GrammarNode::RuleReference(name) = result.unwrap() {
assert_eq!(name, "unknown");
} else {
panic!("Expected RuleReference");
}
}
// Tests moved from tests/grammar_features.rs (unit tests only)
#[test]
fn test_grammar_parser_basic_expressions_still_work() {
// Test that basic expressions still work with our new Grammar system
let grammar_parser = GrammarParser::new();
// Test digits keyword
let mut input = Cursor::new(b"digits");
let mut source = crate::parser::Source::new(&mut input);
let grammar = parse_with_context(
grammar_parser.clone(),
&mut source,
&mut GrammarContext::default(),
)
.expect("Should parse 'digits' keyword");
// Test that the grammar can parse actual digits
let mut test_input = Cursor::new(b"123");
let mut test_source = crate::parser::Source::new(&mut test_input);
let result = parse_with_context(grammar, &mut test_source, &mut GrammarContext::default())
.expect("Should parse '123' with digits grammar");
match result {
GrammarResult::Unicode(s) => assert_eq!(s, "123"),
_ => panic!("Expected Unicode result for digits, got {result:?}"),
}
}
#[test]
fn test_grammar_parser_terminal_strings() {
let grammar_parser = GrammarParser::new();
// Test terminal string
let mut input = Cursor::new(b"\"hello\"");
let mut source = crate::parser::Source::new(&mut input);
let grammar =
parse_with_context(grammar_parser, &mut source, &mut GrammarContext::default())
.expect("Should parse terminal string");
// Test that the grammar can parse the literal "hello"
let mut test_input = Cursor::new(b"hello");
let mut test_source = crate::parser::Source::new(&mut test_input);
let result = parse_with_context(grammar, &mut test_source, &mut GrammarContext::default())
.expect("Should parse 'hello' with terminal grammar");
match result {
GrammarResult::Literal(bytes) => assert_eq!(bytes, b"hello"),
_ => panic!("Expected Literal result for terminal, got {result:?}"),
}
}
#[test]
fn test_rule_reference_succeeds_without_context() {
// This test asserts that parsing an identifier as a rule reference succeeds
// when no rules are defined in the context (forward reference allowed)
let grammar_parser = GrammarParser::new();
let mut input = Cursor::new(b"unknownrule");
let mut source = crate::parser::Source::new(&mut input);
// This should succeed because "unknownrule" is parsed as a rule reference
// (failure happens at execution time, not parse time)
let result =
parse_with_context(grammar_parser, &mut source, &mut GrammarContext::default());
// With our updated implementation, this succeeds as a rule reference
// but will fail at execution time if the rule doesn't exist
assert!(
result.is_ok(),
"Parsing unknown identifier should succeed as a rule reference"
);
}
#[test]
fn test_start_directive_parsed_fail_if_no_such_rule() {
let grammar_parser = GrammarParser::new();
let mut input = Cursor::new(b"@start expr\nother = alpha");
let mut source = crate::parser::Source::new(&mut input);
let result = parse(grammar_parser, &mut source);
result.expect_err(
"GrammarParser should fail if @start directive refers to an nonexistent rule",
);
}
#[test]
fn test_rule_reference_with_context() {
// Test that a rule reference works when the rule exists in context
let grammar_parser = GrammarParser::new();
// First parse a grammar that defines a rule
let mut input1 = Cursor::new(b"number = digits");
let mut source1 = crate::parser::Source::new(&mut input1);
let _grammar1 = parse_with_context(
grammar_parser.clone(),
&mut source1,
&mut GrammarContext::default(),
)
.expect("Should parse rule definition");
// Now test parsing a rule reference using a grammar with that context
let mut input2 = Cursor::new(b"number");
let mut source2 = crate::parser::Source::new(&mut input2);
// Create a new grammar parser instance but we need to use the context from grammar1
// This reveals a limitation - we need a way to parse with existing context
// For now, test that parsing "number" as an identifier succeeds without context
let result =
parse_with_context(grammar_parser, &mut source2, &mut GrammarContext::default());
// With our updated implementation, this succeeds as a rule reference
// but will fail at execution time if the rule doesn't exist
assert!(
result.is_ok(),
"Rule reference parsing should succeed, execution will fail if rule doesn't exist"
);
}
}