use ::neotoma::{ Source, cache::BasicCache, grammar::{Grammar, GrammarParser, GrammarResult}, parse, parser::Parser, }; use pyo3::prelude::*; use pyo3::types::{PyList, PyTuple}; use std::io::Cursor; use thiserror::Error; #[derive(Error, Debug)] pub enum PyNeotomaError { #[error("Parse error: {0}")] ParseError(String), #[error("IO error: {0}")] IoError(String), } impl From for PyErr { fn from(err: PyNeotomaError) -> PyErr { match err { PyNeotomaError::ParseError(msg) => pyo3::exceptions::PyValueError::new_err(msg), PyNeotomaError::IoError(msg) => pyo3::exceptions::PyIOError::new_err(msg), } } } /// Flatten nested sequences into a single tuple fn flatten_sequence(py: Python, results: &[GrammarResult]) -> PyResult> { let mut flattened = Vec::new(); for item in results { match item { GrammarResult::Sequence(nested_results) => { // Recursively flatten nested sequences flattened.extend(flatten_sequence(py, nested_results)?); } _ => { // Convert non-sequence items normally flattened.push(grammar_result_to_python(py, item)?); } } } Ok(flattened) } /// Flatten nested repetitions into a single list fn flatten_repetition(py: Python, results: &[GrammarResult]) -> PyResult> { let mut flattened = Vec::new(); for item in results { match item { GrammarResult::Repetition(nested_results) => { // Recursively flatten nested repetitions flattened.extend(flatten_repetition(py, nested_results)?); } _ => { // Convert non-repetition items normally flattened.push(grammar_result_to_python(py, item)?); } } } Ok(flattened) } /// Convert GrammarResult to Python objects fn grammar_result_to_python(py: Python, result: &GrammarResult) -> PyResult { match result { GrammarResult::Literal(bytes) => { // Return literal as Python bytes Ok(pyo3::types::PyBytes::new(py, bytes).into_any().unbind()) } GrammarResult::Unicode(string) => { // Return unicode string directly Ok(pyo3::types::PyString::new(py, string).into_any().unbind()) } GrammarResult::Bytes(bytes) => { // Return as Python bytes Ok(pyo3::types::PyBytes::new(py, bytes).into_any().unbind()) } GrammarResult::Sequence(results) => { // Return flattened sequence as Python tuple let py_items = flatten_sequence(py, results)?; Ok(PyTuple::new(py, py_items)?.into_any().unbind()) } GrammarResult::Alternative(result) => { // Return the alternative result directly (unwrap the box) grammar_result_to_python(py, result) } GrammarResult::Repetition(results) => { // Return flattened repetition as Python list let py_items = flatten_repetition(py, results)?; Ok(PyList::new(py, py_items)?.into_any().unbind()) } GrammarResult::Optional(Some(result)) => { // Return the optional result directly (unwrap the box) grammar_result_to_python(py, result) } GrammarResult::Optional(None) | GrammarResult::Empty => { // Return None for empty/missing results Ok(py.None()) } } } /// A compiled grammar that can parse input text according to defined rules. /// /// Grammar objects are created from grammar text using Grammar() /// Once created, they can be used repeatedly to parse different input texts. #[pyclass(name = "Grammar")] pub struct PyGrammar { grammar: Grammar, } #[pymethods] impl PyGrammar { /// Create a new grammar from grammar text. /// /// Args: /// grammar_text: A string containing the grammar definition /// /// Returns: /// A compiled Grammar object /// /// Raises: /// ValueError: If the grammar text contains syntax errors /// /// Example: /// >>> grammar = Grammar('digits') /// >>> grammar = Grammar('(+ alpha / whitespace)') #[new] fn new(grammar_text: &str) -> PyResult { let parser = GrammarParser::new(); let mut source = Source::new(Cursor::new(grammar_text.as_bytes())); let grammar = parse(parser, &mut source) .map_err(|e| PyNeotomaError::ParseError(format!("{:?}", e)))?; Ok(Self { grammar }) } /// Parse input text using this grammar. /// /// Args: /// input_text: The text to parse /// /// Returns: /// A structured representation of the parsing result /// /// Raises: /// ValueError: If the input doesn't match the grammar /// /// Example: /// >>> grammar = Grammar('digits') /// >>> result = grammar.parse("12345") fn parse(&self, py: Python, input_text: &str) -> PyResult { let mut source = Source::new(Cursor::new(input_text.as_bytes())); let mut cache = BasicCache::new(); let mut context = (); let result = self .grammar .parse(&mut source, &mut cache, &mut context) .map_err(|e| PyNeotomaError::ParseError(format!("{:?}", e)))?; grammar_result_to_python(py, &result) } } /// Parse input with a grammar in one step (convenience function). /// /// This creates a grammar and immediately uses it to parse input. /// Use this for one-off parsing; for repeated parsing, create a Grammar object. /// /// Args: /// grammar_text: A string containing the grammar definition /// input_text: The text to parse /// /// Returns: /// A structured representation of the parsing result /// /// Raises: /// ValueError: If the grammar text contains syntax errors or input doesn't match /// /// Example: /// >>> result = parse_with_grammar('digits', "12345") #[pyfunction] fn parse_with_grammar(py: Python, grammar_text: &str, input_text: &str) -> PyResult { let grammar = PyGrammar::new(grammar_text)?; grammar.parse(py, input_text) } /// neotoma: Python bindings for the Neotoma parsing library /// /// This module provides Python access to Neotoma's powerful grammar-based parsing capabilities. /// /// ## Quick Start /// /// ```python /// import neotoma /// /// # Parse simple patterns /// result = neotoma.parse_with_grammar('digits', "12345") /// /// # Create reusable grammars /// grammar = neotoma.Grammar('(+ alpha / whitespace)') /// result = grammar.parse("hello world") /// /// # Ensure complete input consumption with eof /// strict_grammar = neotoma.Grammar('(alpha eof)') /// strict_grammar.parse("hello") # succeeds /// strict_grammar.parse("hello123") # fails - has trailing content /// ``` /// /// ## Grammar Syntax Summary /// /// - Literals: `"text"` /// - Built-ins: `digits`, `alpha`, `alphanumeric`, `whitespace`, `hexdigits`, `eof` /// - Unicode: `udigits`, `ualpha`, `ualphanumeric`, `uwhitespace` /// - Character classes: `[abc]`, `[^abc]` /// - Sequences: `(A B C)` /// - Alternatives: `(| A B C)` /// - Repetition: `(* A)`, `(+ A)`, `(? A)` /// - Separated: `(+ A / B)`, `(* A / B)` /// - Special: `(< A)`, `(< A B)` /// - Named rules: `name = rule`, `name`, `@start name` /// /// ## Built-in Parsers /// /// - `digits`: Matches one or more decimal digits (0-9) /// - `alpha`: Matches one or more alphabetic characters (ASCII a-z, A-Z) /// - `alphanumeric`: Matches one or more alphanumeric characters (ASCII) /// - `whitespace`: Matches one or more whitespace characters (ASCII) /// - `hexdigits`: Matches one or more hexadecimal digits (0-9, a-f, A-F) /// - `eof`: Matches end of file - ensures complete input consumption /// - `udigits`, `ualpha`, `ualphanumeric`, `uwhitespace`: Unicode versions /// /// ## End-of-File (`eof`) Usage /// /// The `eof` parser is essential for creating strict parsers that require /// complete input consumption. Without `eof`, parsers may succeed on partial /// matches, leaving trailing content unprocessed. /// /// ```python /// # Without eof - partial matches succeed /// loose_grammar = neotoma.Grammar('alpha') /// loose_grammar.parse("hello123") # succeeds, matches "hello", ignores "123" /// /// # With eof - requires complete consumption /// strict_grammar = neotoma.Grammar('(alpha eof)') /// strict_grammar.parse("hello") # succeeds /// strict_grammar.parse("hello123") # fails - "123" remains after "hello" /// /// # Common pattern for complete expressions /// expr_grammar = neotoma.Grammar('@start complete_expr\ncomplete_expr = (expression eof)') /// ``` /// /// See the README for complete documentation. #[pymodule] fn neotoma(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(parse_with_grammar, m)?)?; Ok(()) }