Parse Implementations

Convert token streams into AST nodes.

Basic Pattern

impl Parse for MyNode {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        Ok(Self {
            field1: stream.parse()?,
            field2: stream.parse()?,
        })
    }
}

Implementing Peek

For types used in conditionals:

impl Peek for SimpleKey {
    fn is(token: &Token) -> bool {
        matches!(token, Token::BareKey(_) | Token::BasicString(_))
    }
}

impl Parse for SimpleKey {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        match stream.peek_token().map(|t| &t.value) {
            Some(Token::BareKey(_)) => {
                let tok: Spanned<tokens::BareKeyToken> = stream.parse()?;
                Ok(SimpleKey::Bare(tok.value))
            }
            Some(Token::BasicString(_)) => {
                let tok: Spanned<tokens::BasicStringToken> = stream.parse()?;
                Ok(SimpleKey::Quoted(tok.value))
            }
            Some(other) => Err(TomlError::Expected {
                expect: "key",
                found: format!("{}", other),
            }),
            None => Err(TomlError::Empty { expect: "key" }),
        }
    }
}

Peek::is() checks a token variant; Peek::peek() checks the stream’s next token.

Parsing Keys

impl Peek for Key {
    fn is(token: &Token) -> bool {
        SimpleKey::is(token)
    }
}

impl Parse for Key {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        let first: Spanned<SimpleKey> = stream.parse()?;

        // Check if this is a dotted key
        if stream.peek::<tokens::DotToken>() {
            let mut rest = Vec::new();
            while stream.peek::<tokens::DotToken>() {
                let dot: Spanned<tokens::DotToken> = stream.parse()?;
                let key: Spanned<SimpleKey> = stream.parse()?;
                rest.push((dot, key));
            }
            Ok(Key::Dotted(DottedKey { first, rest }))
        } else {
            // Single key
            match first.value {
                SimpleKey::Bare(tok) => Ok(Key::Bare(tok)),
                SimpleKey::Quoted(tok) => Ok(Key::Quoted(tok)),
            }
        }
    }
}

Parsing Values

Match on peeked token to determine variant:

impl Peek for Value {
    fn is(token: &Token) -> bool {
        matches!(
            token,
            Token::BasicString(_)
                | Token::Integer(_)
                | Token::True
                | Token::False
                | Token::LBracket
                | Token::LBrace
        )
    }
}

impl Parse for Value {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        match stream.peek_token().map(|t| &t.value) {
            Some(Token::BasicString(_)) => {
                let tok: Spanned<tokens::BasicStringToken> = stream.parse()?;
                Ok(Value::String(tok.value))
            }
            Some(Token::Integer(_)) => {
                let tok: Spanned<tokens::IntegerToken> = stream.parse()?;
                Ok(Value::Integer(tok.value))
            }
            Some(Token::True) => {
                let tok: Spanned<tokens::TrueToken> = stream.parse()?;
                Ok(Value::True(tok.value))
            }
            Some(Token::False) => {
                let tok: Spanned<tokens::FalseToken> = stream.parse()?;
                Ok(Value::False(tok.value))
            }
            Some(Token::LBracket) => {
                let arr = Array::parse(stream)?;
                Ok(Value::Array(arr))
            }
            Some(Token::LBrace) => {
                let tbl = InlineTable::parse(stream)?;
                Ok(Value::InlineTable(tbl))
            }
            Some(other) => Err(TomlError::Expected {
                expect: "value",
                found: format!("{}", other),
            }),
            None => Err(TomlError::Empty { expect: "value" }),
        }
    }
}

Arrays with Delimiters

Use the bracket! macro to extract delimited content:

impl Peek for Array {
    fn is(token: &Token) -> bool {
        matches!(token, Token::LBracket)
    }
}

impl Parse for Array {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        let lbracket: Spanned<tokens::LBracketToken> = stream.parse()?;

        let mut items = Vec::new();

        // Skip any leading newlines inside array
        while peek_newline(stream) {
            let _: Spanned<tokens::NewlineToken> = stream.parse()?;
        }

        // Parse array items
        while stream.peek::<Value>() {
            let value: Spanned<Value> = stream.parse()?;

            // Skip newlines after value
            while peek_newline(stream) {
                let _: Spanned<tokens::NewlineToken> = stream.parse()?;
            }

            let comma = if stream.peek::<tokens::CommaToken>() {
                let c: Spanned<tokens::CommaToken> = stream.parse()?;
                // Skip newlines after comma
                while peek_newline(stream) {
                    let _: Spanned<tokens::NewlineToken> = stream.parse()?;
                }
                Some(c)
            } else {
                None
            };

            items.push(ArrayItem { value, comma });
        }

        let rbracket: Spanned<tokens::RBracketToken> = stream.parse()?;

        Ok(Array {
            lbracket,
            items,
            rbracket,
        })
    }
}

Key points:

• bracket!(inner in stream) extracts content between [ and ]
• Returns a Bracket struct with span information
• inner is a new TokenStream containing only bracket contents

Inline Tables

Similar pattern with brace!:

impl Peek for InlineTable {
    fn is(token: &Token) -> bool {
        matches!(token, Token::LBrace)
    }
}

impl Parse for InlineTable {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        let lbrace: Spanned<tokens::LBraceToken> = stream.parse()?;

        let mut items = Vec::new();

        // Parse inline table items
        while stream.peek::<Key>() {
            let kv: Spanned<KeyValue> = stream.parse()?;

            let comma = if stream.peek::<tokens::CommaToken>() {
                Some(stream.parse()?)
            } else {
                None
            };

            items.push(InlineTableItem { kv, comma });
        }

        let rbrace: Spanned<tokens::RBraceToken> = stream.parse()?;

        Ok(InlineTable {
            lbrace,
            items,
            rbrace,
        })
    }
}

Tables and Documents

impl Peek for Table {
    fn is(token: &Token) -> bool {
        matches!(token, Token::LBracket)
    }
}

impl Parse for Table {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        let lbracket: Spanned<tokens::LBracketToken> = stream.parse()?;
        let name: Spanned<Key> = stream.parse()?;
        let rbracket: Spanned<tokens::RBracketToken> = stream.parse()?;

        let mut items = Vec::new();

        // Consume trailing content on the header line
        while Trivia::peek(stream) {
            let trivia = Trivia::parse(stream)?;
            items.push(TableItem::Trivia(trivia));
            // Stop after we hit a newline
            if matches!(items.last(), Some(TableItem::Trivia(Trivia::Newline(_)))) {
                break;
            }
        }

        // Parse table contents until we hit another table or EOF
        loop {
            // Check for trivia (newlines, comments)
            if Trivia::peek(stream) {
                let trivia = Trivia::parse(stream)?;
                items.push(TableItem::Trivia(trivia));
                continue;
            }

            // Check for key-value pair
            if stream.peek::<Key>() {
                // But first make sure this isn't a table header by checking for `[`
                // This is tricky - we need to distinguish `[table]` from key-value
                // Since Key::peek checks for bare keys and strings, and table headers
                // start with `[`, we need to check `[` first in the document parser
                let kv = Box::new(stream.parse()?);
                items.push(TableItem::KeyValue(kv));
                continue;
            }

            // Either EOF or another table section
            break;
        }

        Ok(Table {
            lbracket,
            name,
            rbracket,
            items,
        })
    }
}

impl Peek for DocumentItem {
    fn is(token: &Token) -> bool {
        Trivia::is(token) || Key::is(token) || matches!(token, Token::LBracket)
    }
}

impl Parse for Document {
    fn parse(stream: &mut TokenStream) -> Result<Self, TomlError> {
        let mut items = Vec::new();

        loop {
            // Check for trivia (newlines, comments)
            if Trivia::peek(stream) {
                let trivia = Trivia::parse(stream)?;
                items.push(DocumentItem::Trivia(trivia));
                continue;
            }

            // Check for table header `[name]`
            if stream.peek::<tokens::LBracketToken>() {
                let table: Spanned<Table> = stream.parse()?;
                items.push(DocumentItem::Table(table));
                continue;
            }

            // Check for key-value pair
            if stream.peek::<Key>() {
                let kv: Spanned<KeyValue> = stream.parse()?;
                items.push(DocumentItem::KeyValue(kv));
                continue;
            }

            // EOF or unknown token
            if stream.is_empty() {
                break;
            }

            // Unknown token - error
            if let Some(tok) = stream.peek_token() {
                return Err(TomlError::Expected {
                    expect: "key, table, or end of file",
                    found: format!("{}", tok.value),
                });
            }
            break;
        }

        Ok(Document { items })
    }
}

Error Handling

Expected Token Errors

Some(other) => Err(TomlError::Expected {
    expect: "key",
    found: format!("{}", other),
}),

EOF Errors

None => Err(TomlError::Empty { expect: "key" }),

Using Diagnostic

// Auto-generated for tokens
impl Diagnostic for BareKeyToken {
    fn fmt() -> &'static str { "bare key" }  // From #[fmt("bare key")]
}

// Use in errors
Err(TomlError::expected::<BareKeyToken>(found_token))

Parsing Tips

Use peek Before Consuming

if SimpleKey::peek(stream) {
    // Safe to parse
    let key: Spanned<SimpleKey> = stream.parse()?;
}

Fork for Lookahead

let mut fork = stream.fork();
if try_parse(&mut fork).is_ok() {
    stream.advance_to(&fork);
}

Handle Optional Elements

// Option<T> auto-implements Parse if T implements Peek
let comma: Option<Spanned<CommaToken>> = stream.parse()?;

Raw Token Access

For tokens in skip_tokens (like Newline):

// Use peek_token_raw to see skipped tokens
fn peek_raw(stream: &TokenStream) -> Option<&Token> {
    stream.peek_token_raw().map(|t| &t.value)
}