| //! Parsing interface for parsing a token stream into a syntax tree node. |
| //! |
| //! Parsing in Syn is built on parser functions that take in a [`ParseStream`] |
| //! and produce a [`Result<T>`] where `T` is some syntax tree node. Underlying |
| //! these parser functions is a lower level mechanism built around the |
| //! [`Cursor`] type. `Cursor` is a cheaply copyable cursor over a range of |
| //! tokens in a token stream. |
| //! |
| //! [`Result<T>`]: Result |
| //! [`Cursor`]: crate::buffer::Cursor |
| //! |
| //! # Example |
| //! |
| //! Here is a snippet of parsing code to get a feel for the style of the |
| //! library. We define data structures for a subset of Rust syntax including |
| //! enums (not shown) and structs, then provide implementations of the [`Parse`] |
| //! trait to parse these syntax tree data structures from a token stream. |
| //! |
| //! Once `Parse` impls have been defined, they can be called conveniently from a |
| //! procedural macro through [`parse_macro_input!`] as shown at the bottom of |
| //! the snippet. If the caller provides syntactically invalid input to the |
| //! procedural macro, they will receive a helpful compiler error message |
| //! pointing out the exact token that triggered the failure to parse. |
| //! |
| //! [`parse_macro_input!`]: crate::parse_macro_input! |
| //! |
| //! ``` |
| //! # extern crate proc_macro; |
| //! # |
| //! use proc_macro::TokenStream; |
| //! use syn::{braced, parse_macro_input, token, Field, Ident, Result, Token}; |
| //! use syn::parse::{Parse, ParseStream}; |
| //! use syn::punctuated::Punctuated; |
| //! |
| //! enum Item { |
| //! Struct(ItemStruct), |
| //! Enum(ItemEnum), |
| //! } |
| //! |
| //! struct ItemStruct { |
| //! struct_token: Token![struct], |
| //! ident: Ident, |
| //! brace_token: token::Brace, |
| //! fields: Punctuated<Field, Token![,]>, |
| //! } |
| //! # |
| //! # enum ItemEnum {} |
| //! |
| //! impl Parse for Item { |
| //! fn parse(input: ParseStream) -> Result<Self> { |
| //! let lookahead = input.lookahead1(); |
| //! if lookahead.peek(Token![struct]) { |
| //! input.parse().map(Item::Struct) |
| //! } else if lookahead.peek(Token![enum]) { |
| //! input.parse().map(Item::Enum) |
| //! } else { |
| //! Err(lookahead.error()) |
| //! } |
| //! } |
| //! } |
| //! |
| //! impl Parse for ItemStruct { |
| //! fn parse(input: ParseStream) -> Result<Self> { |
| //! let content; |
| //! Ok(ItemStruct { |
| //! struct_token: input.parse()?, |
| //! ident: input.parse()?, |
| //! brace_token: braced!(content in input), |
| //! fields: content.parse_terminated(Field::parse_named)?, |
| //! }) |
| //! } |
| //! } |
| //! # |
| //! # impl Parse for ItemEnum { |
| //! # fn parse(input: ParseStream) -> Result<Self> { |
| //! # unimplemented!() |
| //! # } |
| //! # } |
| //! |
| //! # const IGNORE: &str = stringify! { |
| //! #[proc_macro] |
| //! # }; |
| //! pub fn my_macro(tokens: TokenStream) -> TokenStream { |
| //! let input = parse_macro_input!(tokens as Item); |
| //! |
| //! /* ... */ |
| //! # "".parse().unwrap() |
| //! } |
| //! ``` |
| //! |
| //! # The `syn::parse*` functions |
| //! |
| //! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve |
| //! as an entry point for parsing syntax tree nodes that can be parsed in an |
| //! obvious default way. These functions can return any syntax tree node that |
| //! implements the [`Parse`] trait, which includes most types in Syn. |
| //! |
| //! [`syn::parse`]: crate::parse() |
| //! [`syn::parse2`]: crate::parse2() |
| //! [`syn::parse_str`]: crate::parse_str() |
| //! |
| //! ``` |
| //! use syn::Type; |
| //! |
| //! # fn run_parser() -> syn::Result<()> { |
| //! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?; |
| //! # Ok(()) |
| //! # } |
| //! # |
| //! # run_parser().unwrap(); |
| //! ``` |
| //! |
| //! The [`parse_quote!`] macro also uses this approach. |
| //! |
| //! [`parse_quote!`]: crate::parse_quote! |
| //! |
| //! # The `Parser` trait |
| //! |
| //! Some types can be parsed in several ways depending on context. For example |
| //! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like |
| //! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`] |
| //! may or may not allow trailing punctuation, and parsing it the wrong way |
| //! would either reject valid input or accept invalid input. |
| //! |
| //! [`Attribute`]: crate::Attribute |
| //! [`Punctuated`]: crate::punctuated |
| //! |
| //! The `Parse` trait is not implemented in these cases because there is no good |
| //! behavior to consider the default. |
| //! |
| //! ```compile_fail |
| //! # extern crate proc_macro; |
| //! # |
| //! # use syn::punctuated::Punctuated; |
| //! # use syn::{PathSegment, Result, Token}; |
| //! # |
| //! # fn f(tokens: proc_macro::TokenStream) -> Result<()> { |
| //! # |
| //! // Can't parse `Punctuated` without knowing whether trailing punctuation |
| //! // should be allowed in this context. |
| //! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?; |
| //! # |
| //! # Ok(()) |
| //! # } |
| //! ``` |
| //! |
| //! In these cases the types provide a choice of parser functions rather than a |
| //! single `Parse` implementation, and those parser functions can be invoked |
| //! through the [`Parser`] trait. |
| //! |
| //! |
| //! ``` |
| //! # extern crate proc_macro; |
| //! # |
| //! use proc_macro::TokenStream; |
| //! use syn::parse::Parser; |
| //! use syn::punctuated::Punctuated; |
| //! use syn::{Attribute, Expr, PathSegment, Result, Token}; |
| //! |
| //! fn call_some_parser_methods(input: TokenStream) -> Result<()> { |
| //! // Parse a nonempty sequence of path segments separated by `::` punctuation |
| //! // with no trailing punctuation. |
| //! let tokens = input.clone(); |
| //! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty; |
| //! let _path = parser.parse(tokens)?; |
| //! |
| //! // Parse a possibly empty sequence of expressions terminated by commas with |
| //! // an optional trailing punctuation. |
| //! let tokens = input.clone(); |
| //! let parser = Punctuated::<Expr, Token![,]>::parse_terminated; |
| //! let _args = parser.parse(tokens)?; |
| //! |
| //! // Parse zero or more outer attributes but not inner attributes. |
| //! let tokens = input.clone(); |
| //! let parser = Attribute::parse_outer; |
| //! let _attrs = parser.parse(tokens)?; |
| //! |
| //! Ok(()) |
| //! } |
| //! ``` |
| //! |
| //! --- |
| //! |
| //! *This module is available only if Syn is built with the `"parsing"` feature.* |
| |
| #[path = "discouraged.rs"] |
| pub mod discouraged; |
| |
| use crate::buffer::{Cursor, TokenBuffer}; |
| use crate::error; |
| use crate::lookahead; |
| #[cfg(all( |
| not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), |
| feature = "proc-macro" |
| ))] |
| use crate::proc_macro; |
| use crate::punctuated::Punctuated; |
| use crate::token::Token; |
| use proc_macro2::{self, Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree}; |
| use std::cell::Cell; |
| use std::fmt::{self, Debug, Display}; |
| #[cfg(feature = "extra-traits")] |
| use std::hash::{Hash, Hasher}; |
| use std::marker::PhantomData; |
| use std::mem; |
| use std::ops::Deref; |
| use std::rc::Rc; |
| use std::str::FromStr; |
| |
| pub use crate::error::{Error, Result}; |
| pub use crate::lookahead::{Lookahead1, Peek}; |
| |
| /// Parsing interface implemented by all types that can be parsed in a default |
| /// way from a token stream. |
| /// |
| /// Refer to the [module documentation] for details about implementing and using |
| /// the `Parse` trait. |
| /// |
| /// [module documentation]: self |
| pub trait Parse: Sized { |
| fn parse(input: ParseStream) -> Result<Self>; |
| } |
| |
| /// Input to a Syn parser function. |
| /// |
| /// See the methods of this type under the documentation of [`ParseBuffer`]. For |
| /// an overview of parsing in Syn, refer to the [module documentation]. |
| /// |
| /// [module documentation]: self |
| pub type ParseStream<'a> = &'a ParseBuffer<'a>; |
| |
| /// Cursor position within a buffered token stream. |
| /// |
| /// This type is more commonly used through the type alias [`ParseStream`] which |
| /// is an alias for `&ParseBuffer`. |
| /// |
| /// `ParseStream` is the input type for all parser functions in Syn. They have |
| /// the signature `fn(ParseStream) -> Result<T>`. |
| /// |
| /// ## Calling a parser function |
| /// |
| /// There is no public way to construct a `ParseBuffer`. Instead, if you are |
| /// looking to invoke a parser function that requires `ParseStream` as input, |
| /// you will need to go through one of the public parsing entry points. |
| /// |
| /// - The [`parse_macro_input!`] macro if parsing input of a procedural macro; |
| /// - One of [the `syn::parse*` functions][syn-parse]; or |
| /// - A method of the [`Parser`] trait. |
| /// |
| /// [syn-parse]: self#the-synparse-functions |
| pub struct ParseBuffer<'a> { |
| scope: Span, |
| // Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a. |
| // The rest of the code in this module needs to be careful that only a |
| // cursor derived from this `cell` is ever assigned to this `cell`. |
| // |
| // Cell<Cursor<'a>> cannot be covariant in 'a because then we could take a |
| // ParseBuffer<'a>, upcast to ParseBuffer<'short> for some lifetime shorter |
| // than 'a, and then assign a Cursor<'short> into the Cell. |
| // |
| // By extension, it would not be safe to expose an API that accepts a |
| // Cursor<'a> and trusts that it lives as long as the cursor currently in |
| // the cell. |
| cell: Cell<Cursor<'static>>, |
| marker: PhantomData<Cursor<'a>>, |
| unexpected: Cell<Option<Rc<Cell<Unexpected>>>>, |
| } |
| |
| impl<'a> Drop for ParseBuffer<'a> { |
| fn drop(&mut self) { |
| if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(self.cursor()) { |
| let (inner, old_span) = inner_unexpected(self); |
| if old_span.is_none() { |
| inner.set(Unexpected::Some(unexpected_span)); |
| } |
| } |
| } |
| } |
| |
| impl<'a> Display for ParseBuffer<'a> { |
| fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| Display::fmt(&self.cursor().token_stream(), f) |
| } |
| } |
| |
| impl<'a> Debug for ParseBuffer<'a> { |
| fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| Debug::fmt(&self.cursor().token_stream(), f) |
| } |
| } |
| |
| /// Cursor state associated with speculative parsing. |
| /// |
| /// This type is the input of the closure provided to [`ParseStream::step`]. |
| /// |
| /// [`ParseStream::step`]: ParseBuffer::step |
| /// |
| /// # Example |
| /// |
| /// ``` |
| /// use proc_macro2::TokenTree; |
| /// use syn::Result; |
| /// use syn::parse::ParseStream; |
| /// |
| /// // This function advances the stream past the next occurrence of `@`. If |
| /// // no `@` is present in the stream, the stream position is unchanged and |
| /// // an error is returned. |
| /// fn skip_past_next_at(input: ParseStream) -> Result<()> { |
| /// input.step(|cursor| { |
| /// let mut rest = *cursor; |
| /// while let Some((tt, next)) = rest.token_tree() { |
| /// match &tt { |
| /// TokenTree::Punct(punct) if punct.as_char() == '@' => { |
| /// return Ok(((), next)); |
| /// } |
| /// _ => rest = next, |
| /// } |
| /// } |
| /// Err(cursor.error("no `@` was found after this point")) |
| /// }) |
| /// } |
| /// # |
| /// # fn remainder_after_skipping_past_next_at( |
| /// # input: ParseStream, |
| /// # ) -> Result<proc_macro2::TokenStream> { |
| /// # skip_past_next_at(input)?; |
| /// # input.parse() |
| /// # } |
| /// # |
| /// # use syn::parse::Parser; |
| /// # let remainder = remainder_after_skipping_past_next_at |
| /// # .parse_str("a @ b c") |
| /// # .unwrap(); |
| /// # assert_eq!(remainder.to_string(), "b c"); |
| /// ``` |
| pub struct StepCursor<'c, 'a> { |
| scope: Span, |
| // This field is covariant in 'c. |
| cursor: Cursor<'c>, |
| // This field is contravariant in 'c. Together these make StepCursor |
| // invariant in 'c. Also covariant in 'a. The user cannot cast 'c to a |
| // different lifetime but can upcast into a StepCursor with a shorter |
| // lifetime 'a. |
| // |
| // As long as we only ever construct a StepCursor for which 'c outlives 'a, |
| // this means if ever a StepCursor<'c, 'a> exists we are guaranteed that 'c |
| // outlives 'a. |
| marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>, |
| } |
| |
| impl<'c, 'a> Deref for StepCursor<'c, 'a> { |
| type Target = Cursor<'c>; |
| |
| fn deref(&self) -> &Self::Target { |
| &self.cursor |
| } |
| } |
| |
| impl<'c, 'a> Copy for StepCursor<'c, 'a> {} |
| |
| impl<'c, 'a> Clone for StepCursor<'c, 'a> { |
| fn clone(&self) -> Self { |
| *self |
| } |
| } |
| |
| impl<'c, 'a> StepCursor<'c, 'a> { |
| /// Triggers an error at the current position of the parse stream. |
| /// |
| /// The `ParseStream::step` invocation will return this same error without |
| /// advancing the stream state. |
| pub fn error<T: Display>(self, message: T) -> Error { |
| error::new_at(self.scope, self.cursor, message) |
| } |
| } |
| |
| pub(crate) fn advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a> { |
| // Refer to the comments within the StepCursor definition. We use the |
| // fact that a StepCursor<'c, 'a> exists as proof that 'c outlives 'a. |
| // Cursor is covariant in its lifetime parameter so we can cast a |
| // Cursor<'c> to one with the shorter lifetime Cursor<'a>. |
| let _ = proof; |
| unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(to) } |
| } |
| |
| pub(crate) fn new_parse_buffer( |
| scope: Span, |
| cursor: Cursor, |
| unexpected: Rc<Cell<Unexpected>>, |
| ) -> ParseBuffer { |
| ParseBuffer { |
| scope, |
| // See comment on `cell` in the struct definition. |
| cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }), |
| marker: PhantomData, |
| unexpected: Cell::new(Some(unexpected)), |
| } |
| } |
| |
| pub(crate) enum Unexpected { |
| None, |
| Some(Span), |
| Chain(Rc<Cell<Unexpected>>), |
| } |
| |
| impl Default for Unexpected { |
| fn default() -> Self { |
| Unexpected::None |
| } |
| } |
| |
| impl Clone for Unexpected { |
| fn clone(&self) -> Self { |
| match self { |
| Unexpected::None => Unexpected::None, |
| Unexpected::Some(span) => Unexpected::Some(*span), |
| Unexpected::Chain(next) => Unexpected::Chain(next.clone()), |
| } |
| } |
| } |
| |
| // We call this on Cell<Unexpected> and Cell<Option<T>> where temporarily |
| // swapping in a None is cheap. |
| fn cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T { |
| let prev = cell.take(); |
| let ret = prev.clone(); |
| cell.set(prev); |
| ret |
| } |
| |
| fn inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<Span>) { |
| let mut unexpected = get_unexpected(buffer); |
| loop { |
| match cell_clone(&unexpected) { |
| Unexpected::None => return (unexpected, None), |
| Unexpected::Some(span) => return (unexpected, Some(span)), |
| Unexpected::Chain(next) => unexpected = next, |
| } |
| } |
| } |
| |
| pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>> { |
| cell_clone(&buffer.unexpected).unwrap() |
| } |
| |
| fn span_of_unexpected_ignoring_nones(mut cursor: Cursor) -> Option<Span> { |
| if cursor.eof() { |
| return None; |
| } |
| while let Some((inner, _span, rest)) = cursor.group(Delimiter::None) { |
| if let Some(unexpected) = span_of_unexpected_ignoring_nones(inner) { |
| return Some(unexpected); |
| } |
| cursor = rest; |
| } |
| if cursor.eof() { |
| None |
| } else { |
| Some(cursor.span()) |
| } |
| } |
| |
| impl<'a> ParseBuffer<'a> { |
| /// Parses a syntax tree node of type `T`, advancing the position of our |
| /// parse stream past it. |
| pub fn parse<T: Parse>(&self) -> Result<T> { |
| T::parse(self) |
| } |
| |
| /// Calls the given parser function to parse a syntax tree node of type `T` |
| /// from this stream. |
| /// |
| /// # Example |
| /// |
| /// The parser below invokes [`Attribute::parse_outer`] to parse a vector of |
| /// zero or more outer attributes. |
| /// |
| /// [`Attribute::parse_outer`]: crate::Attribute::parse_outer |
| /// |
| /// ``` |
| /// use syn::{Attribute, Ident, Result, Token}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// |
| /// // Parses a unit struct with attributes. |
| /// // |
| /// // #[path = "s.tmpl"] |
| /// // struct S; |
| /// struct UnitStruct { |
| /// attrs: Vec<Attribute>, |
| /// struct_token: Token![struct], |
| /// name: Ident, |
| /// semi_token: Token![;], |
| /// } |
| /// |
| /// impl Parse for UnitStruct { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// Ok(UnitStruct { |
| /// attrs: input.call(Attribute::parse_outer)?, |
| /// struct_token: input.parse()?, |
| /// name: input.parse()?, |
| /// semi_token: input.parse()?, |
| /// }) |
| /// } |
| /// } |
| /// ``` |
| pub fn call<T>(&self, function: fn(ParseStream) -> Result<T>) -> Result<T> { |
| function(self) |
| } |
| |
| /// Looks at the next token in the parse stream to determine whether it |
| /// matches the requested type of token. |
| /// |
| /// Does not advance the position of the parse stream. |
| /// |
| /// # Syntax |
| /// |
| /// Note that this method does not use turbofish syntax. Pass the peek type |
| /// inside of parentheses. |
| /// |
| /// - `input.peek(Token![struct])` |
| /// - `input.peek(Token![==])` |
| /// - `input.peek(Ident)` *(does not accept keywords)* |
| /// - `input.peek(Ident::peek_any)` |
| /// - `input.peek(Lifetime)` |
| /// - `input.peek(token::Brace)` |
| /// |
| /// # Example |
| /// |
| /// In this example we finish parsing the list of supertraits when the next |
| /// token in the input is either `where` or an opening curly brace. |
| /// |
| /// ``` |
| /// use syn::{braced, token, Generics, Ident, Result, Token, TypeParamBound}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// use syn::punctuated::Punctuated; |
| /// |
| /// // Parses a trait definition containing no associated items. |
| /// // |
| /// // trait Marker<'de, T>: A + B<'de> where Box<T>: Clone {} |
| /// struct MarkerTrait { |
| /// trait_token: Token![trait], |
| /// ident: Ident, |
| /// generics: Generics, |
| /// colon_token: Option<Token![:]>, |
| /// supertraits: Punctuated<TypeParamBound, Token![+]>, |
| /// brace_token: token::Brace, |
| /// } |
| /// |
| /// impl Parse for MarkerTrait { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// let trait_token: Token![trait] = input.parse()?; |
| /// let ident: Ident = input.parse()?; |
| /// let mut generics: Generics = input.parse()?; |
| /// let colon_token: Option<Token![:]> = input.parse()?; |
| /// |
| /// let mut supertraits = Punctuated::new(); |
| /// if colon_token.is_some() { |
| /// loop { |
| /// supertraits.push_value(input.parse()?); |
| /// if input.peek(Token![where]) || input.peek(token::Brace) { |
| /// break; |
| /// } |
| /// supertraits.push_punct(input.parse()?); |
| /// } |
| /// } |
| /// |
| /// generics.where_clause = input.parse()?; |
| /// let content; |
| /// let empty_brace_token = braced!(content in input); |
| /// |
| /// Ok(MarkerTrait { |
| /// trait_token, |
| /// ident, |
| /// generics, |
| /// colon_token, |
| /// supertraits, |
| /// brace_token: empty_brace_token, |
| /// }) |
| /// } |
| /// } |
| /// ``` |
| pub fn peek<T: Peek>(&self, token: T) -> bool { |
| let _ = token; |
| T::Token::peek(self.cursor()) |
| } |
| |
| /// Looks at the second-next token in the parse stream. |
| /// |
| /// This is commonly useful as a way to implement contextual keywords. |
| /// |
| /// # Example |
| /// |
| /// This example needs to use `peek2` because the symbol `union` is not a |
| /// keyword in Rust. We can't use just `peek` and decide to parse a union if |
| /// the very next token is `union`, because someone is free to write a `mod |
| /// union` and a macro invocation that looks like `union::some_macro! { ... |
| /// }`. In other words `union` is a contextual keyword. |
| /// |
| /// ``` |
| /// use syn::{Ident, ItemUnion, Macro, Result, Token}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// |
| /// // Parses either a union or a macro invocation. |
| /// enum UnionOrMacro { |
| /// // union MaybeUninit<T> { uninit: (), value: T } |
| /// Union(ItemUnion), |
| /// // lazy_static! { ... } |
| /// Macro(Macro), |
| /// } |
| /// |
| /// impl Parse for UnionOrMacro { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// if input.peek(Token![union]) && input.peek2(Ident) { |
| /// input.parse().map(UnionOrMacro::Union) |
| /// } else { |
| /// input.parse().map(UnionOrMacro::Macro) |
| /// } |
| /// } |
| /// } |
| /// ``` |
| pub fn peek2<T: Peek>(&self, token: T) -> bool { |
| fn peek2(buffer: &ParseBuffer, peek: fn(Cursor) -> bool) -> bool { |
| if let Some(group) = buffer.cursor().group(Delimiter::None) { |
| if group.0.skip().map_or(false, peek) { |
| return true; |
| } |
| } |
| buffer.cursor().skip().map_or(false, peek) |
| } |
| |
| let _ = token; |
| peek2(self, T::Token::peek) |
| } |
| |
| /// Looks at the third-next token in the parse stream. |
| pub fn peek3<T: Peek>(&self, token: T) -> bool { |
| fn peek3(buffer: &ParseBuffer, peek: fn(Cursor) -> bool) -> bool { |
| if let Some(group) = buffer.cursor().group(Delimiter::None) { |
| if group.0.skip().and_then(Cursor::skip).map_or(false, peek) { |
| return true; |
| } |
| } |
| buffer |
| .cursor() |
| .skip() |
| .and_then(Cursor::skip) |
| .map_or(false, peek) |
| } |
| |
| let _ = token; |
| peek3(self, T::Token::peek) |
| } |
| |
| /// Parses zero or more occurrences of `T` separated by punctuation of type |
| /// `P`, with optional trailing punctuation. |
| /// |
| /// Parsing continues until the end of this parse stream. The entire content |
| /// of this parse stream must consist of `T` and `P`. |
| /// |
| /// # Example |
| /// |
| /// ``` |
| /// # use quote::quote; |
| /// # |
| /// use syn::{parenthesized, token, Ident, Result, Token, Type}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// use syn::punctuated::Punctuated; |
| /// |
| /// // Parse a simplified tuple struct syntax like: |
| /// // |
| /// // struct S(A, B); |
| /// struct TupleStruct { |
| /// struct_token: Token![struct], |
| /// ident: Ident, |
| /// paren_token: token::Paren, |
| /// fields: Punctuated<Type, Token![,]>, |
| /// semi_token: Token![;], |
| /// } |
| /// |
| /// impl Parse for TupleStruct { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// let content; |
| /// Ok(TupleStruct { |
| /// struct_token: input.parse()?, |
| /// ident: input.parse()?, |
| /// paren_token: parenthesized!(content in input), |
| /// fields: content.parse_terminated(Type::parse)?, |
| /// semi_token: input.parse()?, |
| /// }) |
| /// } |
| /// } |
| /// # |
| /// # let input = quote! { |
| /// # struct S(A, B); |
| /// # }; |
| /// # syn::parse2::<TupleStruct>(input).unwrap(); |
| /// ``` |
| pub fn parse_terminated<T, P: Parse>( |
| &self, |
| parser: fn(ParseStream) -> Result<T>, |
| ) -> Result<Punctuated<T, P>> { |
| Punctuated::parse_terminated_with(self, parser) |
| } |
| |
| /// Returns whether there are tokens remaining in this stream. |
| /// |
| /// This method returns true at the end of the content of a set of |
| /// delimiters, as well as at the very end of the complete macro input. |
| /// |
| /// # Example |
| /// |
| /// ``` |
| /// use syn::{braced, token, Ident, Item, Result, Token}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// |
| /// // Parses a Rust `mod m { ... }` containing zero or more items. |
| /// struct Mod { |
| /// mod_token: Token![mod], |
| /// name: Ident, |
| /// brace_token: token::Brace, |
| /// items: Vec<Item>, |
| /// } |
| /// |
| /// impl Parse for Mod { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// let content; |
| /// Ok(Mod { |
| /// mod_token: input.parse()?, |
| /// name: input.parse()?, |
| /// brace_token: braced!(content in input), |
| /// items: { |
| /// let mut items = Vec::new(); |
| /// while !content.is_empty() { |
| /// items.push(content.parse()?); |
| /// } |
| /// items |
| /// }, |
| /// }) |
| /// } |
| /// } |
| /// ``` |
| pub fn is_empty(&self) -> bool { |
| self.cursor().eof() |
| } |
| |
| /// Constructs a helper for peeking at the next token in this stream and |
| /// building an error message if it is not one of a set of expected tokens. |
| /// |
| /// # Example |
| /// |
| /// ``` |
| /// use syn::{ConstParam, Ident, Lifetime, LifetimeDef, Result, Token, TypeParam}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// |
| /// // A generic parameter, a single one of the comma-separated elements inside |
| /// // angle brackets in: |
| /// // |
| /// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... } |
| /// // |
| /// // On invalid input, lookahead gives us a reasonable error message. |
| /// // |
| /// // error: expected one of: identifier, lifetime, `const` |
| /// // | |
| /// // 5 | fn f<!Sized>() {} |
| /// // | ^ |
| /// enum GenericParam { |
| /// Type(TypeParam), |
| /// Lifetime(LifetimeDef), |
| /// Const(ConstParam), |
| /// } |
| /// |
| /// impl Parse for GenericParam { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// let lookahead = input.lookahead1(); |
| /// if lookahead.peek(Ident) { |
| /// input.parse().map(GenericParam::Type) |
| /// } else if lookahead.peek(Lifetime) { |
| /// input.parse().map(GenericParam::Lifetime) |
| /// } else if lookahead.peek(Token![const]) { |
| /// input.parse().map(GenericParam::Const) |
| /// } else { |
| /// Err(lookahead.error()) |
| /// } |
| /// } |
| /// } |
| /// ``` |
| pub fn lookahead1(&self) -> Lookahead1<'a> { |
| lookahead::new(self.scope, self.cursor()) |
| } |
| |
| /// Forks a parse stream so that parsing tokens out of either the original |
| /// or the fork does not advance the position of the other. |
| /// |
| /// # Performance |
| /// |
| /// Forking a parse stream is a cheap fixed amount of work and does not |
| /// involve copying token buffers. Where you might hit performance problems |
| /// is if your macro ends up parsing a large amount of content more than |
| /// once. |
| /// |
| /// ``` |
| /// # use syn::{Expr, Result}; |
| /// # use syn::parse::ParseStream; |
| /// # |
| /// # fn bad(input: ParseStream) -> Result<Expr> { |
| /// // Do not do this. |
| /// if input.fork().parse::<Expr>().is_ok() { |
| /// return input.parse::<Expr>(); |
| /// } |
| /// # unimplemented!() |
| /// # } |
| /// ``` |
| /// |
| /// As a rule, avoid parsing an unbounded amount of tokens out of a forked |
| /// parse stream. Only use a fork when the amount of work performed against |
| /// the fork is small and bounded. |
| /// |
| /// When complex speculative parsing against the forked stream is |
| /// unavoidable, use [`parse::discouraged::Speculative`] to advance the |
| /// original stream once the fork's parse is determined to have been |
| /// successful. |
| /// |
| /// For a lower level way to perform speculative parsing at the token level, |
| /// consider using [`ParseStream::step`] instead. |
| /// |
| /// [`parse::discouraged::Speculative`]: discouraged::Speculative |
| /// [`ParseStream::step`]: ParseBuffer::step |
| /// |
| /// # Example |
| /// |
| /// The parse implementation shown here parses possibly restricted `pub` |
| /// visibilities. |
| /// |
| /// - `pub` |
| /// - `pub(crate)` |
| /// - `pub(self)` |
| /// - `pub(super)` |
| /// - `pub(in some::path)` |
| /// |
| /// To handle the case of visibilities inside of tuple structs, the parser |
| /// needs to distinguish parentheses that specify visibility restrictions |
| /// from parentheses that form part of a tuple type. |
| /// |
| /// ``` |
| /// # struct A; |
| /// # struct B; |
| /// # struct C; |
| /// # |
| /// struct S(pub(crate) A, pub (B, C)); |
| /// ``` |
| /// |
| /// In this example input the first tuple struct element of `S` has |
| /// `pub(crate)` visibility while the second tuple struct element has `pub` |
| /// visibility; the parentheses around `(B, C)` are part of the type rather |
| /// than part of a visibility restriction. |
| /// |
| /// The parser uses a forked parse stream to check the first token inside of |
| /// parentheses after the `pub` keyword. This is a small bounded amount of |
| /// work performed against the forked parse stream. |
| /// |
| /// ``` |
| /// use syn::{parenthesized, token, Ident, Path, Result, Token}; |
| /// use syn::ext::IdentExt; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// |
| /// struct PubVisibility { |
| /// pub_token: Token![pub], |
| /// restricted: Option<Restricted>, |
| /// } |
| /// |
| /// struct Restricted { |
| /// paren_token: token::Paren, |
| /// in_token: Option<Token![in]>, |
| /// path: Path, |
| /// } |
| /// |
| /// impl Parse for PubVisibility { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// let pub_token: Token![pub] = input.parse()?; |
| /// |
| /// if input.peek(token::Paren) { |
| /// let ahead = input.fork(); |
| /// let mut content; |
| /// parenthesized!(content in ahead); |
| /// |
| /// if content.peek(Token![crate]) |
| /// || content.peek(Token![self]) |
| /// || content.peek(Token![super]) |
| /// { |
| /// return Ok(PubVisibility { |
| /// pub_token, |
| /// restricted: Some(Restricted { |
| /// paren_token: parenthesized!(content in input), |
| /// in_token: None, |
| /// path: Path::from(content.call(Ident::parse_any)?), |
| /// }), |
| /// }); |
| /// } else if content.peek(Token![in]) { |
| /// return Ok(PubVisibility { |
| /// pub_token, |
| /// restricted: Some(Restricted { |
| /// paren_token: parenthesized!(content in input), |
| /// in_token: Some(content.parse()?), |
| /// path: content.call(Path::parse_mod_style)?, |
| /// }), |
| /// }); |
| /// } |
| /// } |
| /// |
| /// Ok(PubVisibility { |
| /// pub_token, |
| /// restricted: None, |
| /// }) |
| /// } |
| /// } |
| /// ``` |
| pub fn fork(&self) -> Self { |
| ParseBuffer { |
| scope: self.scope, |
| cell: self.cell.clone(), |
| marker: PhantomData, |
| // Not the parent's unexpected. Nothing cares whether the clone |
| // parses all the way unless we `advance_to`. |
| unexpected: Cell::new(Some(Rc::new(Cell::new(Unexpected::None)))), |
| } |
| } |
| |
| /// Triggers an error at the current position of the parse stream. |
| /// |
| /// # Example |
| /// |
| /// ``` |
| /// use syn::{Expr, Result, Token}; |
| /// use syn::parse::{Parse, ParseStream}; |
| /// |
| /// // Some kind of loop: `while` or `for` or `loop`. |
| /// struct Loop { |
| /// expr: Expr, |
| /// } |
| /// |
| /// impl Parse for Loop { |
| /// fn parse(input: ParseStream) -> Result<Self> { |
| /// if input.peek(Token![while]) |
| /// || input.peek(Token![for]) |
| /// || input.peek(Token![loop]) |
| /// { |
| /// Ok(Loop { |
| /// expr: input.parse()?, |
| /// }) |
| /// } else { |
| /// Err(input.error("expected some kind of loop")) |
| /// } |
| /// } |
| /// } |
| /// ``` |
| pub fn error<T: Display>(&self, message: T) -> Error { |
| error::new_at(self.scope, self.cursor(), message) |
| } |
| |
| /// Speculatively parses tokens from this parse stream, advancing the |
| /// position of this stream only if parsing succeeds. |
| /// |
| /// This is a powerful low-level API used for defining the `Parse` impls of |
| /// the basic built-in token types. It is not something that will be used |
| /// widely outside of the Syn codebase. |
| /// |
| /// # Example |
| /// |
| /// ``` |
| /// use proc_macro2::TokenTree; |
| /// use syn::Result; |
| /// use syn::parse::ParseStream; |
| /// |
| /// // This function advances the stream past the next occurrence of `@`. If |
| /// // no `@` is present in the stream, the stream position is unchanged and |
| /// // an error is returned. |
| /// fn skip_past_next_at(input: ParseStream) -> Result<()> { |
| /// input.step(|cursor| { |
| /// let mut rest = *cursor; |
| /// while let Some((tt, next)) = rest.token_tree() { |
| /// match &tt { |
| /// TokenTree::Punct(punct) if punct.as_char() == '@' => { |
| /// return Ok(((), next)); |
| /// } |
| /// _ => rest = next, |
| /// } |
| /// } |
| /// Err(cursor.error("no `@` was found after this point")) |
| /// }) |
| /// } |
| /// # |
| /// # fn remainder_after_skipping_past_next_at( |
| /// # input: ParseStream, |
| /// # ) -> Result<proc_macro2::TokenStream> { |
| /// # skip_past_next_at(input)?; |
| /// # input.parse() |
| /// # } |
| /// # |
| /// # use syn::parse::Parser; |
| /// # let remainder = remainder_after_skipping_past_next_at |
| /// # .parse_str("a @ b c") |
| /// # .unwrap(); |
| /// # assert_eq!(remainder.to_string(), "b c"); |
| /// ``` |
| pub fn step<F, R>(&self, function: F) -> Result<R> |
| where |
| F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>, |
| { |
| // Since the user's function is required to work for any 'c, we know |
| // that the Cursor<'c> they return is either derived from the input |
| // StepCursor<'c, 'a> or from a Cursor<'static>. |
| // |
| // It would not be legal to write this function without the invariant |
| // lifetime 'c in StepCursor<'c, 'a>. If this function were written only |
| // in terms of 'a, the user could take our ParseBuffer<'a>, upcast it to |
| // a ParseBuffer<'short> which some shorter lifetime than 'a, invoke |
| // `step` on their ParseBuffer<'short> with a closure that returns |
| // Cursor<'short>, and we would wrongly write that Cursor<'short> into |
| // the Cell intended to hold Cursor<'a>. |
| // |
| // In some cases it may be necessary for R to contain a Cursor<'a>. |
| // Within Syn we solve this using `advance_step_cursor` which uses the |
| // existence of a StepCursor<'c, 'a> as proof that it is safe to cast |
| // from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it would be |
| // safe to expose that API as a method on StepCursor. |
| let (node, rest) = function(StepCursor { |
| scope: self.scope, |
| cursor: self.cell.get(), |
| marker: PhantomData, |
| })?; |
| self.cell.set(rest); |
| Ok(node) |
| } |
| |
| /// Returns the `Span` of the next token in the parse stream, or |
| /// `Span::call_site()` if this parse stream has completely exhausted its |
| /// input `TokenStream`. |
| pub fn span(&self) -> Span { |
| let cursor = self.cursor(); |
| if cursor.eof() { |
| self.scope |
| } else { |
| crate::buffer::open_span_of_group(cursor) |
| } |
| } |
| |
| /// Provides low-level access to the token representation underlying this |
| /// parse stream. |
| /// |
| /// Cursors are immutable so no operations you perform against the cursor |
| /// will affect the state of this parse stream. |
| pub fn cursor(&self) -> Cursor<'a> { |
| self.cell.get() |
| } |
| |
| fn check_unexpected(&self) -> Result<()> { |
| match inner_unexpected(self).1 { |
| Some(span) => Err(Error::new(span, "unexpected token")), |
| None => Ok(()), |
| } |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl<T: Parse> Parse for Box<T> { |
| fn parse(input: ParseStream) -> Result<Self> { |
| input.parse().map(Box::new) |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl<T: Parse + Token> Parse for Option<T> { |
| fn parse(input: ParseStream) -> Result<Self> { |
| if T::peek(input.cursor()) { |
| Ok(Some(input.parse()?)) |
| } else { |
| Ok(None) |
| } |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl Parse for TokenStream { |
| fn parse(input: ParseStream) -> Result<Self> { |
| input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty()))) |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl Parse for TokenTree { |
| fn parse(input: ParseStream) -> Result<Self> { |
| input.step(|cursor| match cursor.token_tree() { |
| Some((tt, rest)) => Ok((tt, rest)), |
| None => Err(cursor.error("expected token tree")), |
| }) |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl Parse for Group { |
| fn parse(input: ParseStream) -> Result<Self> { |
| input.step(|cursor| { |
| for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] { |
| if let Some((inside, span, rest)) = cursor.group(*delim) { |
| let mut group = Group::new(*delim, inside.token_stream()); |
| group.set_span(span); |
| return Ok((group, rest)); |
| } |
| } |
| Err(cursor.error("expected group token")) |
| }) |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl Parse for Punct { |
| fn parse(input: ParseStream) -> Result<Self> { |
| input.step(|cursor| match cursor.punct() { |
| Some((punct, rest)) => Ok((punct, rest)), |
| None => Err(cursor.error("expected punctuation token")), |
| }) |
| } |
| } |
| |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| impl Parse for Literal { |
| fn parse(input: ParseStream) -> Result<Self> { |
| input.step(|cursor| match cursor.literal() { |
| Some((literal, rest)) => Ok((literal, rest)), |
| None => Err(cursor.error("expected literal token")), |
| }) |
| } |
| } |
| |
| /// Parser that can parse Rust tokens into a particular syntax tree node. |
| /// |
| /// Refer to the [module documentation] for details about parsing in Syn. |
| /// |
| /// [module documentation]: self |
| /// |
| /// *This trait is available only if Syn is built with the `"parsing"` feature.* |
| pub trait Parser: Sized { |
| type Output; |
| |
| /// Parse a proc-macro2 token stream into the chosen syntax tree node. |
| /// |
| /// This function will check that the input is fully parsed. If there are |
| /// any unparsed tokens at the end of the stream, an error is returned. |
| fn parse2(self, tokens: TokenStream) -> Result<Self::Output>; |
| |
| /// Parse tokens of source code into the chosen syntax tree node. |
| /// |
| /// This function will check that the input is fully parsed. If there are |
| /// any unparsed tokens at the end of the stream, an error is returned. |
| /// |
| /// *This method is available only if Syn is built with both the `"parsing"` and |
| /// `"proc-macro"` features.* |
| #[cfg(all( |
| not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), |
| feature = "proc-macro" |
| ))] |
| fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> { |
| self.parse2(proc_macro2::TokenStream::from(tokens)) |
| } |
| |
| /// Parse a string of Rust code into the chosen syntax tree node. |
| /// |
| /// This function will check that the input is fully parsed. If there are |
| /// any unparsed tokens at the end of the string, an error is returned. |
| /// |
| /// # Hygiene |
| /// |
| /// Every span in the resulting syntax tree will be set to resolve at the |
| /// macro call site. |
| fn parse_str(self, s: &str) -> Result<Self::Output> { |
| self.parse2(proc_macro2::TokenStream::from_str(s)?) |
| } |
| |
| // Not public API. |
| #[doc(hidden)] |
| #[cfg(any(feature = "full", feature = "derive"))] |
| fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> { |
| let _ = scope; |
| self.parse2(tokens) |
| } |
| |
| // Not public API. |
| #[doc(hidden)] |
| #[cfg(any(feature = "full", feature = "derive"))] |
| fn __parse_stream(self, input: ParseStream) -> Result<Self::Output> { |
| input.parse().and_then(|tokens| self.parse2(tokens)) |
| } |
| } |
| |
| fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer { |
| let scope = Span::call_site(); |
| let cursor = tokens.begin(); |
| let unexpected = Rc::new(Cell::new(Unexpected::None)); |
| new_parse_buffer(scope, cursor, unexpected) |
| } |
| |
| impl<F, T> Parser for F |
| where |
| F: FnOnce(ParseStream) -> Result<T>, |
| { |
| type Output = T; |
| |
| fn parse2(self, tokens: TokenStream) -> Result<T> { |
| let buf = TokenBuffer::new2(tokens); |
| let state = tokens_to_parse_buffer(&buf); |
| let node = self(&state)?; |
| state.check_unexpected()?; |
| if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(state.cursor()) { |
| Err(Error::new(unexpected_span, "unexpected token")) |
| } else { |
| Ok(node) |
| } |
| } |
| |
| #[doc(hidden)] |
| #[cfg(any(feature = "full", feature = "derive"))] |
| fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> { |
| let buf = TokenBuffer::new2(tokens); |
| let cursor = buf.begin(); |
| let unexpected = Rc::new(Cell::new(Unexpected::None)); |
| let state = new_parse_buffer(scope, cursor, unexpected); |
| let node = self(&state)?; |
| state.check_unexpected()?; |
| if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(state.cursor()) { |
| Err(Error::new(unexpected_span, "unexpected token")) |
| } else { |
| Ok(node) |
| } |
| } |
| |
| #[doc(hidden)] |
| #[cfg(any(feature = "full", feature = "derive"))] |
| fn __parse_stream(self, input: ParseStream) -> Result<Self::Output> { |
| self(input) |
| } |
| } |
| |
| #[cfg(any(feature = "full", feature = "derive"))] |
| pub(crate) fn parse_scoped<F: Parser>(f: F, scope: Span, tokens: TokenStream) -> Result<F::Output> { |
| f.__parse_scoped(scope, tokens) |
| } |
| |
| #[cfg(any(feature = "full", feature = "derive"))] |
| pub(crate) fn parse_stream<F: Parser>(f: F, input: ParseStream) -> Result<F::Output> { |
| f.__parse_stream(input) |
| } |
| |
| /// An empty syntax tree node that consumes no tokens when parsed. |
| /// |
| /// This is useful for attribute macros that want to ensure they are not |
| /// provided any attribute args. |
| /// |
| /// ``` |
| /// # extern crate proc_macro; |
| /// # |
| /// use proc_macro::TokenStream; |
| /// use syn::parse_macro_input; |
| /// use syn::parse::Nothing; |
| /// |
| /// # const IGNORE: &str = stringify! { |
| /// #[proc_macro_attribute] |
| /// # }; |
| /// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { |
| /// parse_macro_input!(args as Nothing); |
| /// |
| /// /* ... */ |
| /// # "".parse().unwrap() |
| /// } |
| /// ``` |
| /// |
| /// ```text |
| /// error: unexpected token |
| /// --> src/main.rs:3:19 |
| /// | |
| /// 3 | #[my_attr(asdf)] |
| /// | ^^^^ |
| /// ``` |
| pub struct Nothing; |
| |
| impl Parse for Nothing { |
| fn parse(_input: ParseStream) -> Result<Self> { |
| Ok(Nothing) |
| } |
| } |
| |
| #[cfg(feature = "extra-traits")] |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| impl Debug for Nothing { |
| fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| f.write_str("Nothing") |
| } |
| } |
| |
| #[cfg(feature = "extra-traits")] |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| impl Eq for Nothing {} |
| |
| #[cfg(feature = "extra-traits")] |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| impl PartialEq for Nothing { |
| fn eq(&self, _other: &Self) -> bool { |
| true |
| } |
| } |
| |
| #[cfg(feature = "extra-traits")] |
| #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| impl Hash for Nothing { |
| fn hash<H: Hasher>(&self, _state: &mut H) {} |
| } |
