boost/spirit/home/lex/tokenize_and_parse.hpp
// Copyright (c) 2001-2011 Hartmut Kaiser // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #if !defined(BOOST_SPIRIT_LEXER_PARSE_NOV_17_2007_0246PM) #define BOOST_SPIRIT_LEXER_PARSE_NOV_17_2007_0246PM #if defined(_MSC_VER) #pragma once #endif #include <boost/spirit/home/qi/skip_over.hpp> #include <boost/spirit/home/qi/parse.hpp> #include <boost/spirit/home/qi/nonterminal/grammar.hpp> #include <boost/spirit/home/support/unused.hpp> #include <boost/spirit/home/lex/lexer.hpp> #include <boost/mpl/assert.hpp> namespace boost { namespace spirit { namespace lex { /////////////////////////////////////////////////////////////////////////// // Import skip_flag enumerator type from Qi namespace using qi::skip_flag; /////////////////////////////////////////////////////////////////////////// // // The tokenize_and_parse() function is one of the main Spirit API // functions. It simplifies using a lexer as the underlying token source // while parsing a given input sequence. // // The function takes a pair of iterators spanning the underlying input // stream to parse, the lexer object (built from the token definitions) // and a parser object (built from the parser grammar definition). // // The second version of this function additionally takes an attribute to // be used as the top level data structure instance the parser should use // to store the recognized input to. // // The function returns true if the parsing succeeded (the given input // sequence has been successfully matched by the given grammar). // // first, last: The pair of iterators spanning the underlying input // sequence to parse. These iterators must at least // conform to the requirements of the std::intput_iterator // category. // On exit the iterator 'first' will be updated to the // position right after the last successfully matched // token. // lex: The lexer object (encoding the token definitions) to be // used to convert the input sequence into a sequence of // tokens. This token sequence is passed to the parsing // process. The LexerExpr type must conform to the // lexer interface described in the corresponding section // of the documentation. // xpr: The grammar object (encoding the parser grammar) to be // used to match the token sequence generated by the lex // object instance. The ParserExpr type must conform to // the grammar interface described in the corresponding // section of the documentation. // attr: The top level attribute passed to the parser. It will // be populated during the parsing of the input sequence. // On exit it will hold the 'parser result' corresponding // to the matched input sequence. // /////////////////////////////////////////////////////////////////////////// template <typename Iterator, typename Lexer, typename ParserExpr> inline bool tokenize_and_parse(Iterator& first, Iterator last, Lexer const& lex, ParserExpr const& xpr) { // Report invalid expression error as early as possible. // If you got an error_invalid_expression error message here, // then the expression (expr) is not a valid spirit qi expression. BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); typename Lexer::iterator_type iter = lex.begin(first, last); return compile<qi::domain>(xpr).parse( iter, lex.end(), unused, unused, unused); } /////////////////////////////////////////////////////////////////////////// template <typename Iterator, typename Lexer, typename ParserExpr , typename Attribute> inline bool tokenize_and_parse(Iterator& first, Iterator last, Lexer const& lex , ParserExpr const& xpr, Attribute& attr) { // Report invalid expression error as early as possible. // If you got an error_invalid_expression error message here, // then the expression (expr) is not a valid spirit qi expression. BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); typename Lexer::iterator_type iter = lex.begin(first, last); return compile<qi::domain>(xpr).parse( iter, lex.end(), unused, unused, attr); } /////////////////////////////////////////////////////////////////////////// // // The tokenize_and_phrase_parse() function is one of the main Spirit API // functions. It simplifies using a lexer as the underlying token source // while phrase parsing a given input sequence. // // The function takes a pair of iterators spanning the underlying input // stream to parse, the lexer object (built from the token definitions) // and a parser object (built from the parser grammar definition). The // additional skipper parameter will be used as the skip parser during // the parsing process. // // The second version of this function additionally takes an attribute to // be used as the top level data structure instance the parser should use // to store the recognized input to. // // The function returns true if the parsing succeeded (the given input // sequence has been successfully matched by the given grammar). // // first, last: The pair of iterators spanning the underlying input // sequence to parse. These iterators must at least // conform to the requirements of the std::intput_iterator // category. // On exit the iterator 'first' will be updated to the // position right after the last successfully matched // token. // lex: The lexer object (encoding the token definitions) to be // used to convert the input sequence into a sequence of // tokens. This token sequence is passed to the parsing // process. The LexerExpr type must conform to the // lexer interface described in the corresponding section // of the documentation. // xpr: The grammar object (encoding the parser grammar) to be // used to match the token sequence generated by the lex // object instance. The ParserExpr type must conform to // the grammar interface described in the corresponding // section of the documentation. // skipper: The skip parser to be used while parsing the given // input sequence. Note, the skip parser will have to // act on the same token sequence as the main parser // 'xpr'. // post_skip: The post_skip flag controls whether the function will // invoke an additional post skip after the main parser // returned. // attr: The top level attribute passed to the parser. It will // be populated during the parsing of the input sequence. // On exit it will hold the 'parser result' corresponding // to the matched input sequence. // /////////////////////////////////////////////////////////////////////////// template <typename Iterator, typename Lexer, typename ParserExpr , typename Skipper> inline bool tokenize_and_phrase_parse(Iterator& first, Iterator last , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper , BOOST_SCOPED_ENUM(skip_flag) post_skip = skip_flag::postskip) { // Report invalid expression error as early as possible. // If you got an error_invalid_expression error message here, // then the expression (expr) is not a valid spirit qi expression. BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); BOOST_SPIRIT_ASSERT_MATCH(qi::domain, Skipper); typedef typename spirit::result_of::compile<qi::domain, Skipper>::type skipper_type; skipper_type const skipper_ = compile<qi::domain>(skipper); typename Lexer::iterator_type iter = lex.begin(first, last); typename Lexer::iterator_type end = lex.end(); if (!compile<qi::domain>(xpr).parse( iter, end, unused, skipper_, unused)) return false; // do a final post-skip if (post_skip == skip_flag::postskip) qi::skip_over(iter, end, skipper_); return true; } template <typename Iterator, typename Lexer, typename ParserExpr , typename Skipper, typename Attribute> inline bool tokenize_and_phrase_parse(Iterator& first, Iterator last , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper , BOOST_SCOPED_ENUM(skip_flag) post_skip, Attribute& attr) { // Report invalid expression error as early as possible. // If you got an error_invalid_expression error message here, // then the expression (expr) is not a valid spirit qi expression. BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); BOOST_SPIRIT_ASSERT_MATCH(qi::domain, Skipper); typedef typename spirit::result_of::compile<qi::domain, Skipper>::type skipper_type; skipper_type const skipper_ = compile<qi::domain>(skipper); typename Lexer::iterator_type iter = lex.begin(first, last); typename Lexer::iterator_type end = lex.end(); if (!compile<qi::domain>(xpr).parse( iter, end, unused, skipper_, attr)) return false; // do a final post-skip if (post_skip == skip_flag::postskip) qi::skip_over(iter, end, skipper_); return true; } /////////////////////////////////////////////////////////////////////////// template <typename Iterator, typename Lexer, typename ParserExpr , typename Skipper, typename Attribute> inline bool tokenize_and_phrase_parse(Iterator& first, Iterator last , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper , Attribute& attr) { return tokenize_and_phrase_parse(first, last, lex, xpr, skipper , skip_flag::postskip, attr); } /////////////////////////////////////////////////////////////////////////// // // The tokenize() function is one of the main Spirit API functions. It // simplifies using a lexer to tokenize a given input sequence. It's main // purpose is to use the lexer to tokenize all the input. // // The second version below discards all generated tokens afterwards. // This is useful whenever all the needed functionality has been // implemented directly inside the lexer semantic actions, which are being // executed while the tokens are matched. // // The function takes a pair of iterators spanning the underlying input // stream to scan, the lexer object (built from the token definitions), // and a (optional) functor being called for each of the generated tokens. // // The function returns true if the scanning of the input succeeded (the // given input sequence has been successfully matched by the given token // definitions). // // first, last: The pair of iterators spanning the underlying input // sequence to parse. These iterators must at least // conform to the requirements of the std::intput_iterator // category. // On exit the iterator 'first' will be updated to the // position right after the last successfully matched // token. // lex: The lexer object (encoding the token definitions) to be // used to convert the input sequence into a sequence of // tokens. The LexerExpr type must conform to the // lexer interface described in the corresponding section // of the documentation. // f: A functor (callable object) taking a single argument of // the token type and returning a bool, indicating whether // the tokenization should be canceled. // initial_state: The name of the state the lexer should start matching. // The default value is zero, causing the lexer to start // in its 'INITIAL' state. // /////////////////////////////////////////////////////////////////////////// namespace detail { template <typename Token, typename F> bool tokenize_callback(Token const& t, F f) { return f(t); } template <typename Token, typename Eval> bool tokenize_callback(Token const& t, phoenix::actor<Eval> const& f) { f(t); return true; } template <typename Token> bool tokenize_callback(Token const& t, void (*f)(Token const&)) { f(t); return true; } template <typename Token> bool tokenize_callback(Token const& t, bool (*f)(Token const&)) { return f(t); } } template <typename Iterator, typename Lexer, typename F> inline bool tokenize(Iterator& first, Iterator last, Lexer const& lex, F f , typename Lexer::char_type const* initial_state = 0) { typedef typename Lexer::iterator_type iterator_type; iterator_type iter = lex.begin(first, last, initial_state); iterator_type end = lex.end(); for (/**/; iter != end && token_is_valid(*iter); ++iter) { if (!detail::tokenize_callback(*iter, f)) return false; } return (iter == end) ? true : false; } /////////////////////////////////////////////////////////////////////////// template <typename Iterator, typename Lexer> inline bool tokenize(Iterator& first, Iterator last, Lexer const& lex , typename Lexer::char_type const* initial_state = 0) { typedef typename Lexer::iterator_type iterator_type; iterator_type iter = lex.begin(first, last, initial_state); iterator_type end = lex.end(); while (iter != end && token_is_valid(*iter)) ++iter; return (iter == end) ? true : false; } }}} #endif