Question

让我们考虑以下代码：

#include <boost/phoenix.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <algorithm>
#include <iostream>
#include <string>
#include <utility>
#include <vector>

namespace lex = boost::spirit::lex;
namespace qi = boost::spirit::qi;
namespace phoenix = boost::phoenix;

struct operation
{
    enum type
    {
        add,
        sub,
        mul,
        div
    };
};

template<typename Lexer>
class expression_lexer
    : public lex::lexer<Lexer>
{
public:
    typedef lex::token_def<operation::type> operator_token_type;
    typedef lex::token_def<double> value_token_type;
    typedef lex::token_def<std::string> variable_token_type;
    typedef lex::token_def<lex::omit> parenthesis_token_type;
    typedef std::pair<parenthesis_token_type, parenthesis_token_type> parenthesis_token_pair_type;
    typedef lex::token_def<lex::omit> whitespace_token_type;

    expression_lexer()
        : operator_add('+'),
          operator_sub('-'),
          operator_mul("[x*]"),
          operator_div("[:/]"),
          value("\\d+(\\.\\d+)?"),
          variable("%(\\w+)"),
          parenthesis({
            std::make_pair(parenthesis_token_type('('), parenthesis_token_type(')')),
            std::make_pair(parenthesis_token_type('['), parenthesis_token_type(']'))
          }),
          whitespace("[ \\t]+")
    {
        this->self
            += operator_add [lex::_val = operation::add]
            | operator_sub [lex::_val = operation::sub]
            | operator_mul [lex::_val = operation::mul]
            | operator_div [lex::_val = operation::div]
            | value
            | variable [lex::_val = phoenix::construct<std::string>(lex::_start + 1, lex::_end)]
            | whitespace [lex::_pass = lex::pass_flags::pass_ignore]
            ;

        std::for_each(parenthesis.cbegin(), parenthesis.cend(),
            [&](parenthesis_token_pair_type const& token_pair)
            {
                this->self += token_pair.first | token_pair.second;
            }
        );
    }

    operator_token_type operator_add;
    operator_token_type operator_sub;
    operator_token_type operator_mul;
    operator_token_type operator_div;

    value_token_type value;
    variable_token_type variable;

    std::vector<parenthesis_token_pair_type> parenthesis;

    whitespace_token_type whitespace;
};

template<typename Iterator>
class expression_grammar
    : public qi::grammar<Iterator>
{
public:
    template<typename Tokens>
    explicit expression_grammar(Tokens const& tokens)
        : expression_grammar::base_type(start)
    {
        start                     %= expression >> qi::eoi;

        expression                %= sum_operand >> -(sum_operator >> expression);
        sum_operator              %= tokens.operator_add | tokens.operator_sub;
        sum_operand               %= fac_operand >> -(fac_operator >> sum_operand);
        fac_operator              %= tokens.operator_mul | tokens.operator_div;

        if(!tokens.parenthesis.empty())
            fac_operand           %= parenthesised | terminal;
        else
            fac_operand           %= terminal;

        terminal                  %= tokens.value | tokens.variable;

        if(!tokens.parenthesis.empty())
        {
            parenthesised         %= tokens.parenthesis.front().first >> expression >> tokens.parenthesis.front().second;
            std::for_each(tokens.parenthesis.cbegin() + 1, tokens.parenthesis.cend(),
                [&](typename Tokens::parenthesis_token_pair_type const& token_pair)
                {
                    parenthesised %= parenthesised.copy() | (token_pair.first >> expression >> token_pair.second);
                }
            );
        }
    }

private:
    qi::rule<Iterator> start;
    qi::rule<Iterator> expression;
    qi::rule<Iterator> sum_operand;
    qi::rule<Iterator> sum_operator;
    qi::rule<Iterator> fac_operand;
    qi::rule<Iterator> fac_operator;
    qi::rule<Iterator> terminal;
    qi::rule<Iterator> parenthesised;
};


int main()
{
    typedef lex::lexertl::token<std::string::const_iterator, boost::mpl::vector<operation::type, double, std::string>> token_type;
    typedef expression_lexer<lex::lexertl::actor_lexer<token_type>> expression_lexer_type;
    typedef expression_lexer_type::iterator_type expression_lexer_iterator_type;
    typedef expression_grammar<expression_lexer_iterator_type> expression_grammar_type;

    expression_lexer_type lexer;
    expression_grammar_type grammar(lexer);

    while(std::cin)
    {
        std::string line;
        std::getline(std::cin, line);

        std::string::const_iterator first = line.begin();
        std::string::const_iterator const last = line.end();

        bool const result = lex::tokenize_and_parse(first, last, lexer, grammar);
        if(!result)
            std::cout << "Parsing failed! Reminder: >" << std::string(first, last) << "<" << std::endl;
        else
        {
            if(first != last)
                std::cout << "Parsing succeeded! Reminder: >" << std::string(first, last) << "<" << std::endl;
            else
                std::cout << "Parsing succeeded!" << std::endl;
        }
    }
}

它是一个带有值和变量的算术表达式的简单解析器。它是使用expression_lexer构建来提取令牌，然后使用expression_grammar来解析令牌。

对于如此小的案例使用词法分析器似乎有点矫枉过正，可能就是一个。但这是简化示例的成本。另请注意，使用词法分析器可以轻松定义具有正则表达式的标记，同时允许通过外部代码（特别是用户提供的配置）轻松定义它们。通过提供的示例，从外部配置文件中读取令牌的定义并且例如允许用户将变量从%name更改为$name完全没有问题。

代码似乎工作正常（在Visual Studio 2013上使用Boost 1.61进行检查）。

expression_lexer具有附加到令牌的属性。我猜他们编译后就可以了。但我真的不知道如何检查。

最终，我希望语法能够为我构建一个std::vector，并使用反向抛光表达式。（其中每个元素都是[{1}}或boost::variant或operator::type上的double。）

然而问题是我在std::string中未能使用令牌属性。例如，如果您尝试按以下方式更改expression_grammar：

sum_operator

您将收到编译错误。我希望这可行，因为qi::rule<Iterator, operation::type ()> sum_operator;是operation::type和operator_add的属性，因此也可以替代它们。它仍然没有编译。从operator_sub中的错误判断，似乎解析器尝试直接从输入流范围构建属性值。这意味着它忽略了我在词法分析器中指定的assign_to_attribute_from_iterators。

将其更改为

[lex::_val = operation::add]

也没有帮助。

我还尝试将定义更改为

qi::rule<Iterator, operation::type (operation::type)> sum_operator;

也没有帮助。

如何解决这个问题？我知道我可以使用Qi的sum_operator %= (tokens.operator_add | tokens.operator_sub) [qi::_val = qi::_1];。但是我希望使用词法分析器来轻松配置令牌的正则表达式。我也可以扩展symbols，如文档中所述，但这种工作加倍。我想我也可以跳过词法分析器上的属性，并将它们放在语法上。但是这对于assign_to_attribute_from_iterators令牌的灵活性来说再也不能很好地工作（在我的实际情况中，那里有更多的逻辑，因此它也可以配置令牌的哪一部分形成变量的实际名称 - 而在这里它被修复为只跳过第一个字符）。还有什么吗？

也是一个问题 - 也许任何人都知道。有没有办法从令牌动作捕获令牌正则表达式的组？所以，而不是

variable

相反，我可以从捕获组中创建一个字符串，因此可以轻松处理variable [lex::_val = phoenix::construct<std::string>(lex::_start + 1, lex::_end)]等格式。

编辑！我根据Whitespace skipper when using Boost.Spirit Qi and Lex得出的结论改进了空格。这是一种简化，不会影响这里提出的问题。

Answer 1

好的，这是我对RPN＆＃39;要求的看法。我非常赞成自然（自动）属性传播优于语义动作（参见Boost Spirit: "Semantic actions are evil"?）

我考虑其他选项（uglifying）优化。如果您对整体设计感到满意并且不介意让它更难维护，您可以这样做：）

<强> Live On Coliru

除了我已经研究过的评论样本之外，我还添加了RPN转换步骤：

namespace RPN {
    using cell      = boost::variant<AST::operation, AST::value, AST::variable>;
    using rpn_stack = std::vector<cell>;

    struct transform : boost::static_visitor<> {
        void operator()(rpn_stack& stack, AST::expression const& e) const {
            boost::apply_visitor(boost::bind(*this, boost::ref(stack), ::_1), e);
        }
        void operator()(rpn_stack& stack, AST::bin_expr const& e) const {
            (*this)(stack, e.lhs);
            (*this)(stack, e.rhs);
            stack.push_back(e.op);
        }
        void operator()(rpn_stack& stack, AST::value    const& v) const { stack.push_back(v); }
        void operator()(rpn_stack& stack, AST::variable const& v) const { stack.push_back(v); }
    };
}

这就是全部！像这样使用它，例如：

RPN::transform compiler;
RPN::rpn_stack program;
compiler(program, expr);

for (auto& instr : program) {
    std::cout << instr << " ";
}

哪个输出：

Parsing success: (3 + (8 * 9))
3 8 9 * +

完整列表

<强> Live On Coliru

//#define BOOST_SPIRIT_DEBUG
#include <boost/phoenix.hpp>
#include <boost/bind.hpp>
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <algorithm>
#include <iostream>
#include <string>
#include <utility>
#include <vector>

namespace lex     = boost::spirit::lex;
namespace qi      = boost::spirit::qi;
namespace phoenix = boost::phoenix;

struct operation
{
    enum type
    {
        add,
        sub,
        mul,
        div
    };

    friend std::ostream& operator<<(std::ostream& os, type op) {
        switch (op) {
            case type::add: return os << "+";
            case type::sub: return os << "-";
            case type::mul: return os << "*";
            case type::div: return os << "/";
        }
        return os << "<" << static_cast<int>(op) << ">";
    }
};

template<typename Lexer>
class expression_lexer
    : public lex::lexer<Lexer>
{
public:
    //typedef lex::token_def<operation::type> operator_token_type;
    typedef lex::token_def<lex::omit> operator_token_type;
    typedef lex::token_def<double> value_token_type;
    typedef lex::token_def<std::string> variable_token_type;

    typedef lex::token_def<lex::omit> parenthesis_token_type;
    typedef std::pair<parenthesis_token_type, parenthesis_token_type> parenthesis_token_pair_type;
    typedef lex::token_def<lex::omit> whitespace_token_type;

    expression_lexer()
        : operator_add('+'),
          operator_sub('-'),
          operator_mul("[x*]"),
          operator_div("[:/]"),
          value("\\d+(\\.\\d+)?"),
          variable("%(\\w+)"),
          parenthesis({
            std::make_pair(parenthesis_token_type('('), parenthesis_token_type(')')),
            std::make_pair(parenthesis_token_type('['), parenthesis_token_type(']'))
          }),
          whitespace("[ \\t]+")
    {
        this->self
            += operator_add [lex::_val = operation::add]
             | operator_sub [lex::_val = operation::sub]
             | operator_mul [lex::_val = operation::mul]
             | operator_div [lex::_val = operation::div]
             | value
             | variable [lex::_val = phoenix::construct<std::string>(lex::_start + 1, lex::_end)]
             | whitespace [lex::_pass = lex::pass_flags::pass_ignore]
             ;

        std::for_each(parenthesis.cbegin(), parenthesis.cend(),
            [&](parenthesis_token_pair_type const& token_pair)
            {
                this->self += token_pair.first | token_pair.second;
            }
        );
    }

    operator_token_type operator_add;
    operator_token_type operator_sub;
    operator_token_type operator_mul;
    operator_token_type operator_div;

    value_token_type value;
    variable_token_type variable;

    std::vector<parenthesis_token_pair_type> parenthesis;

    whitespace_token_type whitespace;
};

namespace AST {
    using operation = operation::type;

    using value     = double;
    using variable  = std::string;

    struct bin_expr;
    using expression = boost::variant<value, variable, boost::recursive_wrapper<bin_expr> >;

    struct bin_expr {
        expression lhs, rhs;
        operation op;

        friend std::ostream& operator<<(std::ostream& os, bin_expr const& be) {
            return os << "(" << be.lhs << " " << be.op << " " << be.rhs << ")";
        }
    };
}

BOOST_FUSION_ADAPT_STRUCT(AST::bin_expr, lhs, op, rhs)

template<typename Iterator>
class expression_grammar : public qi::grammar<Iterator, AST::expression()>
{
public:
    template<typename Tokens>
    explicit expression_grammar(Tokens const& tokens)
        : expression_grammar::base_type(start)
    {
        start                     = expression >> qi::eoi;

        bin_sum_expr              = sum_operand >> sum_operator >> expression;
        bin_fac_expr              = fac_operand >> fac_operator >> sum_operand;

        expression                = bin_sum_expr | sum_operand;
        sum_operand               = bin_fac_expr | fac_operand;

        sum_operator              = tokens.operator_add >> qi::attr(AST::operation::add) | tokens.operator_sub >> qi::attr(AST::operation::sub);
        fac_operator              = tokens.operator_mul >> qi::attr(AST::operation::mul) | tokens.operator_div >> qi::attr(AST::operation::div);

        if(tokens.parenthesis.empty()) {
            fac_operand           = terminal;
        }
        else {
            fac_operand           = parenthesised | terminal;

            parenthesised         = tokens.parenthesis.front().first >> expression >> tokens.parenthesis.front().second;
            std::for_each(tokens.parenthesis.cbegin() + 1, tokens.parenthesis.cend(),
                    [&](typename Tokens::parenthesis_token_pair_type const& token_pair)
                    {
                        parenthesised = parenthesised.copy() | (token_pair.first >> expression >> token_pair.second);
                    });
        }

        terminal                  = tokens.value | tokens.variable;

        BOOST_SPIRIT_DEBUG_NODES(
                (start) (expression) (bin_sum_expr) (bin_fac_expr)
                (fac_operand) (terminal) (parenthesised) (sum_operand)
                (sum_operator) (fac_operator)
            );
    }

private:
    qi::rule<Iterator, AST::expression()> start;
    qi::rule<Iterator, AST::expression()> expression;
    qi::rule<Iterator, AST::expression()> sum_operand;
    qi::rule<Iterator, AST::expression()> fac_operand;
    qi::rule<Iterator, AST::expression()> terminal;
    qi::rule<Iterator, AST::expression()> parenthesised;

    qi::rule<Iterator, int()> sum_operator;
    qi::rule<Iterator, int()> fac_operator;

    // extra rules to help with AST creation
    qi::rule<Iterator, AST::bin_expr()> bin_sum_expr;
    qi::rule<Iterator, AST::bin_expr()> bin_fac_expr;
};

namespace RPN {
    using cell      = boost::variant<AST::operation, AST::value, AST::variable>;
    using rpn_stack = std::vector<cell>;

    struct transform : boost::static_visitor<> {
        void operator()(rpn_stack& stack, AST::expression const& e) const {
            boost::apply_visitor(boost::bind(*this, boost::ref(stack), ::_1), e);
        }
        void operator()(rpn_stack& stack, AST::bin_expr const& e) const {
            (*this)(stack, e.lhs);
            (*this)(stack, e.rhs);
            stack.push_back(e.op);
        }
        void operator()(rpn_stack& stack, AST::value    const& v) const { stack.push_back(v); }
        void operator()(rpn_stack& stack, AST::variable const& v) const { stack.push_back(v); }
    };
}

int main()
{
    typedef lex::lexertl::token<std::string::const_iterator, boost::mpl::vector<operation::type, double, std::string>> token_type;
    typedef expression_lexer<lex::lexertl::actor_lexer<token_type>> expression_lexer_type;
    typedef expression_lexer_type::iterator_type expression_lexer_iterator_type;
    typedef expression_grammar<expression_lexer_iterator_type> expression_grammar_type;

    expression_lexer_type lexer;
    expression_grammar_type grammar(lexer);
    RPN::transform compiler;

    std::string line;
    while(std::getline(std::cin, line) && !line.empty())
    {
        std::string::const_iterator first = line.begin();
        std::string::const_iterator const last = line.end();

        AST::expression expr;
        bool const result = lex::tokenize_and_parse(first, last, lexer, grammar, expr);
        if(!result)
            std::cout << "Parsing failed!\n";
        else
        {
            std::cout << "Parsing success: " << expr << "\n";

            RPN::rpn_stack program;
            compiler(program, expr);

            for (auto& instr : program) {
                std::cout << instr << " ";
            }
        }

        if(first != last)
            std::cout << "Remainder: >" << std::string(first, last) << "<\n";
    }
}

使用来自Boost.Spirit

1 个答案:

完整列表