Boost spirit permutation of rules

Question

I have 2 type of expressions that I want parse and calculate the results.

• Artimetic expressions: +,-,*,/ and sqrt() function; Ex: "2 + 3 * sqrt(100*25)" -> should be calculated as 152

• Functions: GetSubString() and ConcatenateStrings() Ex: "GetSubString('100str1', 0, 3)" -> should be calculated as 100

I have 2 seperate grammars to parse these expression types. Now I want to combine these 2 grammars and make it possible to define these expressions together.

Ex:

• "GetSubString('100str1', 0, 2+1) + sqrt(9)" -> result= 103
• "2 + 3 * sqrt(GetSubString('100str1', 0, 2+1))" -> result= 32

I have tried to combine 2 grammars as below by using permutation operator. But it doesnt compile.

    expr_    =
        ( *( (function_call_ ^ arithmeticexpression_)| string_ ));

So is this a right way to combine my function_call_ and arithmeticexpression_ rules or how should I do this?

typedef boost::variant<int, float, double, std::wstring> RetValue;


RetValue CTranslationFunctions::GetSubString(RetValue const& str, RetValue position, RetValue len)
{
    std::wstring strToCut;
    size_t posInt = 0;
    size_t lenInt = 0;

    try
    {
        strToCut = boost::get<std::wstring>(str);
        posInt = boost::get<int>(position);
        lenInt = boost::get<int>(len);
    }
    catch (const boost::bad_get&)
    {
        throw;
    }

    return strToCut.substr(posInt, lenInt);
}

RetValue CTranslationFunctions::ConcatenateStrings(RetValue const& a, RetValue const& b) 
{
    wostringstream woss;
    woss << a << b;
    return woss.str();
}

double CTranslationFunctions::Negate(double num)
{
    return -num;
}

double CTranslationFunctions::Add(double num1 , const double num2)
{
    return num1 + num2;
};

double CTranslationFunctions::Subtruct(double num1 , double num2)
{
    return num1 - num2;
};

double CTranslationFunctions::Multiply(double num1 , double num2)
{
    return num1 * num2;
};

double CTranslationFunctions::Divide(double num1 , double num2)
{
    return num1 / num2;
};

double CTranslationFunctions::Sqrt(double num)
{
    return sqrt(num);
}

class InvalidParamEx{};

double CTranslationFunctions::ConvertStringToDouble(RetValue val)
{
    wostringstream wss;
    double dNum;
    wss << val;
    std::wistringstream iss;
    iss.str(wss.str());
    try
    {
        iss >> dNum;
    }
    catch (...)
    {
        throw InvalidParamEx();
    }

    return dNum;
}


BOOST_PHOENIX_ADAPT_FUNCTION(RetValue, ConcatenateStrings_, ConcatenateStrings, 2)
BOOST_PHOENIX_ADAPT_FUNCTION(RetValue, GetContainerId_, GetContainerId, 2)

BOOST_PHOENIX_ADAPT_FUNCTION(double, Add_, Add, 2)
BOOST_PHOENIX_ADAPT_FUNCTION(double, Subtruct_, Subtruct, 2)
BOOST_PHOENIX_ADAPT_FUNCTION(double, Multiply_, Multiply, 2)
BOOST_PHOENIX_ADAPT_FUNCTION(double, Divide_, Divide, 2)
BOOST_PHOENIX_ADAPT_FUNCTION(double, Negate_, Negate, 1)
BOOST_PHOENIX_ADAPT_FUNCTION(double, Sqrt_, Sqrt, 1)
BOOST_PHOENIX_ADAPT_FUNCTION(double, ConvertStringToDouble_, ConvertStringToDouble, 1)

// Grammar to parse map functions
template <typename It, typename Skipper = qi::space_type >
struct MapFunctionParser : qi::grammar<It, RetValue(), Skipper, qi::locals<char>  >
{
    MapFunctionParser() : MapFunctionParser::base_type(expr_)
    {
        using namespace qi;

        function_call_ = 

        | (lit(L"GetSubString") > '(' > expr_ > ',' > expr_ > ',' > expr_ > ')')               
            [ _val = GetSubString_(_1, _2, _3) ]
        | (lit(L"ConcatenateStrings") > '(' > expr_ > lit(',') > expr_ > ')')               
            [ _val = ConcatenateStrings_(_1, _2) ];


        string_ = as_wstring[omit    [ char_("'\"") [_a =_1] ]        
        >> no_skip [ *(char_ - char_(_a))  ]
        >> lit(_a)];

        arithmeticexpression_ =
            term_                        [_val = _1]
        >>  *( ('+' >> term_           [_val = Add_(_val,_1)])
             |   ('-' >> term_           [_val = Subtruct_(_val, _1)])
            );

        term_ =
            factor_              [_val = _1]
        >> *( ('*' >> factor_  [_val = Multiply_(_val, _1)])
            |   ('/' >> factor_  [_val = Divide_(_val, _1)])
            );

        factor_ =
            double_                            [_val = _1]
          string_                            [_val = ConvertStringToDouble(_1)]
      |   ('-' >> factor_                  [_val = Negate_(_1)])
      |   ('+' >> factor_                  [_val = _1])
        |   (L"Sqrt" > '(' > double_ > ')'  )     [_val = Sqrt_(_1)]);


        expr_    =
            ( *( (function_call_ ^ arithmeticexpression_)| string_ ));


        on_error<fail> ( expr_, std::cout
            << phx::val("Error! Expecting ") << _4 << phx::val(" here: \"")
            << phx::construct<std::string>(_3, _2) << phx::val("\"\n"));

        BOOST_SPIRIT_DEBUG_NODE(function_call_);
        BOOST_SPIRIT_DEBUG_NODE(expr_);
        BOOST_SPIRIT_DEBUG_NODE(string_);
        BOOST_SPIRIT_DEBUG_NODE(funcparameter_);
        BOOST_SPIRIT_DEBUG_NODE(arithmeticexpression_);
        BOOST_SPIRIT_DEBUG_NODE(factor_);
        BOOST_SPIRIT_DEBUG_NODE(term_);
    }

private:
    qi::rule<It, RetValue(), Skipper, qi::locals<char> > function_call_, expr_, funcparameter_;
    qi::rule<It, wstring(), Skipper, qi::locals<char> > string_;
    qi::rule<It, double(), Skipper> arithmeticexpression_, factor_,  term_;
};

Answer 1

Edit Moved my early response to the bottom

BIG UPDATE

That took a while. Mostly it was because the code shown has strange problems:

• several rules contain syntax errors (function_call and factor_
• there is a reference to GetContainerId and GetSubstring was never Phoenix-adapted
• The type CTranslationFunctions didn't exist, and member functions were being declared
• however the ADAPT_FUNCTION macros still referenced the member function names as if they were supposed to be in the enclosing namespace (?!)

So what I basically ended up doing was a re-write. Yeah I know. I'm crazy. Nevertheless, let me walk you through it, explaining some of the things I changed and why.

#define BOOST_SPIRIT_USE_PHOENIX_V3
// #define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/phoenix/function/adapt_function.hpp>
#include <boost/lexical_cast.hpp>

namespace qi    = boost::spirit::qi;
namespace phx   = boost::phoenix;

typedef boost::variant<int, double>               NumValue;
typedef boost::variant<int, double, std::wstring> GenericValue;

Right away, I split the concept of numeric and generic values. This is because the distinction is important to certain expressions (mainly the arithmetic expressions). I could have still used GenericValue everywhere, but we'll see later how NumValue makes handling the arithmetic evaluations simpler.

struct InvalidParamEx : public virtual std::exception 
{
    const char* what() const noexcept { return "Invalid type of operand/parameter"; }
};

There's your exception type, showing some good practices. We throw it when a numeric value was expected, but the GenericValue contained something incompatible. How? Let's see:

struct AsNumValue : boost::static_visitor<NumValue>
{
    int      operator()(int i)                 const { return i; }
    double   operator()(double d)              const { return d; }
    NumValue operator()(std::wstring const& s) const
    { 
        try { return boost::lexical_cast<int>(s); }    catch(...) {}
        try { return boost::lexical_cast<double>(s); } catch(...) {}
        throw InvalidParamEx();
    }
};

class CTranslationFunctions
{
    // helper
    NumValue static num(GenericValue const& v) { return boost::apply_visitor(AsNumValue(), v); }

There. I defined the missing class for you, and right away added the helper that converts GenericValue → NumValue. As you can see, I used boost::lexical_cast because there is no use in reinventing the wheel. Note that your earlier approach with ConvertStringToDouble had several big problems:

1. it would always result in a double value, whereas your functions may require int
2. it would treate '100str1' as the value 100 without warning
3. it happened at the wrong time: any string a simple term would be converted to double, even if it really was a string. (Why this was relevant, will become clear when you see the modified expr_ and term_ rules.)

Let's move on:

  public:
    static GenericValue GetSubString(GenericValue const& str, GenericValue position, GenericValue len);
    static GenericValue ConcatenateStrings(GenericValue const& a, GenericValue const& b);

Yup, we'll define them later. Now, brace yourself for the arithmetic operation functions:

#define DEFUNOP(name, expr) private: struct do_##name : boost::static_visitor<NumValue> {    \
        template <typename T1> NumValue operator()(T1 const& a) const { return expr; }       \
    }; \
    public: static NumValue name(GenericValue const& a) { auto na=num(a); return boost::apply_visitor(do_##name(), na); }
#define DEFBINOP(name, infix) struct do_##name : boost::static_visitor<NumValue> {           \
        template <typename T1, typename T2> NumValue operator()(T1 const&a, T2 const&b) const\
        { return a infix b; }                                                                \
    }; \
    public: static NumValue name(GenericValue const& a, GenericValue const& b) { auto na=num(a), nb=num(b); return boost::apply_visitor(do_##name(), na, nb); }

    // define the operators polymorphically, so `int` + `double` becomes `double`, but `int` * `int` stays `int`
    DEFBINOP(Add     , +);
    DEFBINOP(Subtruct, -);
    DEFBINOP(Multiply, *);
    DEFBINOP(Divide  , /);
    DEFUNOP (Negate  , -a);
    DEFUNOP (Sqrt    , sqrt(a));
};

Whoaaaaah What happened there? Well, the comment says it all:

• You needed to distinguish between int+int vs. double+int etc. This is known as polymorphic evaluation. Example: GetSubString('100str1', 0, 2+1) could never work, because 2+1 needs to evaluate to an int(3), but your double Add(double,double) always produced a double.
• I've used MACROs to remove the tedious work from creating a polymorphic function object for each operator
• I've let decltype detect the resultant types in mixed cases
• Here's where NumValue has merit above GenericValue: because NumValue can only be int or double, we know that the generic operator() implementation covers all legal combinations.
• To ensure that all parameters are actually NumValues, they are passed through asNumeric before calling the function object.

This thorughly solves your arithmetic operations, and has another bonus: it removes the 'need' for ConvertStringToDouble, since you get conversion to NumValue when it's needed, namely on evaluation of arithmetic operations. This is an important thing, down the road when we fix your grammar to support your desired input expressions.

If you've come this far, you've seen the rough parts. The rest is plain sailing.

GenericValue CTranslationFunctions::GetSubString(GenericValue const& str, GenericValue position, GenericValue len)
{
    using boost::get;
    return get<std::wstring>(str).substr(get<int>(position), get<int>(len));
}

Yeah, I shortened it a bit.

GenericValue CTranslationFunctions::ConcatenateStrings(GenericValue const& a, GenericValue const& b) 
{
    std::wostringstream woss;
    woss << a << b;
    return woss.str();
}

BOOST_PHOENIX_ADAPT_FUNCTION(GenericValue, ConcatenateStrings_, CTranslationFunctions::ConcatenateStrings, 2)
BOOST_PHOENIX_ADAPT_FUNCTION(GenericValue, GetSubString_      , CTranslationFunctions::GetSubString      , 3)

BOOST_PHOENIX_ADAPT_FUNCTION(NumValue    , Add_               , CTranslationFunctions::Add               , 2)
BOOST_PHOENIX_ADAPT_FUNCTION(NumValue    , Subtruct_          , CTranslationFunctions::Subtruct          , 2)
BOOST_PHOENIX_ADAPT_FUNCTION(NumValue    , Multiply_          , CTranslationFunctions::Multiply          , 2)
BOOST_PHOENIX_ADAPT_FUNCTION(NumValue    , Divide_            , CTranslationFunctions::Divide            , 2)
BOOST_PHOENIX_ADAPT_FUNCTION(NumValue    , Negate_            , CTranslationFunctions::Negate            , 1)
BOOST_PHOENIX_ADAPT_FUNCTION(NumValue    , Sqrt_              , CTranslationFunctions::Sqrt              , 1)

Yawn. We know how to adapt functions for Phoenix, already; let's get to the grammar definition!

// Grammar to parse map functions
template <typename It, typename Skipper = qi::space_type >
struct MapFunctionParser : qi::grammar<It, GenericValue(), Skipper>
{
    MapFunctionParser() : MapFunctionParser::base_type(expr_)
    {
        using namespace qi;

        function_call_ = 
          (no_case["GetSubString"]       > '(' > expr_ > ',' > expr_ > ',' > expr_ > ')') [ _val = GetSubString_(_1, _2, _3)   ]
        | (no_case["ConcatenateStrings"] > '(' > expr_ > ',' > expr_ > ')')               [ _val = ConcatenateStrings_(_1, _2) ]
        | (no_case["Sqrt"]               > '(' > expr_ > ')')                             [ _val = Sqrt_(_1)                   ]
        ;

        string_ = // keep it simple, silly (KISS)
            (L'"' > *~char_('"') > L'"')
          | (L"'" > *~char_("'") > L"'");  

        arithmeticexpression_ =
            term_                  [ _val = _1                  ]
        >>  *( ('+' >> term_       [ _val = Add_(_val,_1)       ])
             | ('-' >> term_       [ _val = Subtruct_(_val, _1) ])
             );                      

        term_ =                      
              factor_              [ _val = _1                  ]
            >> *( ('*' >> factor_  [ _val = Multiply_(_val, _1) ])
                | ('/' >> factor_  [ _val = Divide_(_val, _1)   ])
                );

        factor_ =
                int_               [ _val = _1          ]
            |   double_            [ _val = _1          ]
            |   string_            [ _val = _1          ]
            |   ('-' >> factor_)   [ _val = Negate_(_1) ]
            |   ('+' >> factor_)   [ _val = _1          ]
            |   function_call_     [ _val = _1          ]
            ;

        expr_ = arithmeticexpression_;

        on_error<fail> ( expr_, std::cout
            << phx::val("Error! Expecting ") << _4 << phx::val(" here: \"")
            << phx::construct<std::string>(_3, _2) << phx::val("\"\n"));

        BOOST_SPIRIT_DEBUG_NODES((function_call_) (expr_) (string_) (funcparameter_) (arithmeticexpression_) (factor_) (term_))
    }

private:
    qi::rule<It, std::wstring()> 
        string_; // NO SKIPPER (review)
    qi::rule<It, GenericValue(), Skipper> 
        function_call_, expr_, funcparameter_, // NO LOCALS (review)
        arithmeticexpression_, term_, factor_;
};

Well. What have we here. What changed?

• I removed qi::locals which was only ever used in the string_ rule anyways, and I rewrote that to honour the KISS principle
• I also fixed the problem with whitespace in strings (your parser would have parsed " oops " identical to "oops"). I did so by removing the Skipper from the string_ declaration. This has the same effect as enclosing the whole rule in qi::lexeme[].
• I moved Sqrt to the function_call_ rule, because, well, it's a function call.
• I tweaked the function names to be no_case[] case insensitive, since your examples suggested that sqrt(9) should work

• Note that Sqrt now takes any expression whereas the old situation had

  | (L"Sqrt" > '(' > double_ > ')') // Wait, whaaat?
  

  Yeah, this was never going to parse your second example, really :|

Now the real meat of the operation comes. In order to let sqrt(GetSubstring(....)) parse, we'll have to let function_call_ be a possible value for a term_. Once that's the case, we don't have to anything more in expr_ since expr_ might consist of a single factor_ containing a single term_ representing a function_call_ already, so

expr_    = ( *( (function_call_ ^ arithmeticexpression_)| string_ ));

evaporates into

expr_    = arithmeticexpression_;

What happened to string_ there? Well, it's still in term_, where it was, but the ConvertStringToDouble was removed there. Strings will just happily be strings, unless they are required in the context of an arithmetic operation that requires NumValues. That's when they will be coerced into a number, and no earlier (as shown above).

int main()
{
    static const MapFunctionParser<std::wstring::const_iterator> p;

    std::wstring input;
    while (std::getline(std::wcin, input))
    {
        std::wstring::const_iterator f(begin(input)), l(end(input));

        GenericValue value;
        assert(qi::phrase_parse(f, l, p, qi::space, value));

        if (f!=l)
            std::wcout << L"remaining unparsed: '" << std::wstring(f,l) << L"'\n";

        std::wcout << input << " --> " << value << std::endl;
    }
}

When I fed this little test program the two lines from your question, it dutifully churned out the following text:

GetSubString('100str1', 0, 2+1) + sqrt(9) --> 103
2 + 3 * sqrt(GetSubString('100str1', 0, 2+1)) --> 32

You can see the full code on Coliru (sadly, it takes too long to compile).

---

Originally this answer started with the following:

Q. I have tried to combine 2 grammars as below by using permutation operator. But it doesnt compile

What did you expect the permutation operator to do? The documentation states:

The permutation operator, a ^ b, matches one or more operands (a, b, ... etc.) in any order
...

As you can see it would result in an attribute

 boost::variant<
      fusion::vector2<optional<RetValue>, optional<double>>,
      std::wstring>

which clearly is not compatible. Now, I assume you just want either/or semantics, so

 expr_    = string_ | function_call_ | arithmeticexpression_;

should do nicely, resulting in boost::variant<RetValue, double, std::wstring> which is assignable to a RetValue.

Now after jumping through a dozen hoops to make your sample code compile (why...) here's a fix: