why does Spirit not throw an qi::expectation_failure but just parse ok with unparsed lefts?

Question

Its seems that Expectations are only relevant when Spirit already detected that a rule is applicable (from the start) or else the parsing will just fail

It's a simple C-Identifier rule and a Identfier:Identifier rule skipping spaces to show my misunderstanding

Live on Coliru

// #define BOOST_SPIRIT_DEBUG
#include <boost/fusion/include/io.hpp>
#include <boost/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <iomanip>
namespace qi = boost::spirit::qi;

namespace Ast
{
    struct Identifier : std::string
    {
    };

    struct A_B
    {
        Identifier a;
        Identifier b;
    };
} // namespace Ast

BOOST_FUSION_ADAPT_STRUCT( Ast::A_B, a, b )

using Iter = std::string_view::const_iterator;

template <typename ValueType, typename RuleType>
void printing_parse_test( const RuleType& rule_, const std::vector<std::string>& tests )
{
    for( std::string_view test : tests )
    {
        try
        {
            std::cout << "Parsing: '" << test << "'\n";
            Iter f = test.begin(), l = test.end();
            // ">> qi::omit[*qi::space]" at end to eat in-between rest spaces ...
            ValueType v;
            const bool ok = qi::parse( f, l, rule_ >> qi::omit[*qi::space], v );
            std::cout << "parse-result: " << ( ok ? "ok" : "failed" ) << "\n";
            const std::string left( f, l );
            if( ok )
            {
                if( !left.empty() )
                {
                    std::cout << "FAIL: parsed someway but left this '" << left << "'\n ";
                }
                else
                {
                    std::cout << "OK: all parsed\n";
                }
            }
            else
            {
                if( !left.empty() )
                {
                    std::cout << "FAIL: parse failed left this: '" << left << "'\n";
                }
            }
        }
        catch( qi::expectation_failure<Iter> const& ef )
        {
            auto f = begin( test );
            auto p = ef.first - test.begin();
            auto bol = test.find_last_of( "\r\n", p ) + 1;
            auto line = std::count( f, f + bol, '\n' ) + 1;
            auto eol = test.find_first_of( "\r\n", p );

            std::cout << " -> EXPECTED " << ef.what_ << " in line:" << line << " col:" << ( p - bol ) << "\n"
                      << test.substr( bol, eol - bol ) << "\n"
                      << std::setw( p - bol ) << ""
                      << "^--- here" << std::endl;
        }
        std::cout << "--------------------\n";
    }
    std::cout << "============\n";
}

int main()
{
    qi::rule<Iter, Ast::Identifier()> identifier_rule = qi::char_( "a-zA-Z_" ) > *qi::char_( "a-zA-Z0-9_" );
    identifier_rule.name( "identifier_rule" );
    printing_parse_test<Ast::Identifier>( identifier_rule, { "  test blib ", "", "23434", "a$", "test blub" } );

    qi::rule<Iter, Ast::A_B()> A_B_rule = qi::skip( qi::space )[( identifier_rule > ":" > identifier_rule )];
    A_B_rule.name( "A_B_rule" );
    printing_parse_test<Ast::A_B>( A_B_rule, { "e$:bd", "a", "", "23434", "a$", "a:$" } );

    return 0;
}

Output:

Parsing: '  test blib '
parse-result: failed
FAIL: parse failed left this: '  test blib '
--------------------
Parsing: ''
parse-result: failed
--------------------
Parsing: '23434'
parse-result: failed
FAIL: parse failed left this: '23434'
--------------------
Parsing: 'a$'
parse-result: ok
FAIL: parsed someway but left this '$'
 --------------------
Parsing: 'test blub'
parse-result: ok
FAIL: parsed someway but left this 'blub'
 --------------------
============
Parsing: 'e$:bd'
 -> EXPECTED ":" in line:1 col:1
e$:bd
 ^--- here
--------------------
Parsing: 'a'
 -> EXPECTED ":" in line:1 col:1
a
 ^--- here
--------------------
Parsing: ''
parse-result: failed
--------------------
Parsing: '23434'
parse-result: failed
FAIL: parse failed left this: '23434'
--------------------
Parsing: 'a$'
 -> EXPECTED ":" in line:1 col:1
a$
 ^--- here
--------------------
Parsing: 'a:$'
 -> EXPECTED <identifier_rule> in line:1 col:2
a:$
  ^--- here
--------------------
============

Expectation:

why do i not get an EXPECTED <identifier_rule> when parsing '23434', or empty string etc. with both rules? isn't my expectation that the string starts with an identifier? how can i archive that or is that something that will be a problem when combining my rules later?

and an additional question is why "a$" results only in ':' expected not the additional information that also identifier literals are ok (or is that more of an hint?)

Answer 1

You already correctly found the way to cause an expectation point.

You can further simplify by using a skipper and qi::eoi to check that all input is consumed. I'd suggest the simplified listing:

Live On Coliru

// #define BOOST_SPIRIT_DEBUG
#include <boost/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <iomanip>
namespace qi = boost::spirit::qi;

namespace Ast {
    struct Identifier : std::string {};
    struct QualifiedId { Identifier a, b; };
} // namespace Ast

BOOST_FUSION_ADAPT_STRUCT(Ast::QualifiedId, a, b)

using It = std::string_view::const_iterator;

template <typename Attr, typename Skipper>
void run_tests(qi::rule<It, Attr(), Skipper> const& rule_, std::vector<std::string> const& tests) {
    std::cout << "======[ " << boost::core::demangle(typeid(Attr).name()) << " ]======" << std::endl;
    for (std::string_view test : tests) {
        It f = test.begin(), l = test.end();
        try {
            std::cout << "Parsing: " << quoted(test);
            Attr  v;
            bool  ok;

            if constexpr (std::is_same_v<Skipper, qi::unused_type>)
                ok = qi::parse(f, l, qi::eps > rule_ > qi::eoi, v);
            else
                ok = qi::phrase_parse(f, l, qi::eps > rule_ > qi::eoi, Skipper{}, v);

            std::cout << (ok ? "OK" : "FAIL") << " Remaining: " << quoted(std::string(f, l)) << "\n";
        } catch (qi::expectation_failure<It> const& ef) {
            auto p    = ef.first - test.begin();
            auto bol  = test.find_last_of("\r\n", p) + 1;
            auto line = std::count(f, f + bol, '\n') + 1;
            auto eol  = test.find_first_of("\r\n", p);

            std::cout << " -> EXPECTED " << ef.what_ << " in line:" << line << " col:" << (p - bol) << "\n"
                      << "    " << test.substr(bol, eol - bol) << "\n"
                      << "    " << std::setw(p - bol) << "" << "^--- here\n";
        }
        std::cout << "--------------------\n";
    }
}

int main() {
    qi::rule<It, Ast::Identifier()>                  id = qi::char_("a-zA-Z_") >> *qi::char_("a-zA-Z0-9_");
    qi::rule<It, Ast::QualifiedId(), qi::space_type> qualId = qi::eps > id > ":" > id;
    BOOST_SPIRIT_DEBUG_NODES((id)(qualId))

    run_tests(id, {"  test blib ", "", "23434", "a$", "test blub"});
    run_tests(qualId, {"e$:bd", "a", "", "23434", "a$", "a:$"});
}

Which prints

======[ Ast::Identifier ]======
Parsing: "  test blib " -> EXPECTED <id> in line:1 col:0
      test blib 
    ^--- here
--------------------
Parsing: "" -> EXPECTED <id> in line:1 col:0
    
    ^--- here
--------------------
Parsing: "23434" -> EXPECTED <id> in line:1 col:0
    23434
    ^--- here
--------------------
Parsing: "a$" -> EXPECTED <eoi> in line:1 col:1
    a$
     ^--- here
--------------------
Parsing: "test blub" -> EXPECTED <eoi> in line:1 col:4
    test blub
        ^--- here
--------------------
======[ Ast::QualifiedId ]======
Parsing: "e$:bd" -> EXPECTED ":" in line:1 col:1
    e$:bd
     ^--- here
--------------------
Parsing: "a" -> EXPECTED ":" in line:1 col:1
    a
     ^--- here
--------------------
Parsing: "" -> EXPECTED <id> in line:1 col:0
    
    ^--- here
--------------------
Parsing: "23434" -> EXPECTED <id> in line:1 col:0
    23434
    ^--- here
--------------------
Parsing: "a$" -> EXPECTED ":" in line:1 col:1
    a$
     ^--- here
--------------------
Parsing: "a:$" -> EXPECTED <id> in line:1 col:2
    a:$
      ^--- here
--------------------

Answer 2

found it

qi::eps

is the answer - this way i've got a expected sequence right from the start

const bool ok = qi::parse( f, l, qi::eps > rule_ >> qi::omit[*qi::space], v );

gives me the execption also on start

Output

Parsing: '  test blib '
 -> EXPECTED <identifier_rule> in line:1 col:0
  test blib
^--- here
--------------------
Parsing: ''
 -> EXPECTED <identifier_rule> in line:1 col:0

^--- here
--------------------
Parsing: '23434'
 -> EXPECTED <identifier_rule> in line:1 col:0
23434
^--- here
--------------------
Parsing: 'a$'
parse-result: ok
FAIL: parsed someway but left this '$'
 --------------------
Parsing: 'test blub'
parse-result: ok
FAIL: parsed someway but left this 'blub'
 --------------------
============
Parsing: 'e$:bd'
 -> EXPECTED ":" in line:1 col:1
e$:bd
 ^--- here
--------------------
Parsing: 'a'
 -> EXPECTED ":" in line:1 col:1
a
 ^--- here
--------------------
Parsing: ''
 -> EXPECTED <A_B_rule> in line:1 col:0

^--- here
--------------------
Parsing: 'x 23434'
 -> EXPECTED ":" in line:1 col:2
x 23434
  ^--- here
--------------------
Parsing: 'a$'
 -> EXPECTED ":" in line:1 col:1
a$
 ^--- here
--------------------
Parsing: 'a:$'
 -> EXPECTED <identifier_rule> in line:1 col:2
a:$
  ^--- here
--------------------
============