How do I test both sides of a `if @generated`?

Question

Consider I might have some generated functions I want to test as below. Following recommendation of the manual I made them optionally generated.

# This code is taken directly from Base. 
# https://github.com/JuliaLang/julia/blob/592748adb25301a45bd6edef3ac0a93eed069852/base/namedtuple.jl#L220-L231
# So importing some of the helpers
using Base: merge_names, merge_types, sym_in

function my_merge_fail1(a::NamedTuple{an}, b::NamedTuple{bn}) where {an, bn}
    if @generated
        names = merge_names(an, bn)
        types = merge_types(names, a, b)
        vals = Any[ :(getfield($(sym_in(n, bn) ? :b : :a), $(QuoteNode(n)))) for n in names ]
        :(error("typo1"); NamedTuple{$names,$types}(($(vals...),)) )
    else
        names = merge_names(an, bn)
        types = merge_types(names, typeof(a), typeof(b))
        NamedTuple{names,types}(map(n->getfield(sym_in(n, bn) ? b : a, n), names))
    end
end

function my_merge_fail2(a::NamedTuple{an}, b::NamedTuple{bn}) where {an, bn}
    if @generated
        names = merge_names(an, bn)
        types = merge_types(names, a, b)
        vals = Any[ :(getfield($(sym_in(n, bn) ? :b : :a), $(QuoteNode(n)))) for n in names ]
        :(NamedTuple{$names,$types}(($(vals...),)) )
    else
        error("typo2")
        names = merge_names(an, bn)
        types = merge_types(names, typeof(a), typeof(b))
        NamedTuple{names,types}(map(n->getfield(sym_in(n, bn) ? b : a, n), names))
    end
end

Now, it turns out I made a "typo", in each. I mistakenly included error(...) in both of them. In my_merge_fail1 I made the mistike in the @generated branch, and in my_merge_fail2 in the nongenerated branch.

My tests seem to only catch it in one branch:

julia> using Test

julia> @test my_merge_fail1((a=1,), (b=2,)) == (a=1, b=2)
Error During Test at REPL[12]:1
  Test threw exception
  Expression: my_merge_fail1((a = 1,), (b = 2,)) == (a = 1, b = 2)
  typo1
  Stacktrace:
   [1] error(::String) at ./error.jl:33
   [2] macro expansion at ./REPL[6]:2 [inlined]
   [3] my_merge_fail1(::NamedTuple{(:a,),Tuple{Int64}}, ::NamedTuple{(:b,),Tuple{Int64}}) at ./REPL[6]:2
   [4] top-level scope at REPL[12]:1
   [5] eval(::Module, ::Any) at ./boot.jl:331
   [6] eval_user_input(::Any, ::REPL.REPLBackend) at /usr/local/src/julia/julia-master/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:86
   [7] macro expansion at /usr/local/src/julia/julia-master/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:118 [inlined]
   [8] (::REPL.var"#26#27"{REPL.REPLBackend})() at ./task.jl:333

ERROR: There was an error during testing

julia> @test my_merge_fail2((a=1,), (b=2,)) == (a=1, b=2)
Test Passed

How can I improve my tests?

Answer 1

It is possible to achieve this by extracting the lowered code from the method, where you can specify whether you get the one for the generated or non-generated version of this method. This approach uses eval and Base.invokelatest, so it will have some overhead, but it should be fine for testing.

function _invoke(f, x...; generated)
    sig = Core.Typeof(x)
    ci = code_lowered(f, sig; generated)[1]
    Meta.partially_inline!(
        ci.code, [], Tuple{typeof(f),sig.parameters...}, Any[sig.parameters...],
        0, 0, :propagate,
    )
    g = @eval @generated function $(gensym(:g))($([gensym() for _ in x]...))
        return $(QuoteNode(ci))
    end
    Base.invokelatest(g, x...)
end

You can then choose to invoke either the generated or the non-generated version of the method like this:

julia> f(x, y) = @generated() ? :x : y
f (generic function with 1 method)

julia> _invoke(f, 1, 2; generated=true)
1

julia> _invoke(f, 1, 2; generated=false)
2

Answer 2

One solution would be make the two branches their own independent functions which you can test directly i.e.

using Base: merge_names, merge_types, sym_in

function my_merge_fail(a::NamedTuple{an}, b::NamedTuple{bn}) where {an, bn}
    if @generated
        :(my_merge_fail_inner_gen(a, b))
    else
        my_merge_fail_inner(a, b)
    end
end

@generated function my_merge_fail_inner_gen(a::NamedTuple{an}, b::NamedTuple{bn}) where {an, bn}
    names = merge_names(an, bn)
    types = merge_types(names, a, b)
    vals = Any[ :(getfield($(sym_in(n, bn) ? :b : :a), $(QuoteNode(n)))) for n in names ]
    :(NamedTuple{$names,$types}(($(vals...),)) )
end

function my_merge_fail_inner(a::NamedTuple{an}, b::NamedTuple{bn}) where {an, bn}
    error("typo2")
    names = merge_names(an, bn)
    types = merge_types(names, typeof(a), typeof(b))
    NamedTuple{names,types}(map(n->getfield(sym_in(n, bn) ? b : a, n), names))
end

then we run the tests like so:

julia> using Test

julia> @testset "my_merge_fail" begin
           @test my_merge_fail(          (a=1,), (b=2,)) == (a=1, b=2)
           @test my_merge_fail_inner_gen((a=1,), (b=2,)) == (a=1, b=2)
           @test my_merge_fail_inner(    (a=1,), (b=2,)) == (a=1, b=2)
       end
my_merge_fail: Error During Test at REPL[18]:4
  Test threw exception
  Expression: my_merge_fail_inner((a = 1,), (b = 2,)) == (a = 1, b = 2)
  typo2
  Stacktrace:
   [1] error(::String) at ./error.jl:33
   [2] my_merge_fail_inner(::NamedTuple{(:a,),Tuple{Int64}}, ::NamedTuple{(:b,),Tuple{Int64}}) at ./REPL[8]:2
   [3] top-level scope at REPL[18]:4
   [4] top-level scope at /Users/mason/julia/usr/share/julia/stdlib/v1.3/Test/src/Test.jl:1107
   [5] top-level scope at REPL[18]:2

Test Summary: | Pass  Error  Total
my_merge_fail |    2      1      3
ERROR: Some tests did not pass: 2 passed, 0 failed, 1 errored, 0 broken.

---

One thing I'm uncertain about though is if this refactoring has any effect on the heuristics julia uses to decide whether it uses the @generated branch or not. If anyone knows the answer to that, a comment or suggested edit would be appreciated.