Converting "yield from" statement to Python 2.7 code

Question

I had a code below in Python 3.2 and I wanted to run it in Python 2.7. I did convert it (have put the code of missing_elements in both versions) but I am not sure if that is the most efficient way to do it. Basically what happens if there are two yield from calls like below in upper half and lower half in missing_element function? Are the entries from the two halves (upper and lower) appended to each other in one list so that the parent recursion function with the yield from call and use both the halves together?

def missing_elements(L, start, end):  # Python 3.2
    if end - start <= 1: 
        if L[end] - L[start] > 1:
            yield from range(L[start] + 1, L[end])
        return

index = start + (end - start) // 2

# is the lower half consecutive?
consecutive_low =  L[index] == L[start] + (index - start)
if not consecutive_low:
    yield from missing_elements(L, start, index)

# is the upper part consecutive?
consecutive_high =  L[index] == L[end] - (end - index)
if not consecutive_high:
    yield from missing_elements(L, index, end)

def main():
    L = [10, 11, 13, 14, 15, 16, 17, 18, 20]
    print(list(missing_elements(L, 0, len(L)-1)))
    L = range(10, 21)
    print(list(missing_elements(L, 0, len(L)-1)))

def missing_elements(L, start, end):  # Python 2.7
    return_list = []                
    if end - start <= 1: 
        if L[end] - L[start] > 1:
            return range(L[start] + 1, L[end])

    index = start + (end - start) // 2

    # is the lower half consecutive?
    consecutive_low =  L[index] == L[start] + (index - start)
    if not consecutive_low:
        return_list.append(missing_elements(L, start, index))

    # is the upper part consecutive?
    consecutive_high =  L[index] == L[end] - (end - index)
    if not consecutive_high:
        return_list.append(missing_elements(L, index, end))
    return return_list

Answer 1

Another solution: by using my yield-from-as-an-iterator library, you can turn any yield from foo into

for value, handle_send, handle_throw in yield_from(foo):
    try:
        handle_send((yield value))
    except:
        if not handle_throw(*sys.exc_info()):
            raise

To make sure this answer stands alone even if the PyPI package is ever lost, here is an entire copy of that library's yieldfrom.py from the 1.0.0 release:

# SPDX-License-Identifier: 0BSD
# Copyright 2022 Alexander Kozhevnikov <mentalisttraceur@gmail.com>

"""A robust implementation of ``yield from`` behavior.
Allows transpilers, backpilers, and code that needs
to be portable to minimal or old Pythons to replace

    yield from ...

with

    for value, handle_send, handle_throw in yield_from(...):
        try:
            handle_send(yield value)
        except:
            if not handle_throw(*sys.exc_info()):
                raise
"""


__version__ = '1.0.0'
__all__ = ('yield_from',)


class yield_from(object):
    """Implementation of the logic that ``yield from`` adds around ``yield``."""

    __slots__ = ('_iterator', '_next', '_default_next')

    def __init__(self, iterable):
        """Initializes the yield_from instance.

        Arguments:
            iterable: The iterable to yield from and forward to.
        """
        # Mutates:
        #     self._next: Prepares to use built-in function next in __next__
        #         for the first iteration on the iterator.
        #     self._default_next: Saves initial self._next tuple for reuse.
        self._iterator = iter(iterable)
        self._next = self._default_next = next, (self._iterator,)

    def __repr__(self):
        """Represent the yield_from instance as a string."""
        return type(self).__name__ + '(' + repr(self._iterator) + ')'

    def __iter__(self):
        """Return the yield_from instance, which is itself an iterator."""
        return self

    def __next__(self):
        """Execute the next iteration of ``yield from`` on the iterator.

        Returns:
            Any: The next value from the iterator.

        Raises:
            StopIteration: If the iterator is exhausted.
            Any: If the iterator raises an error.
        """
        # Mutates:
        #     self._next: Resets to default, in case handle_send or
        #         or handle_throw changed it for this iteration.
        next_, arguments = self._next
        self._next = self._default_next
        value = next_(*arguments)
        return value, self.handle_send, self.handle_throw

    next = __next__  # Python 2 used `next` instead of ``__next__``

    def handle_send(self, value):
        """Handle a send method call for a yield.

        Arguments:
            value: The value sent through the yield.

        Raises:
            AttributeError: If the iterator has no send method.
        """
        # Mutates:
        #     self._next: If value is not None, prepares to use the
        #         iterator's send attribute instead of the built-in
        #         function next in the next iteration of __next__.
        if value is not None:
            self._next = self._iterator.send, (value,)

    def handle_throw(self, type, exception, traceback):
        """Handle a throw method call for a yield.

        Arguments:
            type: The type of the exception thrown through the yield.
                If this is GeneratorExit, the iterator will be closed
                by callings its close attribute if it has one.
            exception: The exception thrown through the yield.
            traceback: The traceback of the exception thrown through the yield.

        Returns:
            bool: Whether the exception will be forwarded to the iterator.
                If this is false, you should bubble up the exception.
                If this is true, the exception will be thrown into the
                iterator at the start of the next iteration, and will
                either be handled or bubble up at that time.

        Raises:
            TypeError: If type is not a class.
            GeneratorExit: Re-raised after successfully closing the iterator.
            Any: If raised by the close function on the iterator.
        """
        # Mutates:
        #     self._next: If type was not GeneratorExit and the iterator
        #         has a throw attribute, prepares to use that attribute
        #         instead of the built-in function next in the next
        #         iteration of __next__.
        iterator = self._iterator

        if issubclass(type, GeneratorExit):
            try:
                close = iterator.close
            except AttributeError:
                return False
            close()
            return False

        try:
            throw = iterator.throw
        except AttributeError:
            return False

        self._next = throw, (type, exception, traceback)
        return True

What I really like about this way is that:

1. The implementation is much easier to fully think through and verify for correctness than the alternatives*.
2. The usage is still simple, and doesn't require decorators or any other code changes anywhere other than just replacing the yield from ... line.
3. It still has robust forwarding of .send and .throw and handling of errors, StopIteration, and GeneratorExit.
4. This yield_from implementation will work on any Python 3 and on Python 2 all the way back to Python 2.5**.

_{* The formal specification ends up entangling all the logic into one big loop, even with some duplication thrown in. All the fully-featured backport implementations I've seen further add complication on top of that. But we can do better by embracing manually implementing the iterator protocol:}

• _{We get StopIteration handling for free from Python itself around our __next__ method.}
• _{The logic can be split into separate pieces which are entirely decoupled except for the state-saving between them, which frees you from having to de-tangle the logic by yourself - fundamentally yield from is just three simple ideas:}
  1. _{call a method on the iterator to get the next element,}
  2. _{how to handle .send (which may change the method called in step 1), and}
  3. _{how to handle .throw (which may change the method called in step 1).}
• _{By asking for modest boilerplate at each yield from replacement, we can avoid needing any hidden magic with special wrapper types, decorators, and so on.}

_{** Python 2.5 is when PEP-342 made yield an expression and added GeneratorExit. Though if you are ever unfortunate enough to need to backport or "backpile" (transpile to an older version of the language) this yield_from would still do all the hard parts of building yield from on top of yield for you.}

Also, this idea leaves a lot of freedom for how how usage boilerplate looks. For example,

1. handle_throw could be trivially refactored into a context manager, enabling usage like this:

   for value, handle_send, handle_throw in yield_from(foo):
       with handle_throw:
           handle_send(yield value)
   

   and

2. you could make value, handle_send, handle_throw something like a named tuple if you find this usage nicer:

   for step in yield_from(foo):
       with step.handle_throw:
           step.handle_send(yield step.value)
   

Answer 2

What about using the definition from pep-380 in order to construct a Python 2 syntax version:

The statement:

RESULT = yield from EXPR

is semantically equivalent to:

_i = iter(EXPR)
try:
    _y = next(_i)
except StopIteration as _e:
    _r = _e.value
else:
    while 1:
        try:
            _s = yield _y
        except GeneratorExit as _e:
            try:
                _m = _i.close
            except AttributeError:
                pass
            else:
                _m()
            raise _e
        except BaseException as _e:
            _x = sys.exc_info()
            try:
                _m = _i.throw
            except AttributeError:
                raise _e
            else:
                try:
                    _y = _m(*_x)
                except StopIteration as _e:
                    _r = _e.value
                    break
        else:
            try:
                if _s is None:
                    _y = next(_i)
                else:
                    _y = _i.send(_s)
            except StopIteration as _e:
                _r = _e.value
                break
RESULT = _r

In a generator, the statement:

return value

is semantically equivalent to

raise StopIteration(value)

except that, as currently, the exception cannot be caught by except clauses within the returning generator.

The StopIteration exception behaves as though defined thusly:

class StopIteration(Exception):

    def __init__(self, *args):
        if len(args) > 0:
            self.value = args[0]
        else:
            self.value = None
        Exception.__init__(self, *args)

Answer 3

If you don't use the results of your yields,* you can always turn this:

yield from foo

… into this:

for bar in foo:
    yield bar

There might be a performance cost,** but there is never a semantic difference.

---

Are the entries from the two halves (upper and lower) appended to each other in one list so that the parent recursion function with the yield from call and use both the halves together?

No! The whole point of iterators and generators is that you don't build actual lists and append them together.

But the effect is similar: you just yield from one, then yield from another.

If you think of the upper half and the lower half as "lazy lists", then yes, you can think of this as a "lazy append" that creates a larger "lazy list". And if you call list on the result of the parent function, you of course will get an actual list that's equivalent to appending together the two lists you would have gotten if you'd done yield list(…) instead of yield from ….

But I think it's easier to think of it the other way around: What it does is exactly the same the for loops do.

If you saved the two iterators into variables, and looped over itertools.chain(upper, lower), that would be the same as looping over the first and then looping over the second, right? No difference here. In fact, you could implement chain as just:

for arg in *args:
    yield from arg

---

* Not the values the generator yields to its caller, the value of the yield expressions themselves, within the generator (which come from the caller using the send method), as described in PEP 342. You're not using these in your examples. And I'm willing to bet you're not in your real code. But coroutine-style code often uses the value of a yield from expression—see PEP 3156 for examples. Such code usually depends on other features of Python 3.3 generators—in particular, the new StopIteration.value from the same PEP 380 that introduced yield from—so it will have to be rewritten. But if not, you can use the PEP also shows you the complete horrid messy equivalent, and you can of course pare down the parts you don't care about. And if you don't use the value of the expression, it pares down to the two lines above.

** Not a huge one, and there's nothing you can do about it short of using Python 3.3 or completely restructuring your code. It's exactly the same case as translating list comprehensions to Python 1.5 loops, or any other case when there's a new optimization in version X.Y and you need to use an older version.

Answer 4

I just came across this issue and my usage was a bit more difficult since I needed the return value of yield from:

result = yield from other_gen()

This cannot be represented as a simple for loop but can be reproduced with this:

_iter = iter(other_gen())
try:
    while True: #broken by StopIteration
        yield next(_iter)
except StopIteration as e:
    if e.args:
        result = e.args[0]
    else:
        result = None

Hopefully this will help people who come across the same problem. :)

Answer 5

I've found using resource contexts (using the python-resources module) to be an elegant mechanism for implementing subgenerators in Python 2.7. Conveniently I'd already been using the resource contexts anyway.

If in Python 3.3 you would have:

@resources.register_func
def get_a_thing(type_of_thing):
    if type_of_thing is "A":
        yield from complicated_logic_for_handling_a()
    else:
        yield from complicated_logic_for_handling_b()

def complicated_logic_for_handling_a():
    a = expensive_setup_for_a()
    yield a
    expensive_tear_down_for_a()

def complicated_logic_for_handling_b():
    b = expensive_setup_for_b()
    yield b
    expensive_tear_down_for_b()

In Python 2.7 you would have:

@resources.register_func
def get_a_thing(type_of_thing):
    if type_of_thing is "A":
        with resources.complicated_logic_for_handling_a_ctx() as a:
            yield a
    else:
        with resources.complicated_logic_for_handling_b_ctx() as b:
            yield b

@resources.register_func
def complicated_logic_for_handling_a():
    a = expensive_setup_for_a()
    yield a
    expensive_tear_down_for_a()

@resources.register_func
def complicated_logic_for_handling_b():
    b = expensive_setup_for_b()
    yield b
    expensive_tear_down_for_b()

Note how the complicated-logic operations only require the registration as a resource.

Answer 6

I think I found a way to emulate Python 3.x yield from construct in Python 2.x. It's not efficient and it is a little hacky, but here it is:

import types

def inline_generators(fn):
    def inline(value):
        if isinstance(value, InlineGenerator):
            for x in value.wrapped:
                for y in inline(x):
                    yield y
        else:
            yield value
    def wrapped(*args, **kwargs):
        result = fn(*args, **kwargs)
        if isinstance(result, types.GeneratorType):
            result = inline(_from(result))
        return result
    return wrapped

class InlineGenerator(object):
    def __init__(self, wrapped):
        self.wrapped = wrapped

def _from(value):
    assert isinstance(value, types.GeneratorType)
    return InlineGenerator(value)

Usage:

@inline_generators
def outer(x):
    def inner_inner(x):
        for x in range(1, x + 1):
            yield x
    def inner(x):
        for x in range(1, x + 1):
            yield _from(inner_inner(x))
    for x in range(1, x + 1):
        yield _from(inner(x))

for x in outer(3):
    print x,

Produces output:

1 1 1 2 1 1 2 1 2 3

Maybe someone finds this helpful.

Known issues: Lacks support for send() and various corner cases described in PEP 380. These could be added and I will edit my entry once I get it working.

Answer 7

Replace them with for-loops:

yield from range(L[start] + 1, L[end])

==>

for i in range(L[start] + 1, L[end]):
    yield i

The same about elements:

yield from missing_elements(L, index, end)

==>

for el in missing_elements(L, index, end):
    yield el