Tornado gen.sleep add delay

Question

I'm trying to add a delay between requests in an asynchronous way. When I use Tornado gen.sleep(x) my function (launch) doesn't get executed. If I remove yield from yield gen.sleep(1.0), function is called, but no delay is added. How to add delay between requests in my for loop? I need to control Request per second to external API. If I use time.sleep the response is delayed after all requests are completed. Tried to add @gen.engine decorator to launch function and no results.

Code:

import collections
import tornado.httpclient


class BacklogClient(object):
    MAX_CONCURRENT_REQUESTS = 20

    def __init__(self, ioloop):
        self.ioloop = ioloop
        self.client = tornado.httpclient.AsyncHTTPClient(max_clients=self.MAX_CONCURRENT_REQUESTS)
        self.client.configure(None, defaults=dict(connect_timeout=20, request_timeout=30))
        self.backlog = collections.deque()
        self.concurrent_requests = 0

    def __get_callback(self, function):
        def wrapped(*args, **kwargs):
            self.concurrent_requests -= 1
            self.try_run_request()
            return function(*args, **kwargs)

        return wrapped

    def try_run_request(self):
        while self.backlog and self.concurrent_requests < self.MAX_CONCURRENT_REQUESTS:
            request, callback = self.backlog.popleft()
            self.client.fetch(request, callback=callback)
            self.concurrent_requests += 1

    def fetch(self, request, callback=None):
        wrapped = self.__get_callback(callback)

        self.backlog.append((request, wrapped))
        self.try_run_request()


import time
from tornado import ioloop, httpclient, gen


class TornadoBacklog:
    def __init__(self):

        self.queue = 0
        self.debug = 1
        self.toProcess = [
            'http://google.com',
            'http://yahoo.com',
            'http://nytimes.com',
            'http://msn.com',
            'http://cnn.com',
            'http://twitter.com',
            'http://facebook.com',
        ]


    def handle_request(self, response):

        print response.code
        if not self.backlog.backlog and self.backlog.concurrent_requests == 0:
            ioloop.IOLoop.instance().stop()


    def launch(self):
        self.ioloop = ioloop.IOLoop.current()
        self.backlog = BacklogClient(self.ioloop)

        for item in self.toProcess:
            yield gen.sleep(1.0)
            print item
            self.backlog.fetch(
                httpclient.HTTPRequest(
                    item,
                    method='GET',
                    headers=None,
                ),
                self.handle_request
            )

        self.ioloop.start()



def main():
    start_time = time.time()

    scraper = TornadoBacklog()
    scraper.launch()

    elapsed_time = time.time() - start_time
    print('Process took %f seconds processed %d items.' % (elapsed_time, len(scraper.toProcess)))


if __name__ == "__main__":
    main()

Reference: https://github.com/tornadoweb/tornado/issues/1400

Answer 1

Tornado coroutines have two components:

1. They contain "yield" statements
2. They are decorated with "gen.coroutine"

Use the "coroutine" decorator on your "launch" function:

@gen.coroutine
def launch(self):

Run a Tornado coroutine from start to finish like this:

tornado.ioloop.IOLoop.current().run_sync(launch)

Remove the call to "ioloop.start" from your "launch" function: the loop runs the "launch" function, not vice-versa.