Why can I perform an HTTP get request to the same flask app once but not two or more times?

Question

The following Python snippet is a simple flask app with debugging on and two workers. It takes a sequence of ports as routes and performs an HTTP request using urllib2 to the first port given in the path, and passes along the rest of the ports as an argument.

For instance if my application is serving on port 4000, then

curl localhost:4000/4001

will issue an http request to localhost:4001 with path /

For whatever reason, I can only do exactly one hop even if I have multiple flasks running.

PORT=4000 python flasky.py
PORT=4001 python flasky.py
PORT=4002 python flasky.py

If I curl localhost:4000/, I get hi. If I curl localhost:4000/4000 or curl localhost:4000/4001, then everything is fine as well. However, I can't have three "hops" or I'll get an URLError: <urlopen error [Errno 111] Connection refused>, even if all the ports involved are different.

curl localhost:4000/4001/4002

fails, for instance

from flask import Flask
import os
import urllib2
import validators

app = Flask(__name__)

def make_url(host, port, path):
    "make and validate a url, hardcoded http protocol"
    assert isinstance(host, str)
    assert isinstance(port, str) or isinstance(port, int)
    assert isinstance(path, str)

    port = str(port)

    candidate_url = "http://%s:%s%s" % (host, port, path)
    if validators.url(candidate_url):
        return candidate_url
    else:
        raise ValueError(candidate_url, "is not a URL")

# handle an incoming list of ports
# use the first segment as the port and
# then send it back
@app.route('/', defaults={'path': '/'})
@app.route('/<path:path>')
def handle(path):
    # base case!
    if path == '/':
        return "hi"

    segments = path.split('/')
    # reject the empty segments
    segments = [x for x in segments if x != '']
    for x in range(len(segments)):
        segments[x] = int(segments[x])

    downstream_path = "/".join(str(x) for x in segments[1:])
    if downstream_path == '':
        downstream_path = '/'

    # verify that the downstream path is shorter than the incoming path
    assert len(downstream_path) < path

    response = urllib2.urlopen(
        make_url(
            host='127.0.0.1',
            port=segments[0],
            path=downstream_path))
    html = response.read()

    return "hi " + html

if __name__ == "__main__":
    app.run(
        host='127.0.0.1',
        port=int(os.environ["PORT"]),
        processes=2,
        debug=True)

And here is the full stack trace resulting from curl localhost:4000/4000/4000:

Traceback (most recent call last):
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1836, in __call__
    return self.wsgi_app(environ, start_response)
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1820, in wsgi_app
    response = self.make_response(self.handle_exception(e))
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1403, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1817, in wsgi_app
    response = self.full_dispatch_request()
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1477, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1381, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1475, in full_dispatch_request
    rv = self.dispatch_request()
  File "<HOME>/Workspace/python/venv/lib/python2.7/site-packages/flask/app.py", line 1461, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "<HOME>/Workspace/python/flasky.py", line 53, in handle
    path=downstream_path))
  File "/usr/lib/python2.7/urllib2.py", line 154, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/lib/python2.7/urllib2.py", line 431, in open
    response = self._open(req, data)
  File "/usr/lib/python2.7/urllib2.py", line 449, in _open
    '_open', req)
  File "/usr/lib/python2.7/urllib2.py", line 409, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.7/urllib2.py", line 1227, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.7/urllib2.py", line 1197, in do_open
    raise URLError(err)
URLError: <urlopen error [Errno 111] Connection refused>

Answer 1

The clue is in this line of the Traceback:

      File "<HOME>/Workspace/python/flasky.py", line 53, in handle
        path=downstream_path))

This means the downstream_path you are passing as path to the make_url() is not a proper path.

The bug is because "/".join() does not add the leading / for the path in the URL. The request curl localhost:4000/4001 succeeds because there is no path to add to the request localhost:4001.

The validator does not realise the error because you used validators.url() instead of validators.url.url().

Here is the corrected version of your code(i have commented on the changes):

    from flask import Flask
    import os
    import urllib2
    import validators

    app = Flask(__name__)

    def make_url(host, port, path):
        "make and validate a url, hardcoded http protocol"
        assert isinstance(host, str)
        assert isinstance(port, str) or isinstance(port, int)
        assert isinstance(path, str)

        port = str(port)

        #Add a Forward slash between the PORT and PATH 
        candidate_url = "http://%s:%s/%s" % (host, port, path)
        #use the correct validator function here.
        if validators.url.url(candidate_url):
            return candidate_url
        else:
            raise ValueError(candidate_url, "is not a URL")

    # handle an incoming list of ports
    # use the first segment as the port and
    # then send it back
    @app.route('/', defaults={'path': '/'})
    @app.route('/<path:path>')
    def handle(path):
        # base case!
        if path == '/':
            return "hi"

        segments = path.split('/')
        # reject the empty segments
        segments = [x for x in segments if x != '']
        for x in range(len(segments)):
            segments[x] = int(segments[x])

        downstream_path = "/".join(str(x) for x in segments[1:])

        #this if statement is not required since we are explicitly
        #adding forward slash in make_url function. 
        #if downstream_path == '':
            #downstream_path = '/'

        # verify that the downstream path is shorter than the incoming path
        assert len(downstream_path) < path

        response = urllib2.urlopen(
            make_url(
                host='127.0.0.1',
                port=segments[0],
                path=downstream_path))
        html = response.read()

        return "hi " + html

    if __name__ == "__main__":
        app.run(
            host='127.0.0.1',
            port=int(os.environ["PORT"]),
            processes=2,
            debug=True)

Note: use urlparse library to create URL to avoid error.