How to know if a paramiko SSH channel is disconnected?

Question

I'm desinging test cases in which I use paramiko for SSH connections. Test cases usually contain paramiko.exec_command() calls which I have a wrapper for (called run_command()). Here self.ssh is an intance of paramiko.SSHClient(). I use a decorator to check the ssh connection before each call. (self.get_ssh() negotiates the connection)

def check_connections(function):
    ''' A decorator to check SSH connections. '''
    def deco(self, *args, **kwargs):
        if self.ssh is None:
            self.ssh = self.get_ssh()
        else:
            ret = getattr(self.ssh.get_transport(), 'is_active', None)
            if ret is None or (ret is not None and not ret()):
                self.ssh = self.get_ssh()
        return function(self, *args, **kwargs)
    return deco

@check_connections
def run_command(self, command):
    ''' Executes command via SSH. '''
    stdin, stdout, stderr = self.ssh.exec_command(command)
    stdin.flush()
    stdin.channel.shutdown_write()
    ret = stdout.read()
    err = stderr.read()
    if ret:
        return ret
    elif err:
        return err
    else:
        return None

It works perfectly until my remote node reboots, which can happen sometimes. When it occurs the next run_command() call generates a socket.error exception. The problem is, that the paramiko.Transport object seems to be remain in active state until an exception is thrown:

Python 2.7.3 (default, Mar  7 2013, 14:03:36)
[GCC 4.3.4 [gcc-4_3-branch revision 152973]] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> import paramiko
>>> ssh = paramiko.SSHClient()
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
None
>>> ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
>>> ssh.load_host_keys(os.path.expanduser('~') + '/.ssh/known_hosts')
>>> ssh.connect(hostname = '172.31.77.57', username = 'root', password = 'rootroot', timeout = 5.0)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('ls')
(<paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('reboot')
(<paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('ls')
No handlers could be found for logger "paramiko.transport"
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/pytest/lib/python2.7/site-packages/paramiko/client.py", line 370, in exec_command
    chan = self._transport.open_session()
  File "/home/pytest/lib/python2.7/site-packages/paramiko/transport.py", line 662, in open_session
    return self.open_channel('session')
  File "/home/pytest/lib/python2.7/site-packages/paramiko/transport.py", line 764, in open_channel
    raise e
socket.error: [Errno 104] Connection reset by peer
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (unconnected)>
>>> print ssh.get_transport().is_active()
False
>>>

Question: how can I be sure that the connection is really active or not?

Answer 1

I´ll just throw this here since someone might find it usefull. There is one catch with some of these methods. Paramiko internally uses sockets. Every new connection calls socket which opens a new file descriptor. Since processes are limited to certain number of open file descriptors after some time you will run out which will result in:

socket.error: [Errno 24] Too many open files.

So it is better to explicitly try to close the connection before establishing a new one using SSHClient.close() method.

Answer 2

This works:

import paramiko
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())        # Setting the missing host policy to auto add it
client.connect('192.168.1.16', port=22, username='admin', password='admin', timeout=3, banner_timeout=2)

channel = client.invoke_shell()                 # Request an interactive shell session on this channel. If the server allows it, the channel will then be directly connected to the stdin, stdout, and stderr of the shell.
print channel.closed          # False
command = 'reboot'
channel.send(command + '\n')
# wait a while
print channel.closed          # True

Answer 3

My solution basically the same as yours, just organized differently:

def connection(self):
    if not self.is_connected():
        self._ssh = paramiko.SSHClient()
        self._ssh.connect(self.server, self.port,
                          username = self.username, password = self.password)

    return self._ssh

def is_connected(self):
    transport = self._ssh.get_transport() if self._ssh else None
    return transport and transport.is_active()

def do_something(self):
    self.connection().exec_command('ls')

Answer 4

In python, it's easier to ask for forgiveness than permission.

Wrap each call to ssh.exec_command like so:

try:
    ssh.exec_command('ls')
except socket.error as e:
    # Crap, it's closed. Perhaps reopen and retry?