how to perform https requests over an SSLsocket in python

Question

I am using the following code to perform and ssl handshake and certificate validation with an ssl server.

import ssl
import socket

s = socket.socket()
print "connecting..."
#logging.debug("Connecting")
# Connect with SSL mutual authentication
# We only trust our server's CA, and it only trusts user certificates signed by it
c = ssl.wrap_socket(s, cert_reqs=ssl.CERT_REQUIRED,
                    ssl_version=ssl.PROTOCOL_SSLv3, ca_certs='ca.crt',
                    certfile='user.crt', keyfile='user.key')
c.connect((constants.server_addr, constants.port))

I am able to get a connection to the server and the certificate is validated correctly, however, I am not sure what to do from here. I need to perform https actions over the socket, including posting XML to a REST API. How do I go about this?

Answer 1

That's exactly what I do in my project. Here's a REST client module I used in my project. It's been modified to suit my needs, but I think you might find it useful as well. It requires httplib2: http://pypi.python.org/pypi/httplib2

"""
    client.py
    ---------

    Modified to allow validation server's certificate with external cacert list.
    -- Arif Widi Nugroho <arif@sainsmograf.com>

    Copyright (C) 2008 Benjamin O'Steen

    This file is part of python-fedoracommons.

    python-fedoracommons is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    python-fedoracommons is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with python-fedoracommons.  If not, see <http://www.gnu.org/licenses/>.
"""

__license__ = 'GPL http://www.gnu.org/licenses/gpl.txt'
__author__ = "Benjamin O'Steen <bosteen@gmail.com>, Arif Widi Nugroho <arif@sainsmograf.com>"
__version__ = '0.1'

import httplib2
import urlparse
import urllib
import base64
from base64 import encodestring

from mime_types import *

import mimetypes

from cStringIO import StringIO

class Connection(object):
    def __init__(self, base_url, username=None, password=None, cache=None, ca_certs=None, user_agent_name=None):
        self.base_url = base_url
        self.username = username
        m = MimeTypes()
        self.mimetypes = m.get_dictionary()

        self.url = urlparse.urlparse(base_url)

        (scheme, netloc, path, query, fragment) = urlparse.urlsplit(base_url)

        self.scheme = scheme
        self.host = netloc
        self.path = path

        if user_agent_name is None:
            self.user_agent_name = 'Basic Agent'
        else:
            self.user_agent_name = user_agent_name

        # Create Http class with support for Digest HTTP Authentication, if necessary
        # self.h = httplib2.Http(".cache")
        self.h = httplib2.Http(cache=cache, ca_certs=ca_certs)
        self.h.follow_all_redirects = True
        if username and password:
            self.h.add_credentials(username, password)

    def request_get(self, resource, args = None, headers={}):
        return self.request(resource, "get", args, headers=headers)

    def request_delete(self, resource, args = None, headers={}):
        return self.request(resource, "delete", args, headers=headers)

    def request_head(self, resource, args = None, headers={}):
        return self.request(resource, "head", args, headers=headers)

    def request_post(self, resource, args = None, body = None, filename=None, headers={}):
        return self.request(resource, "post", args , body = body, filename=filename, headers=headers)

    def request_put(self, resource, args = None, body = None, filename=None, headers={}):
        return self.request(resource, "put", args , body = body, filename=filename, headers=headers)

    def get_content_type(self, filename):
        extension = filename.split('.')[-1]
        guessed_mimetype = self.mimetypes.get(extension, mimetypes.guess_type(filename)[0])
        return guessed_mimetype or 'application/octet-stream'

    def request(self, resource, method = "get", args = None, body = None, filename=None, headers={}):
        params = None
        path = resource
        headers['User-Agent'] = self.user_agent_name

        BOUNDARY = u'00hoYUXOnLD5RQ8SKGYVgLLt64jejnMwtO7q8XE1'
        CRLF = u'\r\n'

        if filename and body:
            #fn = open(filename ,'r')
            #chunks = fn.read()
            #fn.close()

            # Attempt to find the Mimetype
            content_type = self.get_content_type(filename)
            headers['Content-Type']='multipart/form-data; boundary='+BOUNDARY
            encode_string = StringIO()
            encode_string.write(CRLF)
            encode_string.write(u'--' + BOUNDARY + CRLF)
            encode_string.write(u'Content-Disposition: form-data; name="file"; filename="%s"' % filename)
            encode_string.write(CRLF)
            encode_string.write(u'Content-Type: %s' % content_type + CRLF)
            encode_string.write(CRLF)
            encode_string.write(body)
            encode_string.write(CRLF)
            encode_string.write(u'--' + BOUNDARY + u'--' + CRLF)

            body = encode_string.getvalue()
            headers['Content-Length'] = str(len(body))
        elif body:
            if not headers.get('Content-Type', None):
                headers['Content-Type']='text/xml'
            headers['Content-Length'] = str(len(body))        
        else: 
            headers['Content-Type']='text/xml'

        if method.upper() == 'POST':
            headers['Content-Type']='application/x-www-form-urlencoded'

        if args:
            path += u"?" + urllib.urlencode(args)

        request_path = []
        if self.path != "/":
            if self.path.endswith('/'):
                request_path.append(self.path[:-1])
            else:
                request_path.append(self.path)
            if path.startswith('/'):
                request_path.append(path[1:])
            else:
                request_path.append(path)

        resp, content = self.h.request(u"%s://%s%s" % (self.scheme, self.host, u'/'.join(request_path)), method.upper(), body=body, headers=headers )

        return {u'headers':resp, u'body':content.decode('UTF-8')}

Sample usage (connection will be failed if the server certificate is not signed by specified ca):

c = client.Connection('https://localhost:8000', certs='/path/to/cacert.pem')
# now post some data to the server
response = c.request_post('rest/path/', body=some_urlencoded_data)
if response['headers']['status'] == '200':
    # do something...

Answer 2

You can use your wrap_socket code to extend httplib.HTTPConnection, as described in this answer.

(I'd still consider using something like PycURL, as I've already answered in your previous question.)

Answer 3

You may want to start at urllib2.urlopen: http://docs.python.org/library/urllib2.html#urllib2.urlopen

That can take care of https URLs, fetching, POSTing, etc. You don't need to work directly on a low-level socket or ssl object. If you're using Python 2.x, though, HTTPS connections won't do any verification of the server-side cert, which it looks like you need (and that's good). Python 3's urllib does do that, though.

If you're using Python 2, you have a few options. One is to subclass urllib2.HTTPSHandler so that it does the appropriate verification on its socket. Another is to implement the HTTP protocol bits that you need by yourself (not recommended). You could also instantiate various urllib2 and httplib objects normally, then simply assign your already-authenticated ssl socket in place of the ones they're using, although you'd need to be really careful that their state doesn't get messed up. The source code in the standard library is very readable, though, in case you need to do tinkering like this.