twisted get body of POST request

Question

Ok,

This should be simple, since people do it all the time. I want to get the body of a POST request sent a twisted Agent. This is created with a twisted FileBodyProducer. On the server side, I get a request object for my render_POST method.

How do I retrieve the body?

server:

from twisted.web import server, resource
from twisted.internet import reactor


class Simple(resource.Resource):
    isLeaf = True
    def render_GET(self, request):
        return "{0}".format(request.args.keys())
    def render_POST(self, request):
        return "{0}".format(request.data)
        with open(request.args['filename'][0], 'rb') as fd:
            fd.write(request.write())

site = server.Site(Simple())
reactor.listenTCP(8080, site)
reactor.run()

client:

from StringIO import StringIO

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from pprint import pformat

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.remaining = 1024 * 10

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            print 'Some data received:'
            print display
            self.remaining -= len(display)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.finished.callback(None)

agent = Agent(reactor)
body = FileBodyProducer(StringIO("hello, world"))
d = agent.request(
    'POST',
    'http://127.0.0.1:8080/',
    Headers({'User-Agent': ['Twisted Web Client Example'],
             'Content-Type': ['text/x-greeting']}),
    body)

def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))
    finished = Deferred()
    response.deliverBody(BeginningPrinter(finished))
    return finished
d.addCallback(cbRequest)

def cbShutdown(ignored):
    reactor.stop()
d.addBoth(cbShutdown)

reactor.run()

The only docs I can find for setting up the consumer side leave something to be desired. Primarily, how can a consumer use the write(data) method to receive results?

Which bit am I missing to plug these two components together?

Answer 1

if you want to make a simple POST with body (not a file) you can do as follows

import urllib
from twisted.internet import protocol
from twisted.internet import defer
from twisted.web.http_headers import Headers
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.iweb import IBodyProducer
from zope.interface import implements
from twisted.internet.defer import succeed

class StringProducer(object):
    implements(IBodyProducer)

    def __init__(self, body):
        self.body = body
        self.length = len(body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return succeed(None)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass

class SimpleReceiver(protocol.Protocol):
    def __init__(self, d):
        self.buf = ''; self.d = d

    def dataReceived(self, data):
        self.buf += data

    def connectionLost(self, reason):
        self.d.callback(self.buf)

def httpRequest(url, values=None, headers=None, method='POST'):

    agent = Agent(reactor)
    data = urllib.urlencode(values) if values else None

    d = agent.request(method, url, Headers(headers) if headers else {},
        StringProducer(data) if data else None
        )

    def handle_response(response):
        if response.code == 204:
            d = defer.succeed('')
        else:
            d = defer.Deferred()
            response.deliverBody(SimpleReceiver(d))
        return d

    d.addCallback(handle_response)
    return d

Now to use above in real code you can i.e.

d = httpRequest('htpp://...', post_data_as_dictionary, some_headers, 'POST')
d.addCallback(your_ok_callback_function)
d.addErrback(your_errorback_function)

Example headers should look like

headers = {'Accept' : ['application/json',],
            'Content-Type': ['application/x-www-form-urlencoded',] 
}

I hope that helps

Answer 2

If the content type is application/x-www-form-urlencoded or multipart/form-data, the body will be parsed and put in the request.args dict.

If the body is too big, it is written in temp file, otherwise in StringIO.

After the body is read, the method finish() is called. You can subclass Request and pares the body in this method or do sth else.

Answer 3

All right, so it's as simple as calling request.content.read(). This, as far as I can tell, is undocumented in the API.

Here's the updated code for the client:

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from pprint import pformat

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.remaining = 1024 * 10

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            print 'Some data received:'
            print display
            self.remaining -= len(display)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.finished.callback(None)

class SaveContents(Protocol):
    def __init__(self, finished, filesize, filename):
        self.finished = finished
        self.remaining = filesize
        self.outfile = open(filename, 'wb')

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            self.outfile.write(display)
            self.remaining -= len(display)
        else:
            self.outfile.close()

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.outfile.close()
        self.finished.callback(None)

agent = Agent(reactor)
f = open('70935-new_barcode.pdf', 'rb')
body = FileBodyProducer(f)
d = agent.request(
    'POST',
    'http://127.0.0.1:8080?filename=test.pdf',
    Headers({'User-Agent': ['Twisted Web Client Example'],
             'Content-Type': ['multipart/form-data; boundary=1024'.format()]}),
    body)

def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print 'Response length:', response.length
    print pformat(list(response.headers.getAllRawHeaders()))
    finished = Deferred()
    response.deliverBody(SaveContents(finished, response.length, 'test2.pdf'))
    return finished
d.addCallback(cbRequest)

def cbShutdown(ignored):
    reactor.stop()
d.addBoth(cbShutdown)

reactor.run()

And here's the server:

from twisted.web import server, resource
from twisted.internet import reactor
import os

# multi part encoding example: http://marianoiglesias.com.ar/python/file-uploading-with-multi-part-encoding-using-twisted/
class Simple(resource.Resource):
    isLeaf = True
    def render_GET(self, request):
        return "{0}".format(request.args.keys())
    def render_POST(self, request):
        with open(request.args['filename'][0], 'wb') as fd:
            fd.write(request.content.read())
        request.setHeader('Content-Length', os.stat(request.args['filename'][0]).st_size)
        with open(request.args['filename'][0], 'rb') as fd:
            request.write(fd.read())
        request.finish()
        return server.NOT_DONE_YET

site = server.Site(Simple())
reactor.listenTCP(8080, site)
reactor.run()

I can now write the file contents I receive, and read back the results.