How to log HttpClient requests, response including body?

Question

Jersey provides an excellent client-side logging facility that looks like this:

INFO: 1 * Server has received a request on thread grizzly-http-server-0
1 > GET http://localhost:9998/helloworld
1 > accept: text/plain
1 > accept-encoding: gzip,deflate
1 > connection: Keep-Alive
1 > host: localhost:9998
1 > user-agent: Jersey/3.0-SNAPSHOT (Apache HttpClient 4.5)

INFO: 1 * Server responded with a response on thread grizzly-http-server-0
1 < 200
1 < Content-Type: text/plain
Hello World!

We see the worker thread, request method, URI, headers and the response code, headers, and body.

Is there an equivalent functionality for Jetty? Or do we need to roll our own?

Answer 1

I ended up implementing this myself, with slight variations on the Jetty format:

import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.http.HttpField;
import org.eclipse.jetty.http.HttpFields;
import org.eclipse.jetty.http.HttpHeader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicLong;

import static org.bitbucket.cowwoc.requirements.core.Requirements.requireThat;

/**
 * Logs HttpClient request/response traffic.
 *
 * @author Gili Tzabari
 */
public final class RequestLogger
{
    /**
     * {@code true} if request values should be grouped together, and response values should be grouped together to improve readability.
     * {@code false} if events should be logged as soon as they arrive (useful when diagnosing performance problems).
     */
    private static final boolean IMPROVED_READABILITY = true;
    private final AtomicLong nextId = new AtomicLong();
    private final Logger log = LoggerFactory.getLogger(RequestLogger.class);

    /**
     * Attaches event listeners to a request.
     *
     * @param request the request to listen to
     * @return the request
     * @throws NullPointerException if {@code request} is null
     */
    public Request listenTo(Request request)
    {
        requireThat("request", request).isNotNull();
        if (IMPROVED_READABILITY)
            improvedReadability(request);
        else
            correctTiming(request);
        return request;
    }

    /**
     * Group events for improved readability.
     *
     * @param request the request to listen to
     */
    private void improvedReadability(Request request)
    {
        long id = nextId.getAndIncrement();
        StringBuilder group = new StringBuilder();
        request.onRequestBegin(theRequest -> group.append("Request " + id + "\n" +
            id + " > " + theRequest.getMethod() + " " + theRequest.getURI() + "\n"));
        request.onRequestHeaders(theRequest ->
        {
            for (HttpField header : theRequest.getHeaders())
                group.append(id + " > " + header + "\n");
        });

        StringBuilder contentBuffer = new StringBuilder();
        request.onRequestContent((theRequest, content) ->
            contentBuffer.append(ByteBuffers.toString(content, getCharset(theRequest.getHeaders()))));
        request.onRequestSuccess(theRequest ->
        {
            if (contentBuffer.length() > 0)
            {
                group.append("\n" +
                    contentBuffer.toString());
            }
            log.debug(group.toString());
            contentBuffer.delete(0, contentBuffer.length());
            group.delete(0, group.length());
        });

        request.onResponseBegin(theResponse ->
        {
            group.append("Response " + id + "\n" +
                id + " < " + theResponse.getVersion() + " " + theResponse.getStatus());
            if (theResponse.getReason() != null)
                group.append(" " + theResponse.getReason());
            group.append("\n");
        });
        request.onResponseHeaders(theResponse ->
        {
            for (HttpField header : theResponse.getHeaders())
                group.append(id + " < " + header + "\n");
        });
        request.onResponseContent((theResponse, content) ->
            contentBuffer.append(ByteBuffers.toString(content, getCharset(theResponse.getHeaders()))));
        request.onResponseSuccess(theResponse ->
        {
            if (contentBuffer.length() > 0)
            {
                group.append("\n" +
                    contentBuffer.toString());
            }
            log.debug(group.toString());
        });
    }

    /**
     * Log events as they come in.
     *
     * @param request the request to listen to
     */
    private void correctTiming(Request request)
    {
        long id = nextId.getAndIncrement();
        request.onRequestBegin(theRequest -> log.debug(id + " > " + theRequest.getMethod() + " " + theRequest.getURI()));
        request.onRequestHeaders(theRequest ->
        {
            for (HttpField header : theRequest.getHeaders())
                log.debug(id + " > " + header);
        });
        request.onRequestContent((theRequest, content) -> log.debug(id + " >> " +
            ByteBuffers.toString(content, getCharset(theRequest.getHeaders()))));

        request.onResponseBegin(theResponse ->
        {
            StringBuilder line = new StringBuilder(id + " < " + theResponse.getVersion() + " " + theResponse.getStatus());
            if (theResponse.getReason() != null)
                line.append(" " + theResponse.getReason());
            log.debug(line.toString());
        });
        request.onResponseHeaders(theResponse ->
        {
            for (HttpField header : theResponse.getHeaders())
                log.debug(id + " < " + header);
        });
        StringBuilder responseBody = new StringBuilder();
        request.onResponseContent((theResponse, content) ->
            responseBody.append(ByteBuffers.toString(content, getCharset(theResponse.getHeaders()))));
        request.onResponseSuccess(theResponse -> log.debug(id + " << " + responseBody));
    }

    /**
     * @param headers HTTP headers
     * @return the charset associated with the request or response body
     */
    private Charset getCharset(HttpFields headers)
    {
        String contentType = headers.get(HttpHeader.CONTENT_TYPE);
        if (contentType == null)
            return StandardCharsets.UTF_8;
        String[] tokens = contentType.toLowerCase(Locale.US).split("charset=");
        if (tokens.length != 2)
            return StandardCharsets.UTF_8;
        // Remove semicolons or quotes
        String encoding = tokens[1].replaceAll("[;\"]", "");
        return Charset.forName(encoding);
    }
}

---

import java.nio.ByteBuffer;
import java.nio.charset.Charset;

import static org.bitbucket.cowwoc.requirements.core.Requirements.requireThat;

/**
 * ByteBuffer helper functions.
 *
 * @author Gili Tzabari
 */
public final class ByteBuffers
{
    /**
     * @param buffer  a {@Code ByteBuffer}
     * @param charset the character set of the bytes
     * @return the {@code String} representation of the bytes
     */
    public static String toString(ByteBuffer buffer, Charset charset)
    {
        requireThat("buffer", buffer).isNotNull();
        byte[] bytes;
        if (buffer.hasArray())
            bytes = buffer.array();
        else
        {
            bytes = new byte[buffer.remaining()];
            buffer.get(bytes, 0, bytes.length);
        }
        return new String(bytes, charset);
    }

    /**
     * Prevent construction.
     */
    private ByteBuffers()
    {
    }
}

Usage:

Request request = new HttpClient().newRequest("http://www.test.com/");
new RequestLogger().listenTo(request).send();