http streaming in both directions

Question

Is the following scenario valid for HTTP?

1. Client sends HTTP header to Server
2. Server receives header and sends HTTP response header
3. Client streams HTTP body (chunked transfer)
4. Server receives request body chunks and sends HTTP response body chunks (again chunked transfer)

I tried implementing this using HttpWebRequest and Asp.Net Web Api but got this error on the client

An unhandled exception of type 'System.NotSupportedException' occurred in System.dll

Additional information: The stream does not support concurrent IO read or write operations

Client

static void Main(string[] args)
{
      HttpWebRequest request = WebRequest.Create("http://localhost.fiddler:16462/") as HttpWebRequest;
      request.SendChunked = true;
      request.ContentType = "application/octet-stream";
      request.Method = "POST";
      request.AllowWriteStreamBuffering = false;
      request.AllowReadStreamBuffering = false;
      request.Timeout = 3600000;

      var requestStream = request.GetRequestStream();
      var responseStream = request.GetResponse().GetResponseStream();
      int bufLength = 10 * 1024;

      byte[] requestBuffer = new byte[bufLength];
      byte[] responseBuffer = new byte[bufLength];

      for (int i = 0; i < 1024; ++i)
      {
          requestStream.Write(requestBuffer, 0, bufLength);
          responseStream.Read(responseBuffer, 0, bufLength);
      }

      requestStream.Close();
      responseStream.Close();
}

I verified that TcpClient does support simultaneous request response streaming. However, it would be nice to see HttpWebRequest also support this.

using System;
using System.Net.Sockets;
using System.Text;
using System.Threading;

namespace TcpClientTest
{
    class Program
    {
        static void Main(string[] args)
        {
            TcpClient client = new TcpClient("localhost", 16462);
            var stream = client.GetStream();

            byte[] buffer = Encoding.UTF8.GetBytes("POST http://localhost:16462/ HTTP/1.1\r\n");
            stream.Write(buffer, 0, buffer.Length);

            buffer = Encoding.UTF8.GetBytes("Content-Type: application/octet-stream\r\n");
            stream.Write(buffer, 0, buffer.Length);

            buffer = Encoding.UTF8.GetBytes("Host: localhost:16462\r\n");
            stream.Write(buffer, 0, buffer.Length);

            buffer = Encoding.UTF8.GetBytes("Transfer-Encoding: chunked\r\n\r\n");
            stream.Write(buffer, 0, buffer.Length);

            int chunkLen = 128 * 1024;
            string chunkSizeStr = chunkLen.ToString("X");
            byte[] chunkSizeBytes = Encoding.UTF8.GetBytes(chunkSizeStr + "\r\n");
            buffer = new byte[chunkLen];
            for (int i = 0; i < chunkLen; ++i)
            {
                buffer[i] = (byte)'a';
            }

            // Start reader thread
            var reader = new Thread(() =>
            {
                byte[] response = new byte[128 * 1024];
                int bytesRead = 0;
                while ((bytesRead = stream.Read(response, 0, response.Length)) > 0)
                {
                    Console.WriteLine("Read {0} bytes", bytesRead);
                }
            });

            reader.Start();

            // Streaming chunks
            for (int i = 0; i < 1024 * 1024; ++i)
            {
                stream.Write(chunkSizeBytes, 0, chunkSizeBytes.Length);
                stream.Write(buffer, 0, buffer.Length);
                stream.Write(Encoding.UTF8.GetBytes("\r\n"), 0, 2);
            }

            buffer = Encoding.UTF8.GetBytes("0\r\n\r\n");
            stream.Write(buffer, 0, buffer.Length);
            reader.Join();
        }
    }
}

Answer 1

WinHttp does not support receiving an HTTP response before sending the entire request. I did a test and observed that the call to WinHttpReceiveResponse blocks if all request chunks have been not been sent. Here is the code.

// WinHttpTest.cpp : Defines the entry point for the console application.
//

#include <Windows.h>
#include <winhttp.h>
#include <stdio.h>
#include <memory>
#include <string>
#include <iostream>

#pragma comment(lib, "winhttp.lib")

#define SEND_BUFFER_SIZE 128 * 1024
#define RECV_BUFFER_SIZE 128 * 1024

int main()
{
    HINTERNET session = 0, connection = 0, request = 0;
    bool result = false;
    std::unique_ptr<char[]> sendBuffer = std::make_unique<char[]>(SEND_BUFFER_SIZE);
    std::unique_ptr<char[]> recvBuffer = std::make_unique<char[]>(RECV_BUFFER_SIZE);
    DWORD bytesTransferred = 0;
    std::string chunkHeader = "20000\r\n";
    std::string chunkFooter = "0\r\n\r\n";
    std::string newLine = "\r\n";
    int i = 0;

    session = WinHttpOpen(L"Test", WINHTTP_ACCESS_TYPE_NO_PROXY, NULL, NULL, 0);
    if (!session)
    {
        printf("WinHttpOpen failed %d\n", GetLastError());
        goto cleanup;
    }

    connection = WinHttpConnect(session, L"localhost", 16462, 0);
    if (!connection)
    {
        printf("WinHttpConnect failed %d\n", GetLastError());
        goto cleanup;
    }

    request = WinHttpOpenRequest(connection, L"POST", NULL, NULL, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, 0);
    if (!request)
    {
        printf("WinHttpOpenRequest failed %d\n", GetLastError());
        goto cleanup;
    }

    result = WinHttpSendRequest(request, L"Host: localhost:16462\r\nTransfer-Encoding: chunked\r\nContent-Type: application/octet-stream\r\n", -1L, WINHTTP_NO_REQUEST_DATA, 0, WINHTTP_IGNORE_REQUEST_TOTAL_LENGTH, 0);
    if (!result)
    {
        printf("WinHttpSendRequest failed %d\n", GetLastError());
        goto cleanup;
    }

    // Streaming chunks
    for (i = 0; i < 1024; ++i)
    {
        result = WinHttpWriteData(request, chunkHeader.data(), chunkHeader.length(), &bytesTransferred);
        if (!result)
        {
            printf("WinHttpWriteData ChunkHeader failed %d\n", GetLastError());
            goto cleanup;
        }

        result = WinHttpWriteData(request, sendBuffer.get(), SEND_BUFFER_SIZE, &bytesTransferred);
        if (!result)
        {
            printf("WinHttpWriteData failed %d\n", GetLastError());
            goto cleanup;
        }

        result = WinHttpWriteData(request, newLine.data(), newLine.length(), &bytesTransferred);
        if (!result)
        {
            printf("WinHttpWriteData NewLine failed %d\n", GetLastError());
            goto cleanup;
        }
    }

    result = WinHttpWriteData(request, chunkFooter.data(), chunkFooter.length(), &bytesTransferred);
    if (!result)
    {
        printf("WinHttpWriteData ChunkFooter failed %d\n", GetLastError());
        goto cleanup;
    }    

    result = WinHttpReceiveResponse(request, NULL);
    if (!result)
    {
        printf("WinHttpReceiveResponse failed %d\n", GetLastError());
        goto cleanup;
    }

    do
    {
        result = WinHttpReadData(request, recvBuffer.get(), RECV_BUFFER_SIZE, &bytesTransferred);
        if (!result)
        {
            printf("WinHttpReadData failed %d\n", GetLastError());
            goto cleanup;
        }
        else
        {
            printf("Received %d bytes\n", bytesTransferred);
            if (bytesTransferred < 1024)
            {
                std::string message(recvBuffer.get(), recvBuffer.get() + bytesTransferred);
                std::cout << message << std::endl;
            }
        }
    } while (bytesTransferred > 0);

    printf("All Done\n");

cleanup:
    if (request)
        WinHttpCloseHandle(request);
    if (connection)
        WinHttpCloseHandle(connection);
    if (session)
        WinHttpCloseHandle(session);
}

Interestingly, there is a blog post that mentions IIS 8 supports full-duplex reads and writes but WinHttp is still Half duplex.

IHttpContext3->EnableFullDuplex() -- This API tells the IIS pipeline to go into full duplex mode. This enables an handler to issue one read and one write operation in parallel.

WinHttp is still half duplex. However they introduced new API's to support websocket traffic, which in fact are full duplex.

Answer 2

That doesn't seem to be valid HTTP behaviour. You could make a custom "HTTP-like" client and server that start sending a response before the client finishes the request, but that seems invalid according to the HTTP spec:

6 Response

  After receiving and interpreting a request message, a server responds
  with an HTTP response message.

Web sockets might be your thing though! They allow for bidirectional streams on top of HTTP. https://en.wikipedia.org/wiki/WebSocket