C# HttpWebRequest Timed Out

Question

Using Fiddler, I mimicked all the headers that were sent by an actual browser (Chrome). Then I tried to call the function by

GetResponse("https://www.example.com/");

But get error:

An unhandled exception of type 'System.Net.WebException' occurred in System.dll

Additional information: The operation has timed out

Code:

public static string GetResponse(string sURL, CookieContainer cookies = null, string sParameters = "", string sUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
    {
        HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(sURL);
        httpRequest.UserAgent = sUserAgent;

        if (cookies == null) cookies = new CookieContainer();
        httpRequest.CookieContainer = cookies;

        System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Tls12;

        httpRequest.AllowAutoRedirect = true;
        httpRequest.KeepAlive = true;
        httpRequest.ProtocolVersion = HttpVersion.Version11;

        httpRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
        httpRequest.Headers.Add("Accept-Encoding", "gzip, deflate, br");
        httpRequest.Headers.Add("Accept-Language", "en-US,en;q=0.8");
        httpRequest.Headers.Add("upgrade-insecure-requests", "1");
        httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        httpRequest.KeepAlive = true;

        if (sParameters == "")
        {
            httpRequest.Method = "GET";
        }
        else
        {
            httpRequest.Method = "POST";
            httpRequest.ContentType = "application/x-www-form-urlencoded";
            httpRequest.ContentLength = sParameters.Length;

            using (Stream stream = httpRequest.GetRequestStream())
            {
                stream.Write(Encoding.UTF8.GetBytes(sParameters), 0, sParameters.Length);
            }

        }
                    
        HttpWebResponse httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();
        
        string sResponse;
        using (Stream stream = httpWebResponse.GetResponseStream())
        {
            StreamReader reader = new StreamReader(stream, System.Text.Encoding.GetEncoding(936));
            sResponse = reader.ReadToEnd();
        }

        return sResponse;
    }

After inspecting in Fiddler, compared to a regular browser action, the missing part is cookies, however a cookiecontainer has been attached to the httpwebrequest object in the code. So I don't know why the cookie is missing.

Thank you for your help.

Answer 1

It may look a lot of stuff, but you don't need to be concerned with what this all does right now.

Just create a Button on a Form, then make async it's Click event handler as shown here (the main methods here use all async Http/IO .Net methods).
The remaining code just needs to be pasted (in the same Form, to make it quick).

The StreamObject used to pass data back and forth, after the connection is completed will contain some informations about the WebSite specified in its ResourceURI property (as below).

The StreamObject.Payload is the complete Html page, decoded using it's internal CodePage or the CodePage detected by the server.
If you want to see it, just pass it to a WebBrowser.

Note:
I might have left out something, adapting this to be posted here. This will be immediately obvious.
In case, tell me what it is and I'll update this code.
Also:
Disable Fiddler!

Visual Studio Version: VS Pro 15.7.5
.Net FrameWork: 4.7.1


private async void TestConnection_Click(object sender, EventArgs e)
{
    StreamObject sObject = new StreamObject()
    {
        ResourceURI = new Uri(@"https://www.bestbuy.com/?intl=nosplash"),
        ProcessStream = true
    };

    sObject = await HTTP_GetStream(sObject);
    Console.WriteLine(sObject.Payload.Length);
}


using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Net.Security;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Security.Permissions;
using System.Security.Principal;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;


public class StreamObject
{
    public StreamObject()
    {
        this.Cookies = new CookieContainer();
    }

    public Stream ContentStream { get; set; }
    public bool ProcessStream { get; set; }
    public Uri ResourceURI { get; set; }
    public Uri ResponseURI { get; set; }
    public string Referer { get; set; }
    public string Payload { get; set; }
    public string ServerType { get; set; }
    public string ServerName { get; set; }
    public IPAddress[] ServerIP { get; set; }
    public string ContentName { get; set; }
    public string ContentType { get; set; }
    public string ContentCharSet { get; set; }
    public string ContentLanguage { get; set; }
    public long ContentLenght { get; set; }
    public HttpStatusCode StatusCode { get; set; }
    public string StatusDescription { get; set; }
    public WebExceptionStatus WebException { get; set; }
    public string WebExceptionDescription { get; set; }
    public CookieContainer Cookies { get; set; }
}

const uint COR_E_INVALIDOPERATION = 0x80131509;


public async Task<StreamObject> HTTP_GetStream(StreamObject RequestObject)
{
    if (string.IsNullOrEmpty(RequestObject.ResourceURI.ToString().Trim()))
        return null;

    MemoryStream memstream = new MemoryStream();
    HttpWebRequest httpRequest;
    CookieContainer CookieJar = new CookieContainer();
    HttpStatusCode StatusCode = HttpStatusCode.OK;

    ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 |
                                           SecurityProtocolType.Tls |
                                           SecurityProtocolType.Tls11 |
                                           SecurityProtocolType.Tls12;

    ServicePointManager.Expect100Continue = false;
    ServicePointManager.DefaultConnectionLimit = 10;
    ServicePointManager.ServerCertificateValidationCallback += TlsValidationCallback;

    httpRequest = WebRequest.CreateHttp(RequestObject.ResourceURI);

    try
    {
        HTTP_RequestHeadersInit(ref httpRequest, CookieJar, RequestObject);
        httpRequest.Method = "GET";
        using (HttpWebResponse httpResponse = (HttpWebResponse)await httpRequest.GetResponseAsync())
        {
            Stream ResponseStream = httpResponse.GetResponseStream();
            //SslProtocols Protocol = ExtractSslProtocol(ResponseStream);

            if (StatusCode == HttpStatusCode.OK)
            {
                await ResponseStream.CopyToAsync(memstream);
                RequestObject.ContentStream = memstream;
                RequestObject.ResponseURI = httpResponse.ResponseUri;
                RequestObject.ContentLenght = memstream.Length;
                RequestObject.ContentCharSet = httpResponse.CharacterSet ?? string.Empty;
                RequestObject.ContentLanguage = httpResponse.Headers["Content-Language"] ?? string.Empty;
                RequestObject.ContentType = httpResponse.ContentType.ToLower();
                if (RequestObject.ContentType.IndexOf(@"/") > -1)
                {
                    do {
                        RequestObject.ContentType = RequestObject.ContentType.Substring(RequestObject.ContentType.IndexOf(@"/") + 1);
                        if (RequestObject.ContentType.IndexOf(@"/") < 0)
                            break;
                    } while (true);
                    if (RequestObject.ContentType.IndexOf(";")> -1)
                        RequestObject.ContentType = RequestObject.ContentType.Substring(0, RequestObject.ContentType.IndexOf(@";"));
                    RequestObject.ContentType = "." + RequestObject.ContentType;
                }
                RequestObject.ContentName = httpResponse.Headers["Content-Disposition"] ?? string.Empty;
                if (RequestObject.ContentName.Length == 0)
                    RequestObject.ContentName = RequestObject.ResourceURI.Segments.Last();
                RequestObject.ServerType = httpResponse.Server;
                RequestObject.ServerName = RequestObject.ResponseURI.DnsSafeHost;
                RequestObject.ServerIP = await Dns.GetHostAddressesAsync(RequestObject.ServerName);
                RequestObject.StatusCode = StatusCode;
                RequestObject.StatusDescription = httpResponse.StatusDescription;
                if (RequestObject.ProcessStream)
                    RequestObject.Payload = ProcessResponse(RequestObject.ContentStream,
                                                            Encoding.GetEncoding(RequestObject.ContentCharSet), 
                                                            httpResponse.ContentEncoding);
            }
        }
    }
    catch (WebException exW)
    {
        if (exW.Response != null)
        {
            RequestObject.StatusCode = ((HttpWebResponse)exW.Response).StatusCode;
            RequestObject.StatusDescription = ((HttpWebResponse)exW.Response).StatusDescription;
        }
        RequestObject.WebException = exW.Status;
        RequestObject.WebExceptionDescription = exW.Message;

    }
    catch (Exception exS)
    {
        if ((uint)exS.HResult == COR_E_INVALIDOPERATION)
        {
            //RequestObject.WebException = PingHostAddress("8.8.8.8", 500) > 0
            //                            ? WebExceptionStatus.NameResolutionFailure
            //                            : WebExceptionStatus.ConnectFailure;
            RequestObject.WebException = WebExceptionStatus.ConnectFailure;
            RequestObject.WebExceptionDescription = RequestObject.WebException.ToString();
        }
        else
        {
            RequestObject.WebException = WebExceptionStatus.RequestCanceled;
            RequestObject.WebExceptionDescription = RequestObject.WebException.ToString();
        }
    }
    finally
    {
        ServicePointManager.ServerCertificateValidationCallback -= TlsValidationCallback;
    }

    RequestObject.Cookies = httpRequest.CookieContainer;
    RequestObject.StatusCode = StatusCode;
    return RequestObject;

}   //HTTP_GetStream


private bool TlsValidationCallback(object sender, X509Certificate CACert, X509Chain CAChain, SslPolicyErrors sslPolicyErrors)
{
    //if (sslPolicyErrors == SslPolicyErrors.None)
    //    return true;

    X509Certificate2 _Certificate = new X509Certificate2(CACert);
    //X509Certificate2 _CACert = new X509Certificate2(@"[localstorage]/ca.cert");
    //CAChain.ChainPolicy.ExtraStore.Add(_CACert);


    //X509Certificate2 cert =  GetCertificateFromStore(thumbprint);
    X509Certificate2 cert = (X509Certificate2)CACert;

    //CspKeyContainerInfo cpsKey = (CspKeyContainerInfo)((RSACryptoServiceProvider)cert.PublicKey.Key).CspKeyContainerInfo;
    //if (cert.HasPrivateKey) { RSA rsaKey = (RSA)cert.GetRSAPrivateKey(); }
    //if (cpsKey.Accessible) { Console.WriteLine("Exportable: {0}", cpsKey.Exportable); }

    // next line generates exception "Key does not exist"
    //bool isexportable = provider.CspKeyContainerInfo.Exportable;

    CAChain.Build(_Certificate);
    foreach (X509ChainStatus CACStatus in CAChain.ChainStatus)
    {
        if ((CACStatus.Status != X509ChainStatusFlags.NoError) &
            (CACStatus.Status != X509ChainStatusFlags.UntrustedRoot))
            return false;
    }
    return true;
}


private void HTTP_RequestHeadersInit(ref HttpWebRequest httpreq, CookieContainer cookiecontainer, StreamObject postdata)
{
    httpreq.Date = DateTime.Now;
    httpreq.Timeout = 30000;
    httpreq.ReadWriteTimeout = 30000;
    httpreq.CookieContainer = cookiecontainer;
    httpreq.KeepAlive = true;
    httpreq.AllowAutoRedirect = true;
    httpreq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
    httpreq.ServicePoint.MaxIdleTime = 30000;
    httpreq.Referer = postdata.Referer;
    httpreq.UserAgent = "Mozilla / 5.0(Windows NT 6.1; WOW64; Trident / 7.0; rv: 11.0) like Gecko";
    //httpreq.UserAgent = "Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/61.0";
    httpreq.Accept = "ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
    httpreq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US;q=0.8,en-GB;q=0.5,en;q=0.3");
    httpreq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate;q=0.8");
    httpreq.Headers.Add(HttpRequestHeader.CacheControl, "no-cache");
    httpreq.Headers.Add("DNT", "1");

    if (postdata != null && postdata.UseProxy)
    {
        if (postdata.ProxyParameters != null)
        {
            if ((postdata.ProxyParameters.Host.Length > 0))
            {
                httpreq.Proxy = new WebProxy(postdata.ProxyParameters.Host, postdata.ProxyParameters.Port);
            }
            else
            {
                httpreq.Proxy = new WebProxy(postdata.ProxyParameters.Uri, postdata.ProxyParameters.BypassLocal);
            }
                httpreq.Proxy.Credentials = new NetworkCredential(postdata.ProxyParameters.Credentials.UserID,
                                                                  postdata.ProxyParameters.Credentials.Password);
        }
        else
        {
            httpreq.Proxy = WebRequest.GetSystemWebProxy();
        }
    }
}


private string ProcessResponse(Stream stream, Encoding encoding, string ContentEncoding)
{
    string html = string.Empty;
    stream.Position = 0;
    try
    {
        using (MemoryStream memStream = new MemoryStream())
        {
            if (ContentEncoding.Contains("gzip"))
            {
                using (GZipStream gzipStream = new GZipStream(stream, CompressionMode.Decompress))
                {
                    gzipStream.CopyTo(memStream);
                };
            }
            else if (ContentEncoding.Contains("deflate"))
            {
                using (DeflateStream deflStream = new DeflateStream(stream, CompressionMode.Decompress))
                {
                    deflStream.CopyTo(memStream);
                };
            }
            else
            {
                stream.CopyTo(memStream);
            }

            memStream.Position = 0;
            using (StreamReader reader = new StreamReader(memStream, encoding))
            {
                html = reader.ReadToEnd();
                html = DecodeMetaCharSetEncoding(memStream, html, encoding);
            };
        };
    }
    catch (Exception)
    {
        return string.Empty;
    }
    return html;
}


private string DecodeMetaCharSetEncoding(Stream memStream, string _html, Encoding _encode)
{
    Match _match = new Regex("<meta\\s+.*?charset\\s*=\\s*\"?(?<charset>[A-Za-z0-9_-]+)\"?",
                             RegexOptions.Singleline |
                             RegexOptions.IgnoreCase).Match(_html);
    if (_match.Success)
    {
        string charset = _match.Groups["charset"].Value.ToLower() ?? "utf-8";
        if ((charset == "unicode") | (charset == "utf-7") | (charset == "utf-16"))
           charset = "utf-8";

        try
        {
            Encoding metaEncoding = Encoding.GetEncoding(charset);
            if (_encode.WebName != metaEncoding.WebName)
            {
                memStream.Position = 0L;
                using (StreamReader recodeReader = new StreamReader(memStream, metaEncoding))
                { _html = recodeReader.ReadToEnd().Trim(); }
            }
        }
        catch (ArgumentException)
        {
            _html = string.Empty;
        }
        catch (Exception)
        {
            _html = string.Empty;
        }
    }
    return _html;
}