Send a GET Request to a Webstie which has domain HTTPS by using Winsock2

Question

I'm trying to make a GET request to Webserver, I have been successful toward Websites having HTTP domain. The problem remaining is that I can't send a request successfully toward Websites having HTTPS domain.

I have to use Winsock in C++ to accomplish this assignment. This is my code that have been successful toward HTTP:

void GET_request(URL url)
{
    WSADATA wsaData;
    SOCKET Socket;
    SOCKADDR_IN SockAddr;
    struct hostent* localHost;
    string request_header;
    char* localIP;
    int find_index;
    url.URL_detach();

    //Create a Request Header.
    request_header = "GET "+ url.path + " HTTP/1.0\r\n";
    request_header += "Host: " + url.host + "\r\n";
    request_header += "Connection: close\r\n";
    request_header += "Cache-Control: max-age=0\r\n";
    request_header += "Upgrade-Insecure-Requests: 1\r\n";
    request_header += "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\n";
    request_header += "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\r\n";
    request_header += "Accept-Encoding: gzip, deflate\r\n";
    request_header += "\r\n";

    if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
        cout << "WSAStartup failed.\n";
        system("pause");
    }

    Socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);         

    localHost = gethostbyname(url.host.c_str());                
    localIP = inet_ntoa(*(struct in_addr *)*localHost->h_addr_list);        //Return IP of HOST
    SockAddr.sin_port = htons(80);                                          //PORT 80 (HTTP)
    SockAddr.sin_family = AF_INET;                                          // TCP/IP
    SockAddr.sin_addr.s_addr = inet_addr(localIP);                          

    if (connect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr)) != 0)     
    {
        cout << "Could not connect to Server";
        system("pause");
    }

    send(Socket, request_header.c_str(), strlen(request_header.c_str()), 0);    


    int nDataLength;
    while ((nDataLength = recv(Socket, buffer, 10000, 0)) > 0) {
        int i = 0;
        while (buffer[i] >= 32 || buffer[i] == '\n' || buffer[i] == '\r') {
            Response_Header += buffer[i];
            i += 1;
        }
        find_index = Response_Header.find("</html>");
    }
    Response_Header.erase(find_index + 7);
    closesocket(Socket);
    WSACleanup();

}

Answer 1

Let me thank hugtech for the question, code and for doing work with the HTTP protocol. Also big thanks to Remy Lebeau for informing us about SSL and TLS handshake. From their my research took off.

In my case, sending the raw HTTP GET command caused WinSock send(...) to fail with zero. Imagine that, I am not even allowed to send that unencrypted stuff. After using the ClientHello that I copied from a site, I got www.youtube.com to send me back a ServerHello. And I continually adjusted, compiled my code and tested; I hope Youtube doesn't get frustrated and ask for a CAPTCHA.

I was then able to modify the ClientHello successfully. I simple took out the extensions at the end as they are optional and furthermore I won't be supporting them. And modified the length variables to account for the adjustment.

// Client Hello I copied from the site
//uint8_t clientHello[] = { 0x16, 0x03, 0x01, 0x00, 0xa5, 0x01, 0x00, 0x00, 0xa1, 0x03, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x20, 0xcc, 0xa8, 0xcc, 0xa9, 0xc0, 0x2f, 0xc0, 0x30, 0xc0, 0x2b, 0xc0, 0x2c, 0xc0, 0x13, 0xc0, 0x09, 0xc0, 0x14, 0xc0, 0x0a, 0x00, 0x9c, 0x00, 0x9d, 0x00, 0x2f, 0x00, 0x35, 0xc0, 0x12, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x18, 0x00, 0x16, 0x00, 0x00, 0x13, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x75, 0x6c, 0x66, 0x68, 0x65, 0x69, 0x6d, 0x2e, 0x6e, 0x65, 0x74, 0x00, 0x05, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x0a, 0x00, 0x08, 0x00, 0x1d, 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, 0x00, 0x0d, 0x00, 0x12, 0x00, 0x10, 0x04, 0x01, 0x04, 0x03, 0x05, 0x01, 0x05, 0x03, 0x06, 0x01, 0x06, 0x03, 0x02, 0x01, 0x02, 0x03, 0xff, 0x01, 0x00, 0x01, 0x00, 0x00, 0x12, 0x00, 0x00, };

// modified client hello
// unnecessary extension code has been erased,
// just as how stack overflow moderators will erase
// my unnecessary wordings and sentencing.
uint8_t clientHello[] =
{
 //Record Header
    0x16, //type: handshake
    0x03, 0x01, //TLS version:
    0x00, 75, //HandShake Length

 //Handshake Header
    0x01, //type: client hello
    0x00, 0x00, 71, //handshake data len

 // Client Version
    0x03, 0x03, // TLS 1.2 //actual TLS version client uses

 // Client Random Garbage 
 // Here I see 32 bytes being used; but I don't know if 32 bytes is mandatory.
 // But if it is not mandatory how would one know when the random garbage ends? 
    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,

 //Session ID
 // This ID is used to logon back from a previous session.
 // This eliminates certain computations as per what I read.
 // Here we use zero to say this is a fresh session with the server.
    0x00,

 // Cypher Suites
 // A list of Cypher Suite used to do the encryption. 
 // Cypher Suites are represented by two bytes.
 // Example 0xCCA8 is TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 Cypher Suite.
    0x00, 0x20, // count of bytes
    0xcc, 0xa8, 0xcc, 0xa9, 0xc0, 0x2f, 0xc0, 0x30, 0xc0, 0x2b, 0xc0, 0x2c, 0xc0, 0x13, 0xc0, 0x09,
    0xc0, 0x14, 0xc0, 0x0a, 0x00, 0x9c, 0x00, 0x9d, 0x00, 0x2f, 0x00, 0x35, 0xc0, 0x12, 0x00, 0x0a,

 // Compression Methods
 // A list of compression methods.
 // Here we have only one compression method which is zero for no compression.
    0x01, //bytes of compression methods
    0x00,

// After this we have one long list of Extension information which
// I have no interest in at all.
};

size_t clientHelloLen = sizeof(clientHello);

To be continued...
ServerHello

I have read 350 pages of Joshua Davis's book, Implementing SSL_TLS and it turns out that TLS handshake is much more complicated that anyone is explaining it to be. I managed to prepare my own ClientHello message to www.youtube.com and get back a valid ServerHello and ServerCertificate(s). But when I try this with www.example.org and www.apachefriends.org I don't get back any message. I then tried back the original ClientHello I copied from https://tls.ulfheim.net/ and I got back a properly formated ServerHello and ServerCertificate which I could print with my custom routines. Problem is it takes like one minute to get a response from the server; and I notice that the certificate looks suspicious. It is issued from and issued to the same exact entity. And also it states that is a fake certificate control.

I then used OpenSSL to run the following command. openssl s_client -connect "www.apachefriends.org:https". I got back a response from the server instantly! Which means that the site is rejecting my custom ClientHello message. Which means I wasted my weekend studying Joshua Davis book, and should have instead read a book showing how to use OpenSSL.

Update:
I finally got stuff to work, but HTTPS, TLS/SSL handshake is so darn complicated. I am vex that I studied the details of ClientHello, ServerHello, ServerCertificate and simply have no use for it. All of that is abstracted away by OpenSSL. And for some reason websites refuse to accept my custom ClientHello, even though I did exactly what Joshua Davis showed. I am also feeling bad that OpenSSL abstracts away Linux and Win32 SOCKETS; so learning those details was just to be academic.

The following is my code where I use OpenSSL to fetch a file from the server. Note that when it comes to downloading a file, the server normally redirects you to another server that actually has the file. Here I have hardcoded that redirection.

#include <Jav/test.h> // rep(...)

// My custom library for handling files
// You can substitute this with std::fstream, FILE*, or
// any other file library
#include <Jav/file/file.h>

// Custom exception class, Jav::Error
// derived from std::exception
#include <Jav/error/error.h>

//Trying to get _fileno to be defined
// as it is needed by openssl/applink.c
#undef __STRICT_ANSI__ 

// Undefing strict ansi failed so I simply
// declared _fileno
extern "C"{
_CRTIMP int __cdecl __MINGW_NOTHROW _fileno (FILE*);
}

# include  <openssl/bio.h>
# include  <openssl/ssl.h>
# include  <openssl/err.h>

// Yep we actually include a source file. 
// Other wise openssl gives "no app link" error during runtime.
// I tried adding applink.c to src instead of including it
// but got linker errors.
# include  <openssl/applink.c>

void init_openssl_library()
{
 SSL_library_init();
 SSL_load_error_strings();
 OPENSSL_config(NULL);
}

void on_openssl_fail(int line)
{
 ERR_print_errors_fp(stderr);
 throw Jav::Error("problem at line %i\n",line);
}

#define HOST_REDIRECTED "redirectedexamle.org" 
#define RESOURCE        "/exampleresource.txt"

int main()
{
    SSL_CTX *ctx = NULL;
    BIO *web = NULL, *out = NULL;
    SSL *ssl = NULL;

    init_openssl_library();

    const SSL_METHOD *method = SSLv23_method();
    if(method == NULL) on_openssl_fail(__LINE__);

    ctx = SSL_CTX_new(method);
    if(ctx == NULL) on_openssl_fail(__LINE__);

    // Cannot fail???
    //SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,verify_callback);
    //SSL_CTX_set_verify_depth(ctx,4);

    const long flags = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION;
    SSL_CTX_set_options(ctx,flags);

    web = BIO_new_ssl_connect(ctx);
    if(web == NULL) on_openssl_fail(__LINE__);

    if(BIO_set_conn_hostname(web,HOST":https") != 1)
        on_openssl_fail(__LINE__);

    BIO_get_ssl(web,&ssl);
    if(ssl == NULL) on_openssl_fail(__LINE__);

    const char *PREFERRED_CIPHERS = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4";
    if(SSL_set_cipher_list(ssl,PREFERRED_CIPHERS) != 1)
        on_openssl_fail(__LINE__);

    if(SSL_set_tlsext_host_name(ssl,HOST) != 1)
        on_openssl_fail(__LINE__);

    //out = BIO_new_fp(stdout,BIO_NOCLOSE);
    //if(out == NULL) on_openssl_fail(__LINE__);

    if(BIO_do_connect(web) != 1) on_openssl_fail(__LINE__);
    if(BIO_do_handshake(web) != 1) on_openssl_fail(__LINE__);

    X509 *cert = SSL_get_peer_certificate(ssl);
    if(cert == NULL) on_openssl_fail(__LINE__);
    X509_free(cert);

    //if(SSL_get_verify_result(ssl) != X509_V_OK) on_openssl_fail(__LINE__);

    int len = 10000;
    Jav::cstring buf(len);

    auto cmd = "GET "RESOURCE" HTTP/1.1\r\n"
               "Host: "HOST"\r\n"
               "Connection: close\r\n\r\n";
      
    if(BIO_puts(web, cmd) <= 0)
    {
        rep("bio write failed");

        if(! BIO_should_retry(web))
        {
          rep("bio should retry");
        }
        else return 1;
    }

    int total = 0;
    Jav::wFile file = Jav::createNewFile<Jav::wFile>("download.txt");

    while(true)
    {
        int numRead = BIO_read(web, buf, len);
        total += numRead;

        if(numRead == 0)
        {
         rep("No more data read");
         if(total) break;
         else return 1;
        }
        else if(numRead < 0)
        {
            rep("bio read failed");

            if(! BIO_should_retry(web))
            {
              rep("bio should retry");
        int numRead = BIO_read(out, buf, len);
        rep("num read",numRead);
            on_openssl_fail(__LINE__);
            }
            else return 1;
        }
        else rep("num read",numRead);

        file.write(buf,numRead);
    }

//Jav::wFile file = Jav::createFile<Jav::wFile>("output.txt");
//file.write(buf,numRead);
    BIO_free(out);
    BIO_free(web);
    SSL_CTX_free(ctx);
}

Answer 2

Jav HTTPS Networking: The Settlement

My whole purpose of learning HTTPS was to create a downloader to be used in my custom installer. All I will do is present my code for doing that; which includes my custom library. You can easily replace my custom library code with your own.

To get the job done I will be utilizing the OpenSSL SDK. This library abstracts away Linux/Win32 Sockets and the TLS Handshake and Certificate verification. Which means you don't need to study Brian Hall - Beej's Network Programming, or Anthony Jones - Network Programming For Windows. And you definitely don't need to pick up Joshua Jones - Implementing SSL_TLS.

You only need to study TutorialPoints.com tutorial on HTTP. I never ever could read off all of TutorialPoints.com tutorials. I simple look at the part about GET command and the five types of HTTP Server Resonse. POST command is sometimes required to download a file, but not in my case.

• 100:STATUS_PROCESSING
• 200:STATUS_OK
• 300:STATUS_REDIRECTED
• 400:STATUS_FAILED
• 500:STATUS_SERVER_ERROR

In the case of downloading a file, you are bound to get STATUS_REDIRECTED; as it seems servers give other dedicated servers the job of storing big files to be downloaded.

Final Note: This code makes use of GNU, Mingw extension. Namely the ability to copy the address of a goto label into a void pointer; This simplifies my conditional goto branching. You can use a flag instead. You can even eliminate goto and use your own mechanism. I used goto as it made the code easier to write.

Jav/file/buffer.h

#ifndef JAV_BUFFER_HPP
#define JAV_BUFFER_HPP

#include <Jav/bits.h> /* bitMove */

namespace Jav {

enum { KB=1024, MB=KB*1024, GB=MB*1024 };

/** Represents a chunk of data from a particular source. Eg: file, network
  *
  * Buffer is not meant to operate as dynamic memory,
  * that allocates and reallocates memory to constantly increase/ decrease capacity.
  * Consider using std::string for such purpose.
*/
struct Buffer
{
 uint8_t *begin; // begin of allocated memory
 uint8_t *pos;   // position in buffer
 uint8_t *end;   // end of data written to buffer
 uint8_t *max;   // end of allocated memory

 Buffer();
 explicit Buffer(std::size_t size);
 Buffer(std::size_t size,std::size_t max);
 Buffer(Buffer &&move);
 Buffer& operator= (Buffer &&move);
 ~Buffer();

 explicit operator bool()const { return begin; }
 bool isEmpty()const           { return begin == end; }
 std::size_t size()const       { return end - begin; }
 std::size_t unread()const     { return pos >= end ? 0 : end - pos; }
 std::size_t space()const      { return max - pos; }
 std::size_t freeSpace()const  { return max - end; }
 std::size_t capacity()const   { return max - begin; }
 void clear()                   { end = pos = begin; }
};


/** Represents a chunk of data from a particular source. Eg: file, network */
class FileBuffer
{
 public:
    FileBuffer();
    explicit FileBuffer(std::size_t size);
    FileBuffer(std::size_t size,std::size_t max);
    FileBuffer(FileBuffer &&move);
    FileBuffer& operator= (FileBuffer &&move);
    ~FileBuffer();

 public:
    explicit operator bool()const { return begin; }
    std::size_t size()const       { return end - begin; }
    std::size_t unread()const     { return pos >= end ? 0 : end - pos; }
    std::size_t space()const      { return max - pos; }
    std::size_t freeSpace()const  { return max - end; }
    std::size_t capacity()const   { return max - begin; }
    void clear()                    { end = pos = begin; }

 private:
    int64_t fpos;   // pos in file
    uint8_t *begin; // begin of allocated memory
    uint8_t *pos;   // position in buffer
    uint8_t *end;   // end of data written to buffer
    uint8_t *max;   // end of allocated memory

 private:
    friend class FileBase;
    friend class ReadableFile;
    friend class WritableFile;
};


/// Read From Buffer

void* read   (Buffer&, void *dest, size_t dest_size);
void* readBeg(Buffer&, void *dest, size_t dest_size);
void* read   (Buffer&, int offset, void *dest, size_t dest_size);
void* readBeg(Buffer&, int offset, void *dest, size_t dest_size);


/// Write Into Buffer

void write(Buffer&,const void *src,size_t);
void writeBeg(Buffer&,const void *src,size_t);
void writeEnd(Buffer&,const void *src,size_t);
void insert(Buffer&,const void *src,size_t);
void insertBeg(Buffer&,const void *src,size_t);

void write(Buffer&,int offset,const void *src,size_t);
void writeBeg(Buffer&,int offset,const void *src,size_t);
void writeEnd(Buffer&,int offset,const void *src,size_t);
void insert(Buffer&,int offset,const void *src,size_t);
void insertBeg(Buffer&,int offset,const void *src,size_t);


/// Data Management
void notifyDataWritten(Buffer&,std::size_t);
void notifyDataRead(Buffer&,std::size_t);
bool shiftBytes(Buffer&,byte *from,byte *to);


/// Size Management
void setCapacity(Buffer&,std::size_t);


#include "Buffer.inl"

}

#endif // JAV_BUFFER_HPP

Jav/string/parser.h

#ifndef JAV_PARSER_HPP
#define JAV_PARSER_HPP

#include <Jav/string/cstring.h>
#include <Jav/error/error.h>
#include <string>
#include <stdio.h>

namespace Jav {

class Parser
{
 public:
    Parser(const char *bytes) : bytes(bytes) {}
    Parser(const Parser &parser) : bytes(parser.bytes) {}

 public:
    const char* skipSpaces(const char *it);
    const char* skipAllSpaces(const char *it);
    const char* nextLine(const char *it);
    ...

 public:
    const char* skipSpaces(const char *it, const char *end);
    const char* skipAllSpaces(const char *it, const char *end);
    const char* nextLine(const char *it, const char *end);
    ...

 protected:
    const char *bytes;
};
...

}

#endif // JAV_PARSER_HPP

Net.h

#ifndef JAV_APP_NET_HPP
#define JAV_APP_NET_HPP

#include <Jav/string/parser.h>
#include <Jav/string/cstring.h>
#include <Jav/file/buffer.h>
#include <openssl/bio.h>
#include <openssl/ssl.h>


enum { STATUS_UNKNOWN, STATUS_OK, STATUS_REDIRECT, STATUS_FAIL  };

struct Url
{
 Jav::cstring port;
 Jav::cstring host;
 Jav::cstring path;
};

bool parseUrl(Url&,const char *url);

struct Net
{
 public:
    Net(const char *url);
    Net(Net&&);
    Net& operator=(Net&&);
    ~Net();

 public:
    void requestFile();
    size_t recieveResponse(Jav::Buffer&);

 private:
    void on_openssl_fail(int line);

 private:
    Url url;
    SSL_CTX *ctx;
    BIO *web;
    SSL *ssl;
};

struct ServerResponseParser : Jav::Parser
{
    int status;
    size_t content_size;
    Jav::cstring content_url;
    Jav::cstring status_string;

    ServerResponseParser() : Jav::Parser(NULL) {}
    bool parse(Jav::Buffer &response,Net &store);

 protected:
    bool parseStatusLine(Jav::Buffer &response,Net &store);
    bool parseAttribute(Jav::Buffer &response,Net &store);
};

void init_openssl_library();

#endif

Net.cpp

#include <Jav/error/error.h>
#include <Jav/string/compare.h>
#include <Jav/string/builder.h>
#include <Jav/string/charconv.h>
#include "Net.h"
#include <openssl/err.h>
extern "C" _CRTIMP int __cdecl __MINGW_NOTHROW  _fileno (FILE*);
#include <openssl/applink.c> //essential to avoid no applink runtime error


void init_openssl_library()
{
 SSL_library_init();
 SSL_load_error_strings();
 OPENSSL_config(NULL);
}

bool parseUrl(Url &obj,const char *url)
{
 const char *scheme, *scheme_end;
 const char *host, *host_end;
 const char *path;

 if(url == NULL)
    throw Jav::Error("url is null");

 scheme = url;
 scheme_end = Jav::nextPosOf("://",scheme);
 if(scheme_end == NULL) throw Jav::Error("Url Error: invalid scheme");

 host = scheme_end + 3;
 path = host_end = Jav::nextPos('/',host);

 // Erase port number from host name
 host_end = Jav::nextPosOf(':',{host,host_end});
 if(host_end == NULL) host_end = path;

 obj.port = Jav::cstring(scheme,scheme_end);
 obj.host = Jav::cstring(host,host_end);
 obj.path = path;

 return true;
}


Net::Net(const char *url)
{
    parseUrl(this->url,url);

    const SSL_METHOD *method = SSLv23_method();
    if(method == NULL) on_openssl_fail(__LINE__);

    ctx = SSL_CTX_new(method);
    if(ctx == NULL) on_openssl_fail(__LINE__);

    // Cannot fail???
    //SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,verify_callback);
    //SSL_CTX_set_verify_depth(ctx,4);

    const long flags = SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION;
    SSL_CTX_set_options(ctx,flags);

    web = BIO_new_ssl_connect(ctx);
    if(web == NULL) on_openssl_fail(__LINE__);

    auto hostName = Jav::buildString("?:?",this->url.host,this->url.port);
    if(BIO_set_conn_hostname(web,hostName.c_str()) != 1)
        on_openssl_fail(__LINE__);

    BIO_get_ssl(web,&ssl);
    if(ssl == NULL) on_openssl_fail(__LINE__);

    const char *PREFERRED_CIPHERS = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4";
    if(SSL_set_cipher_list(ssl,PREFERRED_CIPHERS) != 1)
        on_openssl_fail(__LINE__);

    if(SSL_set_tlsext_host_name(ssl,this->url.host) != 1)
        on_openssl_fail(__LINE__);

    //out = BIO_new_fp(stdout,BIO_NOCLOSE);
    //if(out == NULL) on_openssl_fail(__LINE__);

    if(BIO_do_connect(web) != 1) on_openssl_fail(__LINE__);
    if(BIO_do_handshake(web) != 1) on_openssl_fail(__LINE__);

    X509 *cert = SSL_get_peer_certificate(ssl);
    if(cert == NULL) on_openssl_fail(__LINE__);
    X509_free(cert);

    //How to get the following verify method to not fail
    //if(SSL_get_verify_result(ssl) != X509_V_OK) on_openssl_fail(__LINE__);
}

Net::Net(Net &&rhs)
 : ctx(Jav::bitMove<SSL_CTX>(&rhs.ctx)),
   ssl(Jav::bitMove<SSL>(&rhs.ssl)),
   web(Jav::bitMove<BIO>(&rhs.web)),
   url(rhs.url)
{
}

Net& Net::operator=(Net &&rhs)
{
 Jav::bitMove<SSL_CTX>(&ctx,&rhs.ctx);
 Jav::bitMove<SSL>(&ssl,&rhs.ssl);
 Jav::bitMove<BIO>(&web,&rhs.web);
 url = rhs.url;
 return *this;
}

Net::~Net()
{
    //SSL_free(ssl);
    SSL_CTX_free(ctx);
    //BIO_free(out);
    BIO_free(web);
}

void Net::requestFile()
{
    auto cmd = Jav::buildString(
                    "GET ? HTTP/1.1\r\n"
                    "Host: ?\r\n"
                    "Connection: close\r\n\r\n",
                    url.path, url.host
                );

    if(BIO_write(web, cmd.c_str(), cmd.size()) <= 0)
        on_openssl_fail(__LINE__);
}

size_t Net::recieveResponse(Jav::Buffer &buf)
{
    size_t numRead = BIO_read(web, buf.end, buf.freeSpace());
    if(numRead < 0) on_openssl_fail(__LINE__);

    buf.end += numRead;
    return numRead;
}

void Net::on_openssl_fail(int line)
{
 ERR_print_errors_fp(stderr);
 throw Jav::Error("problem at line %i\n",line);
}


bool ServerResponseParser::parseStatusLine(Jav::Buffer &response,Net &store)
{
    void *section = &&PARSE_HTTP_VERSION;

    if(response.unread() == 0) goto STORE;

    PARSE_HTTP_VERSION:
    {
     const char *begin = (char*)response.pos;
     const char *end = Jav::nextPosOf(' ',{begin,(char*)response.end});
     if(end == NULL) goto STORE;

     if( !Jav::isMatch({begin,end},"HTTP") )
        throw Jav::Error("Expected HTTP version in status line of Server response");

     response.pos = (uint8_t*)end+1;
     section = &&PARSE_STATUS_CODE;
    }

    PARSE_STATUS_CODE:
    {
     const char *begin = (char*)response.pos;
     const char *end = Jav::nextPosOf(' ',{begin,(char*)response.end});
     if(end == NULL) goto STORE;

     if(end - begin != 3) throw Jav::Error("status code requires 3 digits: %i",end-begin);

     status = begin[0] == '2' ? STATUS_OK :
              begin[0] == '3' ? STATUS_REDIRECT :
              begin[0] == '4' ? STATUS_FAIL :
                /** ELSE */  STATUS_UNKNOWN;

     if(!std::isdigit(begin[1]) || !std::isdigit(begin[2]))
        throw Jav::Error("status code should be a number");

     response.pos = (uint8_t*)end+1;
     section = &&PARSE_STATUS_STRING;
    }

    PARSE_STATUS_STRING:
    {
     const char *begin = (char*)response.pos;
     const char *end = Jav::nextPosOf("\r\n",{(char*)response.pos,(char*)response.end});
     if(end == NULL) goto STORE;

     status_string = Jav::cstring(begin,end);
     response.pos = (uint8_t*)end+1;
    }

    if(status == STATUS_FAIL)
        throw Jav::Error("Request Failed: %s",status_string.str());

    if(status == STATUS_UNKNOWN)
        throw Jav::Error("Unsupported status code");

    return true;

    STORE:
    {
     if(response.pos == response.begin && !response.isEmpty())
        throw Jav::Error("Failed to parse status line of server response");

     shiftBytes(response,response.pos,response.begin);
     response.pos = response.begin;
     store.recieveResponse(response);

     if(response.isEmpty())
        throw Jav::Error("Failed to parse status line of server response");

     goto *section;
    }
}

bool ServerResponseParser::parseAttribute(Jav::Buffer &response,Net &store)
{
    enum { VAR_LOCATION=1, VAR_CONTENT_LENGTH, VAR_CONTENT_TYPE };

    void *section = &&PARSE_VARIABLE;
    int var;

    PARSE_VARIABLE:
    {
        const char *begin = (char*)response.pos;
        const char *end = Jav::nextPosOf(':',{begin,(char*)response.end});
        if(end == NULL) goto STORE;

        if( Jav::isMatch({begin,end},"Content-Length") ) var = VAR_CONTENT_LENGTH;
        else if( Jav::isMatch({begin,end},"Content-Type") ) var = VAR_CONTENT_TYPE;
        else if( Jav::isMatch({begin,end},"Location") ) var = VAR_LOCATION;
        else                                         var = 0;

        response.pos = (uint8_t*)skipSpaces(end+1);
        section = &&PARSE_VALUE;
    }

    PARSE_VALUE:
    {
        const char *begin = (char*)response.pos;
        const char *end = Jav::nextPosOf("\r\n",{begin,(char*)response.end});
        if(end == NULL) goto STORE;

        switch(var)
        {
         case VAR_LOCATION:
            if(status == STATUS_REDIRECT) content_url = Jav::cstring(begin,end);
            break;

         case VAR_CONTENT_LENGTH:
            if(status == STATUS_OK)
            if(!Jav::ston(begin,end,content_size,Jav::STON_UINT_DEC))
                throw Jav::Error("Content-Length is not a number");
            break;

         case VAR_CONTENT_TYPE:
            if(var == STATUS_OK)
            if(!Jav::isMatch({begin,end},"application/"))
                throw Jav::Error("mime type Expected: application/*, Found: %s",Jav::cstring(begin,end).str());
        }

        response.pos = (uint8_t*)end+2;
        return true;
    }

    STORE:
    {
     if(response.pos == response.begin && !response.isEmpty())
        throw Jav::Error("Failed to parse attribute of server response");

     shiftBytes(response,response.pos,response.begin);
     response.pos = response.begin;
     store.recieveResponse(response);

     if(response.isEmpty())
        throw Jav::Error("Failed to parse attribute of server response");

     goto *section;
    }
}

bool ServerResponseParser::parse(Jav::Buffer &response,Net &store)
{
    parseStatusLine(response,store);

    PARSE_ATTRIBUTES:
    while(true)
    {
        if(response.unread() == 0) goto STORE;

        if(Jav::isMatch(response.pos,"\r\n"))
        {
         response.pos += 2;
         return true;
        }

        parseAttribute(response,store);
    }

    STORE:
    {
     if(response.pos == response.begin && !response.isEmpty())
        throw Jav::Error("Failed to parse server response");

     shiftBytes(response,response.pos,response.begin);
     response.pos = response.begin;
     store.recieveResponse(response);

     if(response.isEmpty())
        throw Jav::Error("Failed to parse server response");

     goto PARSE_ATTRIBUTES;
    }
}

main.cpp

#include <Jav/error/error.h>
#include <Jav/console/screen.h>
#include <Jav/file/file.h>
#include "Net.h"

int main(int argc,const char **argv)
{
    Jav::Buffer buf(4096);

    init_openssl_library();
    Net net("https://www.apachefriends.org/xampp-files/7.4.27/xampp-windows-x64-7.4.27-2-VC15-installer.exe");
    net.requestFile();

    ServerResponseParser parser;
    parser.parse(buf,net);

    if(parser.status == STATUS_REDIRECT)
    {
     buf.clear();
     net = Net(parser.content_url);
     net.requestFile();
     parser.parse(buf,net);

     if(parser.status != STATUS_OK)
        throw Jav::Error("Server is playing games");
    }

    // How to deduce filename from the HTTP Server Response
    Jav::wFile file = Jav::createNewFile<Jav::wFile>("download.exe");

    Jav::ConsoleScreen screen = Jav::openTerminal<Jav::ConsoleScreen>();
    Jav::cstring progress(100);

    size_t totalWrit = 0;

    while(true)
    {
        if(buf.unread() == 0)
        {
         buf.clear();
         net.recieveResponse(buf);
         if(buf.isEmpty()) return 0;
        }

        size_t numWrit = file.write(buf.pos,buf.unread());
        totalWrit += numWrit;
        buf.pos += numWrit;

        Jav::Point screen_pos = getCursorPos(screen);
        Jav::Point screen_sz = getScreenSize(screen);

        auto sz = sprintf(progress,"%i/%i %i%%",totalWrit,parser.content_size,(int)(totalWrit*100.0/parser.content_size));
        memset(progress+sz,' ',screen_sz.x-sz);
        screen.write(progress,sz);
        setCursorPos(screen,0,screen_pos.y);
    }
}