Cookies not set with HttpUrlConnection

Question

I'm trying ton connect to a site, according to this page : http://www.mkyong.com/java/how-to-automate-login-a-website-java-example/

But it seems, that the cookies are not set. Cannot find the problem...

My aim is to authenticate to this sit in order to grab the Html. If there is another, more simple, solution, don't hesitate.

package htmlmKyongTest;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpUrlConnectionExample {

  private List<String> cookies;
  private HttpsURLConnection conn;

  private final String USER_AGENT = "Mozilla/5.0";

  public static void main(String[] args) throws Exception {



    String url = "https://www.entea.fr/";
    String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";

    HttpUrlConnectionExample http = new HttpUrlConnectionExample();

    // make sure cookies is turn on
    CookieHandler.setDefault(new CookieManager());

    // 1. Send a "GET" request, so that you can extract the form's data.
    String page = http.GetPageContent(url);
    String postParams = http.getFormParams(page, "myUsername", "myPassword");

    // 2. Construct above post's content and then send a POST request for
    // authentication
    http.sendPost(url, postParams);

    // 3. success
    String result = http.GetPageContent(gmail);
    System.out.println(result);
  }

  private void sendPost(String url, String postParams) throws Exception {

    URL obj = new URL(url);
    conn = (HttpsURLConnection) obj.openConnection();

    // Acts like a browser
    conn.setUseCaches(false);
    conn.setRequestMethod("POST");
    conn.setRequestProperty("Host", "www.entea.fr");
    conn.setRequestProperty("User-Agent", USER_AGENT);
    conn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
    conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
    conn.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
    for (String cookie : this.cookies) {
        conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
    }

    conn.setRequestProperty("Connection", "keep-alive");
    conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
    conn.setRequestProperty("Accept-Encoding","gzip,deflate");
    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
    conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
    conn.setDoOutput(true);
    conn.setDoInput(true);

    // Send post request
    DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
    wr.writeBytes(postParams);
    wr.flush();
    wr.close();

    int responseCode = conn.getResponseCode();
    System.out.println("\nSending 'POST' request to URL : " + url);
    System.out.println("Post parameters : " + postParams);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = 
             new BufferedReader(new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
    }
    in.close();

  }

  private String GetPageContent(String url) throws Exception {

    URL obj = new URL(url);
    conn = (HttpsURLConnection) obj.openConnection();

    // default is GET
    conn.setRequestMethod("GET");
    conn.setUseCaches(false);

    // act like a browser
    conn.setRequestProperty("User-Agent", USER_AGENT);
    conn.setRequestProperty("Accept",
        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");

    if (cookies != null) {
        for (String cookie : this.cookies) {
            conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
        }
    }
    int responseCode = conn.getResponseCode();
    System.out.println("\nSending 'GET' request to URL : " + url);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = 
            new BufferedReader(new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
    }
    in.close();

    // Get the response cookies
    setCookies(conn.getHeaderFields().get("Set-Cookie"));
    return response.toString();

  }

  public String getFormParams(String html, String username, String password)
        throws UnsupportedEncodingException {

    System.out.println("Extracting form's data...");
    Document doc = Jsoup.parse(html);

    Element loginform = doc.getElementById("logonForm");
    Elements inputElements = loginform.getElementsByTag("input");
    List<String> paramList = new ArrayList<String>();
    for (Element inputElement : inputElements) {
        System.out.println("inputElement : " + inputElement.toString());
        String key = inputElement.attr("name");
        String value = inputElement.attr("value");

        if (key.equals("username"))
            value = username;
        else if (key.equals("password")) {
            value = password;
        }
        if (!(inputElement.attr("style").contains("none"))) paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
    }

    // build parameters list
    StringBuilder result = new StringBuilder();
    for (String param : paramList) {
        if (result.length() == 0) {
            result.append(param);
        } else {
            result.append("&" + param);
        }
    }

    System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
    return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
  }

  public List<String> getCookies() {
    return cookies;
  }

  public void setCookies(List<String> cookies) {
    this.cookies = cookies;
  }

}

the Output is the following :

Sending 'GET' request to URL : https://www.entea.fr/
Response Code : 200
Extracting form's data...
inputElement : <input type="password" id="autocompleteoff" style="display:none" />
inputElement : <input type="hidden" id="curl" name="curl" value="Z2F" />
inputElement : <input type="hidden" id="flags" name="flags" value="0" />
inputElement : <input type="hidden" id="forcedownlevel" name="forcedownlevel" value="0" />
inputElement : <input type="hidden" id="formdir" name="formdir" value="7" />
inputElement : <input type="text" id="username" name="username" />
inputElement : <input id="password" onfocus="g_fFcs=0" type="password" name="password" autocomplete="off" />
inputElement : <input type="radio" name="trusted" id="rdoPblc" value="0" onclick="clkSec()" checked="checked" />
inputElement : <input type="radio" name="trusted" id="rdoPrvt" value="4" onclick="clkSec()" />
inputElement : <input id="SubmitCreds" onclick="clkLgn()" type="image" value="Ouvrir une session" name="SubmitCreds" src="/CookieAuth.dll?GetPic?formdir=7&amp;image=encart_footer_01.png" alt="Connexion" />
result : curl=Z2F&flags=0&forcedownlevel=0&formdir=7&username=xxxxxxxxx&password=xxxxxxxxxxx*&trusted=0&SubmitCreds.x=0&SubmitCreds.y=0&SubmitCreds=Ouvrir+une+session
Exception in thread "main" java.lang.NullPointerException
    at htmlmKyongTest.HttpUrlConnectionExample.sendPost(HttpUrlConnectionExample.java:66)
    at htmlmKyongTest.HttpUrlConnectionExample.main(HttpUrlConnectionExample.java:46)

Answer 1

The cookies field isn't initialized for the first sendPost. You should protect your for loop with a if (cookies != null).

EDIT : The actual solution is to split the initial request + redirect into 2 separate GET requests.

Try this :

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpUrlConnectionExample {

    private List<String> cookies;
    private HttpsURLConnection conn;

    private final String USER_AGENT = "Mozilla/5.0";

    public static void main(String[] args) throws Exception {
        String url = "https://www.entea.fr/";
        String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";

        HttpUrlConnectionExample http = new HttpUrlConnectionExample();

        // make sure cookies is turn on
        CookieHandler.setDefault(new CookieManager());

        // 1. Send a "GET" request, so that you can extract the form's data.
        http.getPageContent(url);
        String page = http.getPageContent("https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");

        String postParams = http.getFormParams(page, "myUsername", "myPassword");

        // 2. Construct above post's content and then send a POST request for
        // authentication
        http.sendPost(url, postParams);

        // 3. success
        String result = http.getPageContent(gmail);
        System.out.println(result);
    }

    private void sendPost(String url, String postParams) throws Exception {

        URL obj = new URL(url);
        conn = (HttpsURLConnection) obj.openConnection();

        // Acts like a browser
        conn.setUseCaches(false);
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Host", "www.entea.fr");
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        for (String cookie : this.cookies) {
            conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
        }

        conn.setRequestProperty("Connection", "keep-alive");
        conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
        conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
        conn.setDoOutput(true);
        conn.setDoInput(true);

        // Send post request
        DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
        wr.writeBytes(postParams);
        wr.flush();
        wr.close();

        int responseCode = conn.getResponseCode();
        System.out.println("\nSending 'POST' request to URL : " + url);
        System.out.println("Post parameters : " + postParams);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in =
                new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

    }

    private String getPageContent(String url) throws Exception {

        URL obj = new URL(url);
        conn = (HttpsURLConnection) obj.openConnection();

        // default is GET
        conn.setRequestMethod("GET");
        conn.setUseCaches(false);

        // act like a browser
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Accept",
                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
        conn.setInstanceFollowRedirects(false);

        if (cookies != null) {
            for (String cookie : this.cookies) {
                conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
            }
        }
        int responseCode = conn.getResponseCode();
        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in =
                new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

        // Get the response cookies
        setCookies(conn.getHeaderFields().get("Set-Cookie"));
        return response.toString();

    }

    public String getFormParams(String html, String username, String password)
            throws UnsupportedEncodingException {

        System.out.println("Extracting form's data...");
        Document doc = Jsoup.parse(html);

        Element loginform = doc.getElementById("logonForm");
        Elements inputElements = loginform.getElementsByTag("input");
        List<String> paramList = new ArrayList<String>();
        for (Element inputElement : inputElements) {
            System.out.println("inputElement : " + inputElement.toString());
            String key = inputElement.attr("name");
            String value = inputElement.attr("value");

            if (key.equals("username"))
                value = username;
            else if (key.equals("password")) {
                value = password;
            }
            if (!(inputElement.attr("style").contains("none")))
                paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
        }

        // build parameters list
        StringBuilder result = new StringBuilder();
        for (String param : paramList) {
            if (result.length() == 0) {
                result.append(param);
            } else {
                result.append("&" + param);
            }
        }

        System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
        return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
    }

    public void setCookies(List<String> cookies) {
        if (cookies != null) {
            this.cookies = cookies;
        }
    }

}