Downloading files from a given SharePoint folder

Question

I have created a Java client for downloading the files from the given SharePoint link. First, I authenticated the SharePoint link through my credentials and in response I get a response object.

I am stuck after the above steps. How to download those files. I have tried using JShare also, but using JShare I am getting a 401 Unauthorized error.

Please guide me through this. I have done the authentication part. Need to download those files.

Answer 1

For development used information - https://paulryan.com.au/2014/spo-remote-authentication-rest/ Everything is well described there.

import java.io.*;
import java.net.*;
import javax.xml.parsers.*;
import javax.xml.xpath.*;
import org.w3c.dom.Document;
import org.xml.sax.*;

public class LoginManagerSharePoint {
    private final String sts = "https://login.microsoftonline.com/extSTS.srf";
    private final String loginContextPath = "/_forms/default.aspx?wa=wsignin1.0";
    //private final String contextInfoPath = "/_api/contextinfo";
    private final String sharepointContext = "xxxxxxx";
    private final String reqXML = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
            "<s:Envelope xmlns:s=\"http://www.w3.org/2003/05/soap-envelope\" " +
            "xmlns:a=\"http://www.w3.org/2005/08/addressing\" " +
            "xmlns:u=\"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd\">" +
            "<s:Header><a:Action s:mustUnderstand=\"1\">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue" +
            "</a:Action><a:ReplyTo><a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>" +
            "</a:ReplyTo><a:To s:mustUnderstand=\"1\">https://login.microsoftonline.com/extSTS.srf</a:To>" +
            "<o:Security s:mustUnderstand=\"1\" " +
            "xmlns:o=\"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd\">" +
            "<o:UsernameToken><o:Username>[username]</o:Username><o:Password>[password]</o:Password>" +
            "</o:UsernameToken></o:Security></s:Header><s:Body><t:RequestSecurityToken " +
            "xmlns:t=\"http://schemas.xmlsoap.org/ws/2005/02/trust\">" +
            "<wsp:AppliesTo xmlns:wsp=\"http://schemas.xmlsoap.org/ws/2004/09/policy\">" +
            "<a:EndpointReference><a:Address>[endpoint]</a:Address></a:EndpointReference>" +
            "</wsp:AppliesTo><t:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey" +
            "</t:KeyType><t:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue" +
            "</t:RequestType><t:TokenType>urn:oasis:names:tc:SAML:1.0:assertion</t:TokenType>" +
            "</t:RequestSecurityToken></s:Body></s:Envelope>";
    private String generateSAML() {
        String saml = reqXML
                .replace("[username]", "username");
        saml = saml.replace("[password]", "password");
        saml = saml.replace("[endpoint]", String.format("https://%s.sharepoint.com/_forms/default.aspx?wa=wsignin1.0", sharepointContext));
        return saml;
    }

    public String getCookie() {
        String token;
        try {
            token = requestToken();
            String cookie = submitToken(token);
            //System.out.println(cookie);
            return cookie;
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return null;
    }

    private String requestToken() throws XPathExpressionException, SAXException,
            ParserConfigurationException, IOException {

        String saml = generateSAML();

        URL u = new URL(sts);
        URLConnection uc = u.openConnection();
        HttpURLConnection connection = (HttpURLConnection) uc;

        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.setRequestMethod("POST");
        // http://stackoverflow.com/questions/12294274/mobile-app-for-sharepoint/12295224#12295224
        connection.addRequestProperty("Content-Type",
                "text/xml; charset=utf-8");

        OutputStream out = connection.getOutputStream();
        Writer wout = new OutputStreamWriter(out);
        wout.write(saml);

        wout.flush();
        wout.close();

        InputStream in = connection.getInputStream();
        int c;
        StringBuilder sb = new StringBuilder("");
        while ((c = in.read()) != -1)
            sb.append((char) (c));
        in.close();
        String result = sb.toString();
        String token = extractToken(result);
        //System.out.println(token);
        return token;
    }

    private String extractToken(String result) throws SAXException, IOException, ParserConfigurationException, XPathExpressionException {
        //http://stackoverflow.com/questions/773012/getting-xml-node-text-value-with-java-dom
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        DocumentBuilder db = dbf.newDocumentBuilder();

        Document document = db.parse(new InputSource(new StringReader(result)));

        XPathFactory xpf = XPathFactory.newInstance();
        XPath xp = xpf.newXPath();
        String token = xp.evaluate("//BinarySecurityToken/text()", document.getDocumentElement());
        //handle error  S:Fault:
        //http://social.microsoft.com/Forums/en-US/crmdevelopment/thread/df862099-d9a1-40a4-b92e-a107af5d4ca2
        //System.out.println(token);
        return token;
    }

    private String submitToken(String token) throws IOException {
        // http://cafeconleche.org/books/xmljava/chapters/ch03s05.html
        String url = String.format("https://%s.sharepoint.com%s", sharepointContext, loginContextPath);
        URL u = new URL(url);
        URLConnection uc = u.openConnection();
        HttpURLConnection connection = (HttpURLConnection) uc;

        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.setRequestMethod("POST");
        connection.addRequestProperty("Accept", "application/x-www-form-urlencoded");
        connection.addRequestProperty("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)");
        // http://stackoverflow.com/questions/12294274/mobile-app-for-sharepoint/12295224#12295224
        connection.addRequestProperty("Content-Type", "text/xml; charset=utf-8");

        connection.setInstanceFollowRedirects(false);

        OutputStream out = connection.getOutputStream();
        Writer wout = new OutputStreamWriter(out);

        wout.write(token);

        wout.flush();
        wout.close();

        InputStream in = connection.getInputStream();

        //http://www.exampledepot.com/egs/java.net/GetHeaders.html
        String setCookieRtFa = null;
        String setCookieFedAuth = null;

        for (int i=0; ; i++) {
            String headerName = connection.getHeaderFieldKey(i);
            String headerValue = connection.getHeaderField(i);
            //System.out.println("header: " + headerName + " : " + headerValue);
            if (headerName == null && headerValue == null) {
                // No more headers
                break;
            }
            if (headerName == null) {
                // The header value contains the server's HTTP version
            }

            if (headerName != null) {
                if (headerName.equals("Set-Cookie")) {
                    if (setCookieRtFa == null) {
                        setCookieRtFa = headerValue;
                    } else {
                        int t = 0;
                        if (headerValue.equals("RpsContextCookie=; path=/"))   t = 1;

                        if (t == 0) {
                            setCookieFedAuth = headerValue;
                        }
                    }
                }
            }

        }
        String cookieContainer = setCookieRtFa.split("\\;")[0] + ";" + setCookieFedAuth.split("\\;")[0];

        in.close();

        return cookieContainer;
    }
    }

        LoginManagerSharePoint loginManagerSharePoint = new LoginManagerSharePoint();
        String cookieContainer = loginManagerSharePoint.getCookie();

        HttpClient httpClient = HttpClientBuilder.create().build();
        HttpGet httpGet = new HttpGet(URL_FILE);
        httpGet.addHeader("Cookie", cookieContainer);
        httpGet.addHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)");

        HttpResponse response = httpClient.execute(httpGet);
        InputStream inputStream = response.getEntity().getContent();

Answer 2

If you are still facing this issue, first create instance of Service using SharePoint URL, user name and password. This example is using JShare library-

Service service = new Service("https://your.sharepoint.com", "username", "password");

Now get InputStream from service instance using-

InputStream inputStream = service.getFileStream("/Shared Documents/Test.docx");

Once you get the InputStream, you can easily download the file as per your need.

The tricky part here is the SharePoint URL and file path. If you are getting 401 or 403 while getting service instance and if you can access the SharePoint on the Web using same credentials, here are few possible reasons for that

• Check your URL while creating service instance. It should be the complete URL till the project or folder you have access to. e.g. https://your.sharepoint.com/{ProjectName}
• External user vs internal user: I am not sure why but SharePoint online doesn't authenticate external users from code. (I was stuck on this for long and i found it in some discussions/forums of SharePoint)