HTTPURLConnection - POST multipart/form-data with large file with FixedLengthStreamingMode

Question

So I'm trying to send a multipart/form-data POST request with large image file. I can't pre-convert file to byte array, my app will crash with OutOfMemory exception, so I have to write contents of the file directly to connection's outputstream. Also, my server doesn't support chunked mode, so I have to calculate content length before sending data and use connection's setFixedLengthStreamingMode.

public void createImagePostWithToken(String accessToken, String text,
        String type, String imagePath) {

    URL imageUrl = null;
    String lineEnd = "\r\n";
    String twoHyphens = "--";

    // generating byte[] boundary here

    HttpURLConnection conn = null;
    DataOutputStream outputStream = null;
    DataInputStream inputStream = null; 

    int bytesRead, bytesAvailable, bufferSize;
    byte[] buffer;
    int maxBufferSize = 1*1024*1024;

    try
    {
        long contentLength;
        int serverResponseCode;
        String serverResponseMessage;
        File file = new File(imagePath);            
        FileInputStream fileInputStream = new FileInputStream(file);
        imageUrl = buildUri("posts").toURL();
        conn = (HttpURLConnection)imageUrl.openConnection();
        conn.setConnectTimeout(30000);
        conn.setReadTimeout(30000);
        conn.setDoOutput(true);
        conn.setDoInput(true);         
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "multipart/form-data;boundary=" + boundary);            

        String stringForLength = new String();  

        stringForLength += "Content-Type: multipart/form-data;boundary=" + boundary;

        stringForLength += twoHyphens + boundary + lineEnd + "Content-Disposition: form-data; name=\"access_token\"" + lineEnd;
        stringForLength += "Content-Type: text/plain;charset=UTF-8" + lineEnd + "Content-Length: " + accessToken.length() + lineEnd + lineEnd;
        stringForLength += accessToken + lineEnd + twoHyphens + boundary + lineEnd;

        stringForLength += "Content-Disposition: form-data; name=\"text\"" + lineEnd;
        stringForLength += "Content-Type: text/plain;charset=UTF-8" + lineEnd + "Content-Length: " + text.length() + lineEnd + lineEnd;
        stringForLength += text + lineEnd + twoHyphens + boundary + lineEnd;

        stringForLength += "Content-Disposition: form-data; name=\"type\"" + lineEnd;
        stringForLength += "Content-Type: text/plain;charset=UTF-8" + lineEnd + "Content-Length: " + type.length() + lineEnd + lineEnd;
        stringForLength += type + lineEnd + twoHyphens + boundary + lineEnd;

        stringForLength += twoHyphens + boundary + lineEnd + "Content-Disposition: form-data; name=\"image\"" + lineEnd;
        stringForLength += "Content-Type: application/octet-stream" + lineEnd + "Content-Length: " + file.length() + lineEnd + lineEnd;
        stringForLength += lineEnd + twoHyphens + boundary + twoHyphens + lineEnd;

        int totalLength = stringForLength.length() + (int)file.length();           
        conn.setFixedLengthStreamingMode(totalLength); 


        outputStream = new DataOutputStream( conn.getOutputStream() );          
        outputStream.writeBytes(twoHyphens + boundary + lineEnd);

        // access token 

        outputStream.writeBytes("Content-Disposition: form-data; name=\"access_token\"" + lineEnd);
        outputStream.writeBytes("Content-Type: text/plain;charset=UTF-8" + lineEnd);
        outputStream.writeBytes("Content-Length: " + accessToken.length() + lineEnd);
        outputStream.writeBytes(lineEnd);
        outputStream.writeBytes(accessToken + lineEnd);
        outputStream.writeBytes(twoHyphens + boundary + lineEnd);

        // text 

        outputStream.writeBytes("Content-Disposition: form-data; name=\"text\"" + lineEnd);
        outputStream.writeBytes("Content-Type: text/plain;charset=UTF-8" + lineEnd);
        outputStream.writeBytes("Content-Length: " + text.length() + lineEnd);
        outputStream.writeBytes(lineEnd);
        outputStream.writeBytes(text + lineEnd);
        outputStream.writeBytes(twoHyphens + boundary + lineEnd);

        // type 

        outputStream.writeBytes("Content-Disposition: form-data; name=\"type\"" + lineEnd);
        outputStream.writeBytes("Content-Type: text/plain;charset=UTF-8" + lineEnd);
        outputStream.writeBytes("Content-Length: " + type.length() + lineEnd);
        outputStream.writeBytes(lineEnd);
        outputStream.writeBytes(type + lineEnd);
        outputStream.writeBytes(twoHyphens + boundary + lineEnd);

        // image

        outputStream.writeBytes(twoHyphens + boundary + lineEnd);
        outputStream.writeBytes("Content-Disposition: form-data; name=\"image\"" + lineEnd);
        //outputStream.writeBytes(lineEnd);
        outputStream.writeBytes("Content-Type: application/octet-stream" + lineEnd);
        outputStream.writeBytes("Content-Length: " + file.length() + lineEnd);
        outputStream.writeBytes(lineEnd);           

        bytesAvailable = fileInputStream.available();
        bufferSize = Math.min(bytesAvailable, maxBufferSize);
        buffer = new byte[bufferSize];
        // Read file
        bytesRead = fileInputStream.read(buffer, 0, bufferSize);

        while (bytesRead > 0)
        {
        outputStream.write(buffer, 0, bufferSize);          
        bytesAvailable = fileInputStream.available();
        bufferSize = Math.min(bytesAvailable, maxBufferSize);
        bytesRead = fileInputStream.read(buffer, 0, bufferSize);
        }

        outputStream.writeBytes(lineEnd);
        outputStream.writeBytes(twoHyphens + boundary + twoHyphens + lineEnd);

        Log.d("posttemplate", "connection outputstream size is " + outputStream.size());

        // finished with POST request body


     // Responses from the server (code and message)
        serverResponseCode = conn.getResponseCode();
        serverResponseMessage = conn.getResponseMessage();

        Log.d("posttemplate", "server response code "+ serverResponseCode);
        Log.d("posttemplate", "server response message "+ serverResponseMessage);

        fileInputStream.close();
        conn.disconnect();
        outputStream.flush();
        outputStream.close();


    } catch (MalformedURLException e)
    {
        Log.d("posttemplate", "malformed url", e);
        //TODO: catch exception;
    } catch (IOException e)
    {
        Log.d("posttemplate", "ioexception", e);
        //TODO: catch exception
    }        

}

Unfortunately, my app crashes with IOException at outputStream.close(), and I have no idea why:

03-16 13:56:51.035: D/posttemplate(6479): java.io.IOException: unexpected end of stream
03-16 13:56:51.035: D/posttemplate(6479):   at org.apache.harmony.luni.internal.net.www.protocol.http.FixedLengthOutputStream.close(FixedLengthOutputStream.java:57)
03-16 13:56:51.035: D/posttemplate(6479):   at java.io.FilterOutputStream.close(FilterOutputStream.java:66)
03-16 13:56:51.035: D/posttemplate(6479):   at com.futubra.api.impl.PostTemplate.createImagePostWithToken(PostTemplate.java:282)
03-16 13:56:51.035: D/posttemplate(6479):   at com.futubra.FutubraNewPostActivity.createPost(FutubraNewPostActivity.java:128)
03-16 13:56:51.035: D/posttemplate(6479):   at com.futubra.FutubraNewPostActivity_.access$2(FutubraNewPostActivity_.java:1)
03-16 13:56:51.035: D/posttemplate(6479):   at com.futubra.FutubraNewPostActivity_$5.run(FutubraNewPostActivity_.java:141)
03-16 13:56:51.035: D/posttemplate(6479):   at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1088)
03-16 13:56:51.035: D/posttemplate(6479):   at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:581)
03-16 13:56:51.035: D/posttemplate(6479):   at java.lang.Thread.run(Thread.java:1019)

Answer 1

   int totalLength = stringForLength.length() + (int)file.length();  

Your above code is wrong.

You should use the byte length of the string as following

 int totalLength = stringForLength.getBytes().length + (int)file.length();  

Answer 2

HTTP header is not part of the body, so do not account them for body length.

 stringForLength += "Content-Type: multipart/form-data;boundary=" + boundary;

remove the above line, so you content length will be correct.

Answer 3

For your information, your code should work fine with HTTP URL connection, but HTTPS will trigger Out of Memory error because there is a bug in HttpsURLConnectionImpl.java in Android 2.3.4 (verified on my tablet), and this bug has been fixed in Android 4.1 (I have checked the source code).

Answer 4

conn.disconnect() after outputStream.close()