The netty server seems to be blocked when I add a ExecutionHandler?

Question

THE SCENE:

I am writing a echo client and server. The data being transfered is a string:

Client encode a string,and send it to server. Server recv data, decode string, then encode the received string, send it back to client.

The above process will be repeated 100000 times.(Note: the connection is persistent).

DEFERENT CONTIONS:

When I run ONE server and TWO client at the same time, everything is ok,every client receives 100000 messages and terminated normally.

But When I add a ExecutionHandler on server, and then run ONE server and TWO client at the same time, one client will never terminate, and the network traffic is zero.

I cant locate the key point of this problem for now, will you give me some suggestions?

MY CODE:

string encoder , string decoder, client handler , server handler , client main ,server main.

//Decoder=======================================================

import java.nio.charset.Charset;

import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.frame.FrameDecoder;

public class Dcd extends FrameDecoder {
    public static final Charset cs = Charset.forName("utf8");

    @Override
    protected Object decode(ChannelHandlerContext ctx, Channel channel,
            ChannelBuffer buffer) throws Exception {

        if (buffer.readableBytes() < 4) {
            return null;
        }

        int headlen = 4;
        int length = buffer.getInt(0);
        if (buffer.readableBytes() < length + headlen) {
            return null;
        }

        String ret = buffer.toString(headlen, length, cs);
        buffer.skipBytes(length + headlen);

        return ret;
    }
}

//Encoder =======================================================

import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.oneone.OneToOneEncoder;

public class Ecd extends OneToOneEncoder {
    @Override
    protected Object encode(ChannelHandlerContext ctx, Channel channel,
            Object msg) throws Exception {
        if (!(msg instanceof String)) {
            return msg;
        }

        byte[] data = ((String) msg).getBytes();

        ChannelBuffer buf = ChannelBuffers.dynamicBuffer(data.length + 4, ctx
                .getChannel().getConfig().getBufferFactory());
        buf.writeInt(data.length);
        buf.writeBytes(data);

        return buf;
    }
}

//Client handler =======================================================

import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;

/**
 * Handler implementation for the echo client. It initiates the ping-pong
 * traffic between the echo client and server by sending the first message to
 * the server.
 */
public class EchoClientHandler extends SimpleChannelUpstreamHandler {

    private static final Logger logger = Logger
            .getLogger(EchoClientHandler.class.getName());

    private final AtomicLong transferredBytes = new AtomicLong();
    private final AtomicInteger counter = new AtomicInteger(0);
    private final AtomicLong startTime = new AtomicLong(0);

    private String dd;

    /**
     * Creates a client-side handler.
     */
    public EchoClientHandler(String data) {
        dd = data;
    }

    public long getTransferredBytes() {
        return transferredBytes.get();
    }

    @Override
    public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
        // Send the first message. Server will not send anything here
        // because the firstMessage's capacity is 0.
        startTime.set(System.currentTimeMillis());

        Channels.write(ctx.getChannel(), dd);
    }

    @Override
    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
        // Send back the received message to the remote peer.
        transferredBytes.addAndGet(((String) e.getMessage()).length());
        int i = counter.incrementAndGet();
        int N = 100000;
        if (i < N) {
            e.getChannel().write(e.getMessage());
        } else {
            ctx.getChannel().close();
            System.out.println(N * 1.0
                    / (System.currentTimeMillis() - startTime.get()) * 1000);
        }
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
        // Close the connection when an exception is raised.
        logger.log(Level.WARNING, "Unexpected exception from downstream.",
                e.getCause());
        e.getChannel().close();
    }
}

//Client main =======================================================

import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;

import org.jboss.netty.bootstrap.ClientBootstrap;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;

/**
 * Sends one message when a connection is open and echoes back any received data
 * to the server. Simply put, the echo client initiates the ping-pong traffic
 * between the echo client and server by sending the first message to the
 * server.
 */
public class EchoClient {

    private final String host;
    private final int port;

    public EchoClient(String host, int port) {
        this.host = host;
        this.port = port;
    }

    public void run() {
        // Configure the client.
        final ClientBootstrap bootstrap = new ClientBootstrap(
                new NioClientSocketChannelFactory(
                        Executors.newCachedThreadPool(),
                        Executors.newCachedThreadPool()));

        // Set up the pipeline factory.
        bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
            public ChannelPipeline getPipeline() throws Exception {
                return Channels.pipeline(new Dcd(), new Ecd(),
                        new EchoClientHandler("abcdd"));
            }
        });

        bootstrap.setOption("sendBufferSize", 1048576);
        bootstrap.setOption("receiveBufferSize", 1048576);
        bootstrap.setOption("tcpNoDelay", true);
        bootstrap.setOption("writeBufferLowWaterMark", 32 * 1024);
        bootstrap.setOption("writeBufferHighWaterMark", 64 * 1024);

        List<ChannelFuture> list = new ArrayList<ChannelFuture>();
        for (int i = 0; i < 1; i++) {
            // Start the connection attempt.
            ChannelFuture future = bootstrap.connect(new InetSocketAddress(
                    host, port));
            // Wait until the connection is closed or the connection
            // attempt
            // fails.
            list.add(future);
        }

        for (ChannelFuture f : list) {
            f.getChannel().getCloseFuture().awaitUninterruptibly();
        }

        // Shut down thread pools to exit.
        bootstrap.releaseExternalResources();
    }

    private static void testOne() {
        final String host = "192.168.0.102";
        final int port = 8000;

        new EchoClient(host, port).run();
    }

    public static void main(String[] args) throws Exception {
        testOne();
    }
}

//server handler =======================================================

import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelUpstreamHandler;

/**
 * Handler implementation for the echo server.
 */
public class EchoServerHandler extends SimpleChannelUpstreamHandler {

    private static final Logger logger = Logger
            .getLogger(EchoServerHandler.class.getName());

    private final AtomicLong transferredBytes = new AtomicLong();

    public long getTransferredBytes() {
        return transferredBytes.get();
    }

    @Override
    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
        // Send back the received message to the remote peer.
        transferredBytes.addAndGet(((String) e.getMessage()).length());
        Channels.write(ctx.getChannel(), e.getMessage());
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
        // Close the connection when an exception is raised.
        logger.log(Level.WARNING, "Unexpected exception from downstream.",
                e.getCause());
        e.getChannel().close();
    }
}

//Server main =======================================================

import java.net.InetSocketAddress;
import java.util.concurrent.Executors;

import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import org.jboss.netty.handler.execution.ExecutionHandler;
import org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor;

/**
 * Echoes back any received data from a client.
 */
public class EchoServer {

    private final int port;

    public EchoServer(int port) {
        this.port = port;
    }

    public void run() {
        // Configure the server.
        ServerBootstrap bootstrap = new ServerBootstrap(
                new NioServerSocketChannelFactory(
                        Executors.newCachedThreadPool(),
                        Executors.newCachedThreadPool()));

        System.out.println(Runtime.getRuntime().availableProcessors() * 2);

        final ExecutionHandler executionHandler = new ExecutionHandler(
                new OrderedMemoryAwareThreadPoolExecutor(16, 1048576, 1048576));

        // Set up the pipeline factory.
        bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
            public ChannelPipeline getPipeline() throws Exception {
                System.out.println("new pipe");
                return Channels.pipeline(new Dcd(), new Ecd(),
                        executionHandler, new EchoServerHandler());
            }
        });

        bootstrap.setOption("child.sendBufferSize", 1048576);
        bootstrap.setOption("child.receiveBufferSize", 1048576);
        bootstrap.setOption("child.tcpNoDelay", true);
        bootstrap.setOption("child.writeBufferLowWaterMark", 32 * 1024);
        bootstrap.setOption("child.writeBufferHighWaterMark", 64 * 1024);

        // Bind and start to accept incoming connections.
        bootstrap.bind(new InetSocketAddress(port));
    }

    public static void main(String[] args) throws Exception {
        int port = 8000;
        new EchoServer(port).run();
    }
}

Answer 1

This was a bug and will be fixed in 3.4.0.Alpha2.

See https://github.com/netty/netty/issues/234

Answer 2

I have found the reason now, it is a hard work, but full with pleasure.

When added a ExecutionHandler, the message will be wrapped into a Runnable task, and will be executed in a ChildExecutor. The key point is here : A task maybe added to ChildExecutor when the executor almostly exit , then is will be ignored by the ChildExecutor.

I added three lines code and some comments, the final code looks like below, and it works now,should I mail to the author? :

private final class ChildExecutor implements Executor, Runnable {
    private final Queue<Runnable> tasks = QueueFactory
            .createQueue(Runnable.class);
    private final AtomicBoolean isRunning = new AtomicBoolean();

    public void execute(Runnable command) {
        // TODO: What todo if the add return false ?
        tasks.add(command);

        if (!isRunning.get()) {
            doUnorderedExecute(this);
        } else {
        }
    }

    public void run() {
        // check if its already running by using CAS. If so just return
        // here. So in the worst case the thread
        // is executed and do nothing
        boolean acquired = false;
        if (isRunning.compareAndSet(false, true)) {
            acquired = true;
            try {
                Thread thread = Thread.currentThread();
                for (;;) {
                    final Runnable task = tasks.poll();
                    // if the task is null we should exit the loop
                    if (task == null) {
                        break;
                    }

                    boolean ran = false;
                    beforeExecute(thread, task);
                    try {
                        task.run();
                        ran = true;
                        onAfterExecute(task, null);
                    } catch (RuntimeException e) {
                        if (!ran) {
                            onAfterExecute(task, e);
                        }
                        throw e;
                    }
                }
                //TODO  NOTE (I added): between here and "isRunning.set(false)",some new tasks maybe added.
            } finally {
                // set it back to not running
                isRunning.set(false);
            }
        }

        //TODO NOTE (I added): Do the remaining works.
        if (acquired && !isRunning.get() && tasks.peek() != null) {
            doUnorderedExecute(this);
        }
    }
}