Netty SslHandler throws IllegalReferenceCountException

Question

I'm writing client-server application using using netty 4.0.36.Final

One of test engineers encountered this problem on client side:

io.netty.handler.codec.DecoderException: io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
    at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:418)
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:245)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:292)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:278)
    at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:292)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:278)
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:962)
    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131)
    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:528)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:485)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:399)
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:371)
    at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112)
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
    at java.base/java.lang.Thread.run(Unknown Source)
Caused by: io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
    at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:101)
    at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1072)
    at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:904)
    at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:387)
    ... 15 common frames omitted

Channel pipeline:

@Override
public void initChannel(SocketChannel ch) throws Exception {
    ChannelPipeline pipeline = ch.pipeline();
    pipeline.addLast(sslContext.newHandler(ch.alloc()));
    pipeline.addLast(new AsnMessageDecoder());
    pipeline.addLast(new AsnMessageEncoder());
    pipeline.addLast(dataChannelHandler);
}

At some point I add another handler to pipeline:

channel.pipeline().addFirst("idleStateHandler", new IdleStateHandler(180, 0, 0));

There are two ChannelInboundHandler 's

public class AsnMessageDecoder extends ReplayingDecoder {
private static final Logger log = LoggerFactory.getLogger(AsnMessageDecoder.class);
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
    DERCoder coder = Asn.getDERCoder();
    coder.disableEncoderDebugging();
    coder.disableDecoderDebugging();
    coder.enableEncoderConstraints();
    coder.enableDecoderConstraints();
    coder.enableAutomaticEncoding();
    coder.enableAutomaticDecoding();
    coder.enableContainedValueEncoding();
    coder.enableContainedValueDecoding();
    try (ByteBufInputStream is = new ByteBufInputStream(in)) {
        out.add(coder.decode(is, new Message()));
    } catch (Exception e) {
        log.error("Failed to decode message", e);
    }
}

And another one:

abstract class BaseChannelHandler<T extends ChannelContext> extends SimpleChannelInboundHandler<Message> {
private final Logger log = LoggerFactory.getLogger(getClass());

private final T channelContext;
private final ClientConfiguration clientConfiguration;
private final Writer writer;
private final ClientService service;

BaseChannelHandler(T channelContext,
                   ClientConfiguration clientConfiguration,
                   Writer writer,
                   ClientService service) {
    this.channelContext = channelContext;
    this.clientConfiguration = clientConfiguration;
    this.writer = writer;
    this.service = service;
}

@Override
public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
    log.debug("{} channelRegistered", getClass().getSimpleName());
}

@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
    log.debug("{} channelActive", getClass().getSimpleName());
    channelContext.connectState();
}

@Override
protected void channelRead0(ChannelHandlerContext ctx, Message msg) throws Exception {
    log.debug("{} channelRead0 msg = {}", getClass().getSimpleName(), msg);
    channelContext.handleMessage(msg);
    if (clientConfiguration.isWritingOutputEnabled()) {
        writer.writeXml(msg);
    }
}

@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
    log.error(getClass().getSimpleName() + " exceptionCaught", cause);
    ctx.close();
    service.stop();
}

@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
    log.debug("{} channelInactive", getClass().getSimpleName());
}

@Override
public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
    log.debug("{} channelUnregistered", getClass().getSimpleName());
}

@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
    if (evt instanceof IdleStateEvent) {
        ctx.disconnect();
    }
}

I can't repdoduce problem. On my local machine it works fine. Looking in wirweshark doesn't give me any hint. It looks like normal disconnect. I'm not quite sure what to do next to localize this problem. Can you give me some advice on next step to solve this problem?

Answer 1

So it seems like this was a bug in the netty version (which was quite old). Upgrading to 4.1.24.Final fixed it.