FFMpeg process created from Java on CentOS doesn't exit

Question

I need to convert a lot of wave files simultaneously. About 300 files in parallel. And new files come constantly. I use ffmpeg process call from my Java 1.8 app, which is running on CentOS. I know that I have to read error and input streams for making created process from Java possible to exit.

My code after several expirements:

    private void ffmpegconverter(String fileIn, String fileOut){
    String[] comand = new String[]{"ffmpeg", "-v", "-8", "-i", fileIn, "-acodec", "pcm_s16le", fileOut};

    Process process = null;
    BufferedReader reader = null;
    try {
        ProcessBuilder pb = new ProcessBuilder(comand);
        pb.redirectErrorStream(true);
        process = pb.start();

        //Reading from error and standard output console buffer of process. Or it could halts because of nobody
        //reads its buffer
        reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
        String s;
        //noinspection StatementWithEmptyBody
        while ((s = reader.readLine()) != null) {
            log.info(Thread.currentThread().getName() + " with fileIn " + fileIn + " and fileOut " + fileOut + " writes " + s);
            //Ignored as we just need to empty the output buffer from process
        }
        log.info(Thread.currentThread().getName() + " ffmpeg process will be waited for");
        if (process.waitFor( 10, TimeUnit.SECONDS )) {
            log.info(Thread.currentThread().getName() + " ffmpeg process exited normally");
        } else {
            log.info(Thread.currentThread().getName() + " ffmpeg process timed out and will be killed");
        }

    } catch (IOException | InterruptedException e) {
        log.error(Thread.currentThread().getName() + "Error during ffmpeg process executing", e);
    } finally {
        if (process != null) {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                    log.error("Error during closing the process streams reader", e);
                }
            }
            try {
                process.getOutputStream().close();
            } catch (IOException e) {
                log.error("Error during closing the process output stream", e);
            }
            process.destroyForcibly();
            log.info(Thread.currentThread().getName() + " ffmpeg process " + process + " must be dead now");
        }
    }
}

If I run separate test with this code it goes normally. But in my app I have hundreds of RUNNING deamon threads "process reaper" which are waiting for ffmpeg process finish. In my real app ffpmeg is started from timer thread. Also I have another activity in separate threads, but I don't think that this is the problem. Max CPU consume is about 10%.

Here is that I usual see in thread dump:

"process reaper" #454 daemon prio=10 os_prio=0 tid=0x00007f641c007000 nid=0x5247 runnable [0x00007f63ec063000]
   java.lang.Thread.State: RUNNABLE
    at java.lang.UNIXProcess.waitForProcessExit(Native Method)
    at java.lang.UNIXProcess.lambda$initStreams$3(UNIXProcess.java:289)
    at java.lang.UNIXProcess$$Lambda$32/2113551491.run(Unknown Source)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)

What am I doing wrong?

UPD: My app accepts a lot of connects with voice traffic. So I have about 300-500 another "good" threads in every moment. Could it be the reason? Deamon threads have low priority. But I don't beleive that they really can't do their jobs in one hour. Ususally it takes some tens of millis.

UPD2: My synthetic test that runs fine. I tried with new threads option and without it just with straigt calling of run method.

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;

public class FFmpegConvert {

    public static void main(String[] args) throws Exception {

        FFmpegConvert f = new FFmpegConvert();
        f.processDir(args[0], args[1], args.length > 2);
    }

    private void processDir(String dirPath, String dirOutPath, boolean isNewThread) {
        File dir = new File(dirPath);
        File dirOut = new File(dirOutPath);
        if(!dirOut.exists()){
            dirOut.mkdir();
        }
        for (int i = 0; i < 1000; i++) {
            for (File f : dir.listFiles()) {
                try {
                    System.out.println(f.getName());
                    FFmpegRunner fFmpegRunner = new FFmpegRunner(f.getAbsolutePath(), dirOut.getAbsolutePath() + "/" + System.currentTimeMillis() + f.getName());
                    if (isNewThread) {
                        new Thread(fFmpegRunner).start();
                    } else {
                        fFmpegRunner.run();
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }

    class FFmpegRunner implements Runnable {
        private String fileIn;
        private String fileOut;

        FFmpegRunner(String fileIn, String fileOut) {
            this.fileIn = fileIn;
            this.fileOut = fileOut;
        }

        @Override
        public void run() {
            try {
                ffmpegconverter(fileIn, fileOut);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        private void ffmpegconverter(String fileIn, String fileOut) throws Exception{
            String[] comand = new String[]{"ffmpeg", "-i", fileIn, "-acodec", "pcm_s16le", fileOut};

            Process process = null;
            try {
                ProcessBuilder pb = new ProcessBuilder(comand);
                pb.redirectErrorStream(true);
                process = pb.start();

                //Reading from error and standard output console buffer of process. Or it could halts because of nobody
                //reads its buffer
                BufferedReader reader =
                        new BufferedReader(new InputStreamReader(process.getInputStream()));
                String line;
                //noinspection StatementWithEmptyBody
                while ((line = reader.readLine()) != null) {
                    System.out.println(line);
                    //Ignored as we just need to empty the output buffer from process
                }

                process.waitFor();
            } catch (IOException | InterruptedException e) {
                throw e;
            } finally {
                if (process != null)
                    process.destroy();
            }
        }

    }

}

UPD3: Sorry, I forgot to notice that I see the work of all these process - they created new converted files but anyway don't exit.

Answer 1

Got it! In some cases ffmpeg wants to ask me should it override already existed file. In my code I close outputstream of this child process. But as it appears this only closes outputstream for Java but not for the process.

So my solution is to make ffmpeg silent at all: no output from this process with "-v -8", no asking question with default "Yes" "-y".

Answer 2

Your application is I/O bound, not CPU bound. If all your files are in the same HDD, then opening simultaneously 300 files will definitely degrade the performance. (that is a likely reason on why you have hundreds of processes waiting).

If I understood correctly, you mentioned that processing 1 file takes some tens of millis?

(and this is doing sequential reads - the fastest that your HDD will read a file)

in this case, processing 300 files sequentially should take no more than 30 seconds.

100 millis - process 1 file

1 second - process 10 files

30 second - process 300 files

EDIT

I did 2 simple changes to your sample code (I removed the first loop, then changed the codec) finally I put one song in "ogg" format in "/tmp/origin" directory. now the program works well).

see code below:

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;

public class FFMpegConvert {

    public static void main(String[] args) throws Exception {

        FFMpegConvert f = new FFMpegConvert();
        f.processDir("/tmp/origin", "/tmp/destination", false);
    }

    private void processDir(String dirPath, String dirOutPath, boolean isNewThread) {
        File dir = new File(dirPath);
        File dirOut = new File(dirOutPath);
        if (!dirOut.exists()) {
            dirOut.mkdir();
        }

        for (File f : dir.listFiles()) {
            try {
                System.out.println(f.getName());
                FFmpegRunner fFmpegRunner = new FFmpegRunner(f.getAbsolutePath(), dirOut.getAbsolutePath() + "/" + System.currentTimeMillis() + f.getName());
                if (isNewThread) {
                    new Thread(fFmpegRunner).start();
                } else {
                    fFmpegRunner.run();
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

    }

    class FFmpegRunner implements Runnable {
        private String fileIn;
        private String fileOut;

        FFmpegRunner(String fileIn, String fileOut) {
            this.fileIn = fileIn;
            this.fileOut = fileOut;
        }

        @Override
        public void run() {
            try {
                ffmpegconverter(fileIn, fileOut);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        private void ffmpegconverter(String fileIn, String fileOut) throws Exception {
            String[] comand = new String[]{"ffmpeg", "-i", fileIn, "-acodec", "copy", fileOut};

            Process process = null;
            try {
                ProcessBuilder pb = new ProcessBuilder(comand);
                pb.redirectErrorStream(true);
                process = pb.start();

                //Reading from error and standard output console buffer of process. Or it could halts because of nobody
                //reads its buffer
                BufferedReader reader =
                        new BufferedReader(new InputStreamReader(process.getInputStream()));
                String line;
                //noinspection StatementWithEmptyBody
                while ((line = reader.readLine()) != null) {
                    System.out.println(line);
                    //Ignored as we just need to empty the output buffer from process
                }

                process.waitFor();
            } catch (IOException | InterruptedException e) {
                throw e;
            } finally {
                if (process != null)
                    process.destroy();
            }
        }

    }

}