Android merging PCM and MP3 into AAC

Question

I have a program that records user input sound as PCM (I needed to do it separately to "play" with the voice) and then I also have a custom audio track, which is in MP3 format, that I want merge with the PCM file.

To start off I convert both of them to WAV separately, then I combine the 2 WAV files, and finally convert the result to AAC because I also need to merge the audio with video later.

I tried merging 2 AAC files, but that did not work out for me.

For audio conversion I am using FFmpeg-Android.

The problem is that it takes way too long, around 1-2min to do the whole conversion and because of that I need a new way to do it all. I have looked into other libraries but this was the only one I could get to work.

Can someone recommend something that would do the whole process faster?

Here is my code to merge all of the files:

public class AudioProcessor {

    private Context context;
    private FFmpeg ffmpeg;
    private AudioProcessorListener listener;

    private File micPcmFile;
    private File backgroundMp3File;

    private File pcmtowavTempFile;
    private File mp3towavTempFile;
    private File combinedwavTempFile;

    private File outputFile;
    private File volumeChangedTempFile;

    private FFtask currentTask;

    private int videoRecordingLength = 0;

    TextView extensionDownload, percentProgress;

    private static final String TAG = "FFMPEG AV Processor";

    public AudioProcessor(Context context, Activity activity) {
        ffmpeg = null;
        ffmpeg = FFmpeg.getInstance(context);
        percentProgress = activity.findViewById(R.id.percentProgress);
        percentProgress.setSingleLine(false);
        this.context = context;
        prepare();
    }

    /**
     * Program main method. Starts running program
     * @throws Exception
     */
    public void process() throws Exception {
        if (!ffmpeg.isSupported()) {
            Log.e(TAG, "FFMPEG not supported! Cannot convert audio!");
            throw new RuntimeException("FFMPeg has to be supported");
        }
        if (!checkIfAllFilesPresent()) {
            Log.e(TAG, "All files are not set yet. Please set file first");
            throw new RuntimeException("Files are not set!");
        }

        Log.e(TAG, "Start processing audio!");
        listener.onStart();

        Handler handler = new Handler();
        handler.postDelayed(new Runnable() {
            @Override
            public void run() {
                convertPCMToWav();
            }
        }, 200);
    }

    /**
     * Prepares program
     */
    private void prepare() {
        Log.d(TAG, "Preparing everything...");
        prepareTempFiles();
    }

    /**
     * Converts PCM to wav file. Automatically create new file.
     */
    private void convertPCMToWav() {
        Log.d(TAG, "Convert PCM TO Wav");
        //ffmpeg -f s16le -ar 44.1k -ac 2 -i file.pcm file.wav
        String[] cmd = { "-f" , "s16le", "-ar", "44.1k", "-i", micPcmFile.toString(), "-y", pcmtowavTempFile.toString()};
        currentTask = ffmpeg.execute(cmd, new ExecuteBinaryResponseHandler() {

            @Override
            public void onStart() {
                super.onStart();
                percentProgress.setVisibility(View.VISIBLE);
                percentProgress.setText("Converting your recording\n"+"1/5");
            }

            @Override
            public void onSuccess(String message) {
                super.onSuccess(message);
                convertMP3ToWav();
            }
            @Override
            public void onFailure(String message) {
                super.onFailure(message);
                onError(message);
                convertPCMToWav();
            }
        });
    }

    /**
     * Converts mp3 file to wav file.
     * Automatically creates Wav file
     */
    private void convertMP3ToWav() {
        Log.e(TAG, "Convert MP3 TO Wav");

        //ffmpeg -ss 0 -t 30 -i file.mp3 file.wav
        //String[] cmd = { "-ss", "0", "-t", Integer.toString(videoRecordingLength), "-i" , backgroundMp3File.toString(), "-y", mp3towavTempFile.toString() };
        String[] cmd = {  "-i" , backgroundMp3File.toString(), "-y", mp3towavTempFile.toString() };
        currentTask = ffmpeg.execute(cmd, new ExecuteBinaryResponseHandler() {
            @Override
            public void onStart() {
                super.onStart();
                percentProgress.setText("Converting background audio\n"+"2/5");
                Log.d(TAG, "Convert MP3 TO Wav");
            }
            @Override
            public void onSuccess(String message) {
                super.onSuccess(message);
                changeMicAudio();
            }
            @Override
            public void onFailure(String message) {
                super.onFailure(message);
                Log.e(TAG, "Failed to convert MP3 TO Wav");
                onError(message);
                throw new RuntimeException("Failed to convert MP3 TO Wav");
            }
        });
    }

    /**
     * Combines 2 wav files into one wav file. Overlays audio
     */
    private void combineWavs() {
        Log.e(TAG, "Combine wavs");
        //ffmpeg -i C:\Users\VR1\Desktop\_mp3.wav -i C:\Users\VR1\Desktop\_pcm.wav -filter_complex amix=inputs=2:duration=first:dropout_transition=3 C:\Users\VR1\Desktop\out.wav

        String[] cmd = { "-i" , pcmtowavTempFile.toString(), "-i", volumeChangedTempFile.toString(), "-filter_complex", "amix=inputs=2:duration=first:dropout_transition=3", "-y",combinedwavTempFile.toString()};
        currentTask = ffmpeg.execute(cmd, new ExecuteBinaryResponseHandler() {
            @Override
            public void onStart() {
                super.onStart();
                percentProgress.setText("Combining the two audio files\n"+"4/5");
            }

            @Override
            public void onSuccess(String message) {
                super.onSuccess(message);
                encodeWavToAAC();

            }
            @Override
            public void onFailure(String message) {
                super.onFailure(message);
                onError(message);
            }
        });
    }

    private void changeMicAudio(){
        Log.e(TAG, "Change audio volume");
        //ffmpeg -i input.wav -filter:a "volume=1.5" output.wav

        String[] cmdy = { "-i", mp3towavTempFile.toString(),  "-af", "volume=0.9", "-y",volumeChangedTempFile.toString()};
        currentTask = ffmpeg.execute(cmdy, new ExecuteBinaryResponseHandler() {

            @Override
            public void onStart() {
                super.onStart();
                percentProgress.setText("Normalizing volume\n"+"3/5");
            }

            @Override
            public void onSuccess(String message) {
                combineWavs();
                super.onSuccess(message);
            }
            @Override
            public void onFailure(String message) {
                super.onFailure(message);
                Log.e("AudioProcessor", message);
            }
        });
    }


    /**
     * Do something on error. Releases program data (deletes files)
     * @param message
     */
    private void onError(String message) {
        completed();
        if (listener != null) {
            //listener.onError(message);
        }
    }
    /**
     * Encode to AAC
     */
    private void encodeWavToAAC() {
        Log.d(TAG, "Encode Wav file to AAC");
        //ffmpeg -i file.wav -c:a aac -b:a 128k -f adts output.m4a
        String[] cmd = { "-i" , combinedwavTempFile.toString(), "-c:a", "aac", "-b:a", "128k", "-f", "adts", "-y",outputFile.toString()};
        currentTask = ffmpeg.execute(cmd, new ExecuteBinaryResponseHandler() {
            @Override
            public void onStart() {
                super.onStart();
                percentProgress.setText("Normalizing volume\n"+"3/5");
            }

            @Override
            public void onSuccess(String message) {
                super.onSuccess(message);
                if (listener != null) {
                    listener.onSuccess(outputFile);
                }
                completed();
            }
            @Override
            public void onFailure(String message) {
                super.onFailure(message);
                onError(message);
                encodeWavToAAC();
            }
        });
    }

    /**
     * Uninitializes class
     */
    private void completed() {
        if (listener != null) {
            listener.onFinish();
        }
        Log.d(TAG, "Process completed successfully!");
        destroyTempFiles();
    }

    /**
     * Prepares temp required files by deleteing them if they exsist.
     * Files cannot exists before ffmpeg actions. FFMpeg automatically creates those files.
     */
    private void prepareTempFiles() {
        Log.d(TAG, "Preparing Temp files...");
        pcmtowavTempFile = new File(context.getFilesDir()+ Common.TEMP_LOCAL_DIR + "/" + "_pcm.wav");
        mp3towavTempFile = new File(context.getFilesDir()+ Common.TEMP_LOCAL_DIR + "/" + "_mp3.wav");
        combinedwavTempFile = new File(context.getFilesDir()+ Common.TEMP_LOCAL_DIR + "/" + "_combined.wav");
        volumeChangedTempFile = new File(context.getFilesDir()+ Common.TEMP_LOCAL_DIR + "/" + "_volumeChanged.wav");
    }

    /**
     * Destroys temp required files
     */
    private void destroyTempFiles() {
        Log.d(TAG, "Destroying Temp files...");
        pcmtowavTempFile.delete();
        mp3towavTempFile.delete();
        combinedwavTempFile.delete();
        volumeChangedTempFile.delete();
        Log.d(TAG, "Destroying files completed!");
    }


    /**
     * Checks if all files are set, so we can process them
     * @return - all files ready
     */
    private boolean checkIfAllFilesPresent() {
        if(micPcmFile == null || backgroundMp3File == null || outputFile == null) {
            Log.e(TAG, "All files are not set! Set all files!");
            throw new RuntimeException("Output file is not present!");
        }
        Log.d(TAG, "All files are present!");
        return true;
    }

    public void setOutputFile(File outputFile) {
        this.outputFile = outputFile;
    }

    public void setListener(AudioProcessorListener listener) {
        this.listener = listener;
    }

    public void setMicPcmFile(File micPcmFile) {
        this.micPcmFile = micPcmFile;
    }

    public void setBackgroundMp3File(File backgroundMp3File) {
        this.backgroundMp3File = backgroundMp3File;
    }

    public void setVideoRecordingLength(int seconds) {
        this.videoRecordingLength = seconds;
    }

    /**
     * Quits current processing ffmpeg task
     */
    public void killCurrentTask() {
        if (currentTask != null) {
            currentTask.killRunningProcess();
        }
    }

    public interface AudioProcessorListener {
        void onStart();
        void onSuccess(File output);
        void onError(String message);
        void onFinish();
    }
}

Answer 1

You can combine all commands in one:

String[] cmd = { "-f" , "s16le", "-ar", "44.1k", "-i", micPcmFile.toString(), "-i" , backgroundMp3File.toString(), "-filter_complex", "[1]volume=0.9[a];[0][a]amix=inputs=2:duration=first:dropout_transition=3", "-c:a", "aac", "-b:a", "128k", "-f", "adts", "-y", "-vn", outputFile.toString()};