How to fix this "borrowed value does not live long enough" error?

Question

I'm writing a program that detects duplicate frames in h264 encoded video. I'm using the ac-ffmpeg crate Here's my code:

use std::fs::File;

use ac_ffmpeg::{
    codec::{
        video::{frame::Planes, VideoDecoder},
        CodecParameters, Decoder,
    },
    format::{
        demuxer::{Demuxer, DemuxerWithStreamInfo},
        io::IO,
        muxer::{Muxer, OutputFormat},
    },
    Error,
};

fn open_input(path: &str) -> Result<DemuxerWithStreamInfo<File>, Error> {
    let input = File::open(path)
        .map_err(|err| Error::new(format!("unable to open input file {}: {}", path, err)))?;

    let io = IO::from_seekable_read_stream(input);

    Demuxer::builder()
        .build(io)?
        .find_stream_info(None)
        .map_err(|(_, err)| err)
}

fn open_output(path: &str, elementary_streams: &[CodecParameters]) -> Result<Muxer<File>, Error> {
    let output_format = OutputFormat::guess_from_file_name(path)
        .ok_or_else(|| Error::new(format!("unable to guess output format for file: {}", path)))?;

    let output = File::create(path)
        .map_err(|err| Error::new(format!("unable to create output file {}: {}", path, err)))?;

    let io = IO::from_seekable_write_stream(output);

    let mut muxer_builder = Muxer::builder();

    for codec_parameters in elementary_streams {
        muxer_builder.add_stream(codec_parameters)?;
    }

    muxer_builder.build(io, output_format)
}

fn plane_difference(planes1: &Planes, planes2: &Planes) -> u64 {
    6
}

fn remove_duplicate_frames(input: &str, output: &str) -> Result<(), Error> {
    let mut demuxer = open_input(input)?;

    let (stream_index, (stream, _)) = demuxer
        .streams()
        .iter()
        .map(|stream| (stream, stream.codec_parameters()))
        .enumerate()
        .find(|(_, (_, params))| params.is_video_codec())
        .ok_or_else(|| Error::new("no video stream"))?;
    let mut decoder = VideoDecoder::from_stream(stream)?.build()?;

    let mut packet_count = 0;
    let mut prev_planes: Option<Planes> = None;
    let mut diffs = Vec::<(i64, u64)>::new();
    while let Some(packet) = demuxer.take()? {
        if packet.stream_index() != stream_index {
            continue;
        }
        decoder.push(packet.clone())?;

        if let Some(frame) = decoder.take()? {
            let planes = frame.planes();

            match prev_planes {
                Some(prev) => {
                    let diff = plane_difference(&planes, &prev);
                    diffs.push((packet.dts().timestamp() / 1000, diff));
                }
                None => (),
            };

            prev_planes = Some(planes);
        }
        if packet_count > 2000 {
            break;
        }
        packet_count += 1;
    }

    diffs.sort_by(|a, b| b.1.cmp(&a.1));

    dbg!(diffs);
    Ok(())
}

with dependency ac-ffmpeg = "0.17.4".

The problem is this gives an error that frame does not live long enough:

error[E0597]: `frame` does not live long enough
   --> src/main.rs:106:26
    |
106 |             let planes = frame.planes();
    |                          ^^^^^^^^^^^^^^ borrowed value does not live long enough
107 | 
108 |             match prev_planes {
    |                   ----------- borrow later used here
...
117 |         }
    |         - `frame` dropped here while still borrowed

My understanding of this error is that frame is borrowed, which via planes ends up in prev_planes. But at the end of every loop the value is dropped so it doesn't exist for the next iteration. Is that correct? And how can I solve that? I tried cloning various things in this code but I'm not able to fix the error.

Answer 1

VideoFrame doesn't allow you to take the planes, it only allows you to borrow them from the frame.

You can see that because a lifetime is attached to the planes:

pub fn planes(&self) -> Planes<'_>

The '_ lifetime says it's identical to the lifetime of the &self reference, meaning it is borrowed from self.

If you want to store the planes, store the frame that contains them instead:

use std::fs::File;

use ac_ffmpeg::{
    codec::{
        video::{frame::Planes, VideoDecoder, VideoFrame},
        CodecParameters, Decoder,
    },
    format::{
        demuxer::{Demuxer, DemuxerWithStreamInfo},
        io::IO,
        muxer::{Muxer, OutputFormat},
    },
    Error,
};

fn open_input(path: &str) -> Result<DemuxerWithStreamInfo<File>, Error> {
    let input = File::open(path)
        .map_err(|err| Error::new(format!("unable to open input file {}: {}", path, err)))?;

    let io = IO::from_seekable_read_stream(input);

    Demuxer::builder()
        .build(io)?
        .find_stream_info(None)
        .map_err(|(_, err)| err)
}

fn open_output(path: &str, elementary_streams: &[CodecParameters]) -> Result<Muxer<File>, Error> {
    let output_format = OutputFormat::guess_from_file_name(path)
        .ok_or_else(|| Error::new(format!("unable to guess output format for file: {}", path)))?;

    let output = File::create(path)
        .map_err(|err| Error::new(format!("unable to create output file {}: {}", path, err)))?;

    let io = IO::from_seekable_write_stream(output);

    let mut muxer_builder = Muxer::builder();

    for codec_parameters in elementary_streams {
        muxer_builder.add_stream(codec_parameters)?;
    }

    muxer_builder.build(io, output_format)
}

fn plane_difference(planes1: &Planes, planes2: &Planes) -> u64 {
    6
}

fn remove_duplicate_frames(input: &str, output: &str) -> Result<(), Error> {
    let mut demuxer = open_input(input)?;

    let (stream_index, (stream, _)) = demuxer
        .streams()
        .iter()
        .map(|stream| (stream, stream.codec_parameters()))
        .enumerate()
        .find(|(_, (_, params))| params.is_video_codec())
        .ok_or_else(|| Error::new("no video stream"))?;
    let mut decoder = VideoDecoder::from_stream(stream)?.build()?;

    let mut packet_count = 0;
    let mut prev_frame: Option<VideoFrame> = None;
    let mut diffs = Vec::<(i64, u64)>::new();
    while let Some(packet) = demuxer.take()? {
        if packet.stream_index() != stream_index {
            continue;
        }
        decoder.push(packet.clone())?;

        if let Some(frame) = decoder.take()? {
            let planes = frame.planes();

            match prev_frame {
                Some(prev) => {
                    let diff = plane_difference(&planes, &prev.planes());
                    diffs.push((packet.dts().timestamp() / 1000, diff));
                }
                None => (),
            };

            prev_frame = Some(frame);
        }
        if packet_count > 2000 {
            break;
        }
        packet_count += 1;
    }

    diffs.sort_by(|a, b| b.1.cmp(&a.1));

    dbg!(diffs);
    Ok(())
}

Answer 2

You need to keep the frame itself from one loop iteration to the next instead of the planes. So have a prev_frame variable instead of prev_planes:

let mut prev_frame = None;
let mut diffs = Vec::<(i64, u64)>::new();
while let Some(packet) = demuxer.take()? {
    if packet.stream_index() != stream_index {
        continue;
    }
    decoder.push(packet.clone())?;

    if let Some(frame) = decoder.take()? {
        let planes = frame.planes();

        match prev_frame {
            Some(prev) => {
                let diff = plane_difference(&planes, &prev.planes());
                diffs.push((packet.dts().timestamp() / 1000, diff));
            }
            None => (),
        };

        prev_frame = Some(frame);
    }
}

Answer 3

It's a bit tricky to reproduce this, so I'll try to answer from the top of my head.

Did you try:

let planes = frame.planes().to_owned();

Mainly you're giving ownership of planes (i.e. the child), while the parent (frame) goes out of scope.

Why to_owned? (if the child is a ref, clone will return a ref, so we use to_owned instead since it converts a ref type to an owned type where it's allowed to transfer ownership.

The solution should be either:

let planes = frame.planes().to_owned();   // line 106

or

prev_planes = Some(planes.clone());       // line 116

If none of these work, try updating your question with their outputs. It'll make it easier to answer.