Open CV RTSP camera buffer lag

Question

I'm struggling to understand why I cant get a "LIVE" feed from my IP camera.

It appears that there is a buffer and it causes the frames to build up if not being read - and as each iteration of my code takes some time there is a backlog and it ends up being almost slow mo to whats actually happening.

I found the below code which triggers a thread to do the reading of the camera on a loop to try and avoid this. But now i'm getting a "LIVE" feed for around 5 frames and then it stalls and shows the same image for another few.

##camera class - this stops the RTSP feed getting caught in the buffer 

class Camera:

    def __init__(self, rtsp_link):

        #init last ready and last frame
        self.last_frame = None
        self.last_ready = None
        self.lock = Lock()

        #set capture decive
        capture = cv2.VideoCapture(rtsp_link,apiPreference=cv2.CAP_FFMPEG)

        #set thread to clear buffer
        thread = threading.Thread(target=self.rtsp_cam_buffer, args=(capture,), name="rtsp_read_thread")
        thread.daemon = True
        thread.start()

        #delay start of next step to avoid errors
        time.sleep(2)

    def rtsp_cam_buffer(self, capture):
        #loop forever 
        while True:
            with self.lock:           
                capture.grab()
                self.last_ready, self.last_frame = capture.retrieve()


    def getFrame(self):        
        #get last frame
        if (self.last_ready is not None) and (self.last_frame is not None):
            return self.last_frame.copy())
        else:
            return None

Whats the correct thing to do in this situation? Is there a way round this?

OR

Should I use something like gstreamer or ffmpeg to get the camera feed? If so which is better and why? Any advice or pages to give me some python examples of getting it working? I couldn't find loads about that made sense to me.

thanks

Answer 1

Lewis's solution was helpful to reduce the lag so far but there was still some lag in my case, and I have found this gist, which is a bit faster:

import os
import sys
import time
import threading
import numpy as np
import cv2 as cv

# also acts (partly) like a cv.VideoCapture
class FreshestFrame(threading.Thread):
    def __init__(self, capture, name='FreshestFrame'):
        self.capture = capture
        assert self.capture.isOpened()

        # this lets the read() method block until there's a new frame
        self.cond = threading.Condition()

        # this allows us to stop the thread gracefully
        self.running = False

        # keeping the newest frame around
        self.frame = None

        # passing a sequence number allows read() to NOT block
        # if the currently available one is exactly the one you ask for
        self.latestnum = 0

        # this is just for demo purposes        
        self.callback = None
        
        super().__init__(name=name)
        self.start()

    def start(self):
        self.running = True
        super().start()

    def release(self, timeout=None):
        self.running = False
        self.join(timeout=timeout)
        self.capture.release()

    def run(self):
        counter = 0
        while self.running:
            # block for fresh frame
            (rv, img) = self.capture.read()
            assert rv
            counter += 1

            # publish the frame
            with self.cond: # lock the condition for this operation
                self.frame = img if rv else None
                self.latestnum = counter
                self.cond.notify_all()

            if self.callback:
                self.callback(img)

    def read(self, wait=True, seqnumber=None, timeout=None):
        # with no arguments (wait=True), it always blocks for a fresh frame
        # with wait=False it returns the current frame immediately (polling)
        # with a seqnumber, it blocks until that frame is available (or no wait at all)
        # with timeout argument, may return an earlier frame;
        #   may even be (0,None) if nothing received yet

        with self.cond:
            if wait:
                if seqnumber is None:
                    seqnumber = self.latestnum+1
                if seqnumber < 1:
                    seqnumber = 1
                
                rv = self.cond.wait_for(lambda: self.latestnum >= seqnumber, timeout=timeout)
                if not rv:
                    return (self.latestnum, self.frame)

            return (self.latestnum, self.frame)

And then you use it like:

# open some camera
    cap = cv.VideoCapture('rtsp://URL')
    cap.set(cv.CAP_PROP_FPS, 30)

    # wrap it
    fresh = FreshestFrame(cap)

Use fresh to deal with the open camera

Answer 2

After searching online through multiple resources the suggestion for using threads to remove frames from the buffer came up ALOT. And although it seemed to work for a while it caused me issues with duplicate frames being displayed for some reason that I could not work out.

I then tried to build opencv from source with gstreamer support but even once it was compiled correctly it still didn't seem to like interfacing with gstreamer correctly.

Eventually I thought the best bet was to go back down the threading approach but again couldnt get it working. So I gave multiprocessing a shot.

I wrote the below class to handle the camera connection:

import cv2
import time
import multiprocessing as mp

class Camera():
    
    def __init__(self,rtsp_url):        
        #load pipe for data transmittion to the process
        self.parent_conn, child_conn = mp.Pipe()
        #load process
        self.p = mp.Process(target=self.update, args=(child_conn,rtsp_url))        
        #start process
        self.p.daemon = True
        self.p.start()
        
    def end(self):
        #send closure request to process
        
        self.parent_conn.send(2)
        
    def update(self,conn,rtsp_url):
        #load cam into seperate process
        
        print("Cam Loading...")
        cap = cv2.VideoCapture(rtsp_url,cv2.CAP_FFMPEG)   
        print("Cam Loaded...")
        run = True
        
        while run:
            
            #grab frames from the buffer
            cap.grab()
            
            #recieve input data
            rec_dat = conn.recv()
            
            
            if rec_dat == 1:
                #if frame requested
                ret,frame = cap.read()
                conn.send(frame)
                
            elif rec_dat ==2:
                #if close requested
                cap.release()
                run = False
                
        print("Camera Connection Closed")        
        conn.close()
    
    def get_frame(self,resize=None):
        ###used to grab frames from the cam connection process
        
        ##[resize] param : % of size reduction or increase i.e 0.65 for 35% reduction  or 1.5 for a 50% increase
             
        #send request
        self.parent_conn.send(1)
        frame = self.parent_conn.recv()
        
        #reset request 
        self.parent_conn.send(0)
        
        #resize if needed
        if resize == None:            
            return frame
        else:
            return self.rescale_frame(frame,resize)
        
    def rescale_frame(self,frame, percent=65):
        
        return cv2.resize(frame,None,fx=percent,fy=percent) 

Displaying the frames can be done as below

cam = Camera("rtsp://admin:[somepassword]@192.168.0.40/h264Preview_01_main")

print(f"Camera is alive?: {cam.p.is_alive()}")

while(1):
    frame = cam.get_frame(0.65)
    
    cv2.imshow("Feed",frame)
    
    key = cv2.waitKey(1)

    if key == 13: #13 is the Enter Key
        break

cv2.destroyAllWindows()     

cam.end()

This solution has resolved all my issues of buffer lag and also repeated frames. #

Hopefully it will help anyone else in the same situation.