How can I call captureOutput at 240fps?

Question

I want to save image at a high frame rate. IO is expensive, not 240fps exactly.But maybe more than 120fps.

What I'm I do is set captureDivece in viewDidLoad, and log timestamp in captureOutput and view the rate.

But I notice that captureOutput is always 30fps. Could you please tell me which I get wrong?

Thanks for your time and answer!

Here is my code and result:

//
//  ViewController.swift
//  CustomCamera
//
//  Created by chunibyo on 2021/3/8.
//

import UIKit
import AVFoundation
import Vision
import VideoToolbox

class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
    
    @IBOutlet weak var captureButton: UIButton!
    let sessionQueue = DispatchQueue(label: "Session Queue")
    var status = false
    private var MyCaptureDevice: AVCaptureDevice?
    
    override func viewDidLoad() {
        super.viewDidLoad()
        // Do any additional setup after loading the view.
        
        captureButton.layer.cornerRadius = captureButton.frame.width / 2
        captureButton.layer.masksToBounds = true
        captureButton.layer.zPosition = 10
        
        guard let captureDevice = AVCaptureDevice.default(.builtInDualCamera, for: .video, position: .back) else {return}

        guard let input = try? AVCaptureDeviceInput(device: captureDevice) else {return}
        
        // 1
        for vFormat in captureDevice.formats {
            // 2
            let ranges = vFormat.videoSupportedFrameRateRanges as [AVFrameRateRange]
            let frameRates = ranges[0]
            // 3
            if frameRates.maxFrameRate == 240 {
                // 4
                try? captureDevice.lockForConfiguration()
                captureDevice.activeFormat = vFormat as AVCaptureDevice.Format
                captureDevice.activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
                captureDevice.activeVideoMaxFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
                captureDevice.videoZoomFactor = captureDevice.minAvailableVideoZoomFactor
                captureDevice.unlockForConfiguration()
            }
        }
        
        let captureSession = AVCaptureSession();
        // captureSession.sessionPreset = .photo
        captureSession.addInput(input)
        captureSession.startRunning();
        
        let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
        view.layer.addSublayer(previewLayer)
        previewLayer.frame = view.frame
        
        let dataOutput = AVCaptureVideoDataOutput()
        dataOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
        dataOutput.alwaysDiscardsLateVideoFrames = true;
        captureSession.addOutput(dataOutput)
        
        print(captureDevice.minAvailableVideoZoomFactor)
        print(captureDevice.maxAvailableVideoZoomFactor)
        
        MyCaptureDevice = captureDevice
    }

    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        print(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)))
//        if !status { return }
//        guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
//        guard let uiImage = UIImage(pixelBuffer: pixelBuffer) else { return }
//
//        sessionQueue.async {
//            guard let captureDevice = self.MyCaptureDevice else { return }
//            if captureDevice.videoZoomFactor >= (captureDevice.maxAvailableVideoZoomFactor - 0.2) { return }
//            UIImageWriteToSavedPhotosAlbum(uiImage, nil, nil, nil)
//            try? captureDevice.lockForConfiguration()
//            captureDevice.videoZoomFactor += 0.1
//            captureDevice.unlockForConfiguration()
//        }
    }

    @IBAction func captureControl(_ sender: UIButton) {
        DispatchQueue.main.async {
            if self.status {
                self.captureButton.backgroundColor = .white
                print("stop")
                self.status = !self.status
            }
            else {
                self.captureButton.backgroundColor = .red
                print("recording...")
                self.status = !self.status
            }
            
        }
    }
}

extension UIImage {
    public convenience init?(pixelBuffer: CVPixelBuffer) {
        var cgImage: CGImage?
        VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
        guard let _cgImage = cgImage else { return nil }
        self.init(cgImage: _cgImage)
    }
}

console log (stackoverflow not allow me to post image, sorry)

Answer 1

captureOutput only logs is 240fps.

captureOutput with take photos to album is about 70~100fps.

This code can get 240fps logs.

//
//  ViewController.swift
//  CustomCamera
//
//  Created by chunibyo on 2021/3/8.
//

import UIKit
import AVFoundation
import Vision
import VideoToolbox

class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
    
    @IBOutlet weak var captureButton: UIButton!
    let sessionQueue = DispatchQueue(label: "Session Queue")
    var status = false
    var zoomStatus = 1
    private var MyCaptureDevice: AVCaptureDevice?
    
    override func viewDidLoad() {
        super.viewDidLoad()
        // Do any additional setup after loading the view.
        
        captureButton.layer.cornerRadius = captureButton.frame.width / 2
        captureButton.layer.masksToBounds = true
        captureButton.layer.zPosition = 10
        
        
        guard let captureDevice = AVCaptureDevice.default(for: AVMediaType.video) else {return}

        guard let input = try? AVCaptureDeviceInput(device: captureDevice) else {return}
        
        let captureSession = AVCaptureSession();
        // captureSession.sessionPreset = .photo
        captureSession.addInput(input)
        
        // 1
        for vFormat in captureDevice.formats {
            // 2
            let ranges = vFormat.videoSupportedFrameRateRanges as [AVFrameRateRange]

            let frameRates = ranges[0]
            // 3
            if frameRates.maxFrameRate == 240 {
                // 4
                try? captureDevice.lockForConfiguration()
                captureDevice.activeFormat = vFormat as AVCaptureDevice.Format
                captureDevice.activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
                captureDevice.activeVideoMaxFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
                captureDevice.videoZoomFactor = captureDevice.minAvailableVideoZoomFactor
                captureDevice.unlockForConfiguration()
            }
        }
        
        captureSession.startRunning();
        
        let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
        view.layer.addSublayer(previewLayer)
        previewLayer.frame = view.frame
        
        let dataOutput = AVCaptureVideoDataOutput()
        dataOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
        dataOutput.alwaysDiscardsLateVideoFrames = true;
        captureSession.addOutput(dataOutput)
        
        print(captureDevice.minAvailableVideoZoomFactor)
        print(captureDevice.maxAvailableVideoZoomFactor)
        
        MyCaptureDevice = captureDevice
    }

    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        print(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)))
//        if !status { return }
//        guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
//        guard let uiImage = UIImage(pixelBuffer: pixelBuffer) else { return }
//        UIImageWriteToSavedPhotosAlbum(uiImage, nil, nil, nil)
//
//        guard let captureDevice = self.MyCaptureDevice else { return }
//        if self.zoomStatus == 1 && captureDevice.videoZoomFactor >= CGFloat(Int32(captureDevice.maxAvailableVideoZoomFactor * 0.6)) { self.zoomStatus = -1
//        }
//        else if self.zoomStatus == -1 && captureDevice.videoZoomFactor <= (captureDevice.minAvailableVideoZoomFactor + 1.0) {
//            self.zoomStatus = 1
//        }
//        UIImageWriteToSavedPhotosAlbum(uiImage, nil, nil, nil)
//        try? captureDevice.lockForConfiguration()
//        captureDevice.videoZoomFactor += (0.1 * CGFloat(self.zoomStatus))
//        captureDevice.unlockForConfiguration()
    }

    @IBAction func captureControl(_ sender: UIButton) {
        DispatchQueue.main.async {
            if self.status {
                self.captureButton.backgroundColor = .white
                print("stop")
                self.status = !self.status
            }
            else {
                self.captureButton.backgroundColor = .red
                print("recording...")
                self.status = !self.status
            }
            
        }
    }
}

extension UIImage {
    public convenience init?(pixelBuffer: CVPixelBuffer) {
        var cgImage: CGImage?
        VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
        guard let _cgImage = cgImage else { return nil }
        self.init(cgImage: _cgImage)
    }
}