How to convert YUV frames (from OTVideoFrame) to CVPixelBuffer

Question

I need to convert YUV Frames to CVPixelBuffer that I get from OTVideoFrame Class

This class provides an array of planes in the video frame which contains three elements for y,u,v frame each at index 0,1,2.

@property (nonatomic, retain) NSPointerArray *planes

and format of the video frame

@property (nonatomic, retain) OTVideoFormat *format

That contains Properties like width, height, bytesPerRow etc. of the frame

I need to add filter to the image I receive in the form of OTVideoFrame, I have already tried these answers :

• How to convert from YUV to CIImage for iOS
• Create CVPixelBuffer from YUV with IOSurface backed

These two links have the solutions in Objective-C but I want to do it in swift. One of the answers in second link was in swift but it lacks some information about the YUVFrame struct that the answer has reference to.

The Format that I receive is NV12

Here is what I have been trying to do till now but I don't know how to proceed next :-

 /**
 * Calcualte the size of each plane from OTVideoFrame.
 *
 * @param frame The frame to render.
 * @return tuple containing three elements for size of each plane
 */
fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
        -> (ySize: Int, uSize: Int, vSize: Int){
            guard let frameFormat = frame.format
                else {
                    return (0, 0 ,0)
            }
            let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
            return (baseSize, baseSize / 4, baseSize / 4)
    }

/**
 * Renders a frame to the video renderer.
 *
 * @param frame The frame to render.
 */
func renderVideoFrame(_ frame: OTVideoFrame) {


    let planeSize = calculatePlaneSize(forFrame: frame)
    let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
    let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
    let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)

    memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
    memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
    memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)

    let yStride = frame.format!.bytesPerRow.object(at: 0) as! Int
    // multiply chroma strides by 2 as bytesPerRow represents 2x2 subsample
    let uStride = frame.format!.bytesPerRow.object(at: 1) as! Int
    let vStride = frame.format!.bytesPerRow.object(at: 2) as! Int

    let width = frame.format!.imageWidth
    let height = frame.format!.imageHeight

    var pixelBuffer: CVPixelBuffer? = nil
    var err: CVReturn;


    err = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
    if (err != 0) {
        NSLog("Error at CVPixelBufferCreate %d", err)
        fatalError()
    }

}

Taking Guidance from those two links I tried to create Pixel buffer but I got stuck every time at this point because the conversion of the Objective-C code after this is not similar to what we have in Swift 3.

Answer 1

For those who are looking for a fast solution, I did with swift Accelerate using vImageConvert_AnyToAny(_:_:_:_:_:) function.

import Foundation
import Accelerate
import UIKit
import OpenTok
class Accelerater{


    var infoYpCbCrToARGB = vImage_YpCbCrToARGB()
    init() {
        _ = configureYpCbCrToARGBInfo()
    }

    func configureYpCbCrToARGBInfo() -> vImage_Error {
        print("Configuring")
        var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 0,
                                                 CbCr_bias: 128,
                                                 YpRangeMax: 255,
                                                 CbCrRangeMax: 255,
                                                 YpMax: 255,
                                                 YpMin: 1,
                                                 CbCrMax: 255,
                                                 CbCrMin: 0)

        let error = vImageConvert_YpCbCrToARGB_GenerateConversion(
            kvImage_YpCbCrToARGBMatrix_ITU_R_601_4!,
            &pixelRange,
            &infoYpCbCrToARGB,
            kvImage420Yp8_Cb8_Cr8,
            kvImageARGB8888,
            vImage_Flags(kvImagePrintDiagnosticsToConsole))



        print("Configration done \(error)")
        return error
    }

    public func convertFrameVImageYUV(toUIImage frame: OTVideoFrame, flag: Bool) -> UIImage {

        var result: UIImage? = nil
        let width = frame.format?.imageWidth ?? 0
        let height = frame.format?.imageHeight ?? 0
        var pixelBuffer: CVPixelBuffer? = nil
        _ = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_32BGRA, nil, &pixelBuffer)

        _ = convertFrameVImageYUV(frame, to: pixelBuffer)
        var ciImage: CIImage? = nil
        if let pixelBuffer = pixelBuffer {
            ciImage = CIImage(cvPixelBuffer: pixelBuffer)
        }

        let temporaryContext = CIContext(options: nil)
        var uiImage: CGImage? = nil
        if let ciImage = ciImage {
            uiImage = temporaryContext.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixelBuffer!), height: CVPixelBufferGetHeight(pixelBuffer!)))
        }

        if let uiImage = uiImage {
            result = UIImage(cgImage: uiImage)
        }
        CVPixelBufferUnlockBaseAddress(pixelBuffer!, [])
        return result!

    }

    func convertFrameVImageYUV(_ frame: OTVideoFrame, to pixelBufferRef: CVPixelBuffer?) -> vImage_Error{
        let start  = CFAbsoluteTimeGetCurrent()
        if pixelBufferRef == nil {
            print("No PixelBuffer refrance found")
            return vImage_Error(kvImageInvalidParameter)
        }

        let width = frame.format?.imageWidth ?? 0
        let height = frame.format?.imageHeight ?? 0
        let subsampledWidth = frame.format!.imageWidth/2
        let subsampledHeight = frame.format!.imageHeight/2
        print("subsample height \(subsampledHeight) \(subsampledWidth)")
        let planeSize = calculatePlaneSize(forFrame: frame)

        print("ysize : \(planeSize.ySize) \(planeSize.uSize) \(planeSize.vSize)")
        let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
        let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
        let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)

        memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
        memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
        memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)

        let yStride = frame.format!.bytesPerRow.object(at: 0) as! Int
        // multiply chroma strides by 2 as bytesPerRow represents 2x2 subsample
        let uStride = frame.format!.bytesPerRow.object(at: 1) as! Int
        let vStride = frame.format!.bytesPerRow.object(at: 2) as! Int

        var yPlaneBuffer = vImage_Buffer(data: yPlane, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: yStride)

        var uPlaneBuffer = vImage_Buffer(data: uPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: uStride)




        var vPlaneBuffer = vImage_Buffer(data: vPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: vStride)
        CVPixelBufferLockBaseAddress(pixelBufferRef!, .readOnly)
        let pixelBufferData = CVPixelBufferGetBaseAddress(pixelBufferRef!)
        let rowBytes = CVPixelBufferGetBytesPerRow(pixelBufferRef!)
        var destinationImageBuffer = vImage_Buffer()
        destinationImageBuffer.data = pixelBufferData
        destinationImageBuffer.height = vImagePixelCount(height)
        destinationImageBuffer.width = vImagePixelCount(width)
        destinationImageBuffer.rowBytes = rowBytes

        var permuteMap: [UInt8] = [3, 2, 1, 0] // BGRA
        let convertError = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&yPlaneBuffer, &uPlaneBuffer, &vPlaneBuffer, &destinationImageBuffer, &infoYpCbCrToARGB, &permuteMap, 255, vImage_Flags(kvImagePrintDiagnosticsToConsole))

        CVPixelBufferUnlockBaseAddress(pixelBufferRef!, [])


        yPlane.deallocate()
        uPlane.deallocate()
        vPlane.deallocate()
        let end = CFAbsoluteTimeGetCurrent()
        print("Decoding time \((end-start)*1000)")
        return convertError

    }
    fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
        -> (ySize: Int, uSize: Int, vSize: Int)
    {
        guard let frameFormat = frame.format
            else {
                return (0, 0 ,0)
        }
        let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
        return (baseSize, baseSize / 4, baseSize / 4)
    }

}

Performance tested on iPhone7, one frame conversion is less than a millisecond.

Answer 2

Here's what worked for me (I've taken your function and changed it a bit):

func createPixelBufferWithVideoFrame(_ frame: OTVideoFrame) -> CVPixelBuffer? {
    if let fLock = frameLock {
        fLock.lock()

        let planeSize = calculatePlaneSize(forFrame: frame)

        let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
        let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
        let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)

        memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
        memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
        memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)

        let width = frame.format!.imageWidth
        let height = frame.format!.imageHeight

        var pixelBuffer: CVPixelBuffer? = nil
        var err: CVReturn;

        err = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
        if (err != 0) {
            NSLog("Error at CVPixelBufferCreate %d", err)
            return nil
        }

        if let pixelBuffer = pixelBuffer {
            CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
            let yPlaneTo = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)
            memcpy(yPlaneTo, yPlane, planeSize.ySize)

            let uvRow: Int = planeSize.uSize*2/Int(width)

            let halfWidth: Int = Int(width)/2

            if let uPlaneTo = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) {
                let uvPlaneTo = uPlaneTo.bindMemory(to: GLubyte.self, capacity: Int(uvRow*halfWidth*2))

                for i in 0..<uvRow {
                    for j in 0..<halfWidth {
                        let dataIndex: Int = Int(i) * Int(halfWidth) + Int(j)
                        let uIndex: Int = (i * Int(width)) + Int(j) * 2
                        let vIndex: Int = uIndex + 1

                        uvPlaneTo[uIndex] = uPlane[dataIndex]
                        uvPlaneTo[vIndex] = vPlane[dataIndex]

                    }
                }

            }

        }

        fLock.unlock()

        return pixelBuffer
    }
    return nil
}