How to port this opengl code from xcode to Monotouch?

Question

I'm trying to convert the following code into Monotouch:

if (videoTextureCache == NULL) {
    //  Create a new CVOpenGLESTexture cache
    CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, self.context, NULL, &videoTextureCache);
    if (err) {
        NSLog(@"Error at CVOpenGLESTextureCacheCreate %d", err);
    }
}

if (videoTextureCache) {
    CMSampleBufferRef sampleBuffer = self.videoFrameReader.sampleBuffer;
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    CVPixelBufferLockBaseAddress(imageBuffer, 0);

    // Create a CVOpenGLESTexture from the CVImageBuffer
    size_t frameWidth = CVPixelBufferGetWidth(imageBuffer);
    size_t frameHeight = CVPixelBufferGetHeight(imageBuffer);
    CVOpenGLESTextureRef texture = NULL;
    CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
                                                                videoTextureCache,
                                                                imageBuffer,
                                                                NULL,
                                                                GL_TEXTURE_2D,
                                                                GL_RGBA,
                                                                frameWidth,
                                                                frameHeight,
                                                                GL_BGRA,
                                                                GL_UNSIGNED_BYTE,
                                                                0,
                                                                &texture);
    if (!texture || err) {
        NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage failed (error: %d)", err);
        return;
    } else {
        self.cubeTextureName = CVOpenGLESTextureGetName(texture);
    }

    // Flush the CVOpenGLESTexture cache and release the texture
    CVOpenGLESTextureCacheFlush(videoTextureCache, 0);
    CFRelease(texture);

    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);

It's related to rendering video into an opengl texture fast. It works fine in xCode, with excellent performance, but now I need to make it work in a Monotouch related project.

However, I am quite lost on how to port this code, as I cannot find the necessary Monotouch bindings and also no Monotouch related documentation about the key methods used above, like CVOpenGLESTextureCacheCreateTextureFromImage and CVOpenGLESTextureCacheCreate. Are they missing from Monotouch?

Is there a way I can invoke them?

Answer 1

Thank you so much, although the sample you provided did not work, it helped a lot to get me there. :)

This is the code I ended up with (I edited it somewhat for clarity):

    [DllImport (MonoTouch.Constants.CoreVideoLibrary)]
    extern static int CVOpenGLESTextureCacheCreateTextureFromImage (
        IntPtr allocator,
        IntPtr textureCache,
        IntPtr sourceImage,
        IntPtr textureAttributes,
        int target,
        int internalFormat,
        int width,
        int height,
        int format,
        int type,
        int planeIndex,
        ref IntPtr textureOut);

[DllImport (MonoTouch.Constants.CoreVideoLibrary)]
extern static int CVOpenGLESTextureCacheFlush (IntPtr texture, int v);

[DllImport (MonoTouch.Constants.CoreVideoLibrary)]
extern static int CVOpenGLESTextureGetName (IntPtr texture);

[DllImport (MonoTouch.Constants.CoreFoundationLibrary, CharSet=CharSet.Unicode)]
internal extern static IntPtr CFRelease (IntPtr obj);

    private IntPtr videoTextureCache;
    unsafe public void setTexture(CVImageBuffer sampleBuffer)
    {
        int err;
    if (videoTextureCache == IntPtr.Zero) {
            EAGLContext eaglContext=Global.localScreen.view.Context;
        err = CVOpenGLESTextureCacheCreate (IntPtr.Zero, IntPtr.Zero, eaglContext.Handle, IntPtr.Zero, out videoTextureCache);
        if (err != 0){
            Console.WriteLine ("Error at CVOpenGLESTextureCacheCreate {0}", err);
            return;
        }
    }

        CVPixelBuffer pixelBuffer = (CVPixelBuffer) sampleBuffer;
    pixelBuffer.Lock (CVOptionFlags.None);
    int frameWidth = pixelBuffer.Width;
    int frameHeight = pixelBuffer.Height;

        IntPtr texture = IntPtr.Zero;
        err = CVOpenGLESTextureCacheCreateTextureFromImage(IntPtr.Zero,
                                                        videoTextureCache,
                                                        pixelBuffer.Handle,
                                                        IntPtr.Zero,
                                                        (int)All.Texture2D,
                                                        (int)All.Rgba,
                                                        frameWidth,
                                                        frameHeight,
                                                        (int)All.Bgra,
                                                        (int)All.UnsignedByte,
                                                        0,
                                                        ref texture);

    if (texture == IntPtr.Zero || err != 0) {
        Console.WriteLine("CVOpenGLESTextureCacheCreateTextureFromImage failed (error: {0})", err);
        return;
    } else {
        Console.WriteLine ("Texture name: {0}", CVOpenGLESTextureGetName(texture));
    }

    CVOpenGLESTextureCacheFlush(videoTextureCache, 0);
        CFRelease(texture);

        pixelBuffer.Unlock (CVOptionFlags.None);            

    }

I still have some weird issues though. When playing the video and sending the individual frames to the method above, the frames are drawn in somewhat random order, making the video flicker, although the frames do indeed come in the correct order into that method.

Basically the program flow is like this:

1. Extract video frame (I know they do indeed come in the right order)
2. Feed frame to the method above, and create a texture
3. Draw texture (a frame +/- 5 frames from the current frame is actually drawn)

Very simple, but seems Monotouch is not playing along well.

This is strange since the code above is an exact replica of the xCode version, which works fine. Not sure what is going on, but I suspect it is related to the automatic memory management in Monotouch.

Also, if the video is not in a resolution that is a power of two, I get a black texture... Not a problem when using Objective C. Again, not sure what is going on with Monotouch here... :(

Answer 2

I am not very familiar with OpenGL, so I might have gotten the sample wrong, but here is a starting point:

using System;
using System.Collections.Generic;
using System.Linq;

using MonoTouch.Foundation;
using MonoTouch.UIKit;
using MonoTouch.CoreVideo;
using MonoTouch.CoreMedia;
using System.Runtime.InteropServices;
using MonoTouch.OpenGLES;
using OpenTK.Platform.iPhoneOS;
using OpenTK.Graphics.ES20;
using OpenTK;

namespace cv1
{
    [Register ("AppDelegate")]
    public partial class AppDelegate : UIApplicationDelegate
    {
        UIWindow window;

        [DllImport (MonoTouch.Constants.CoreVideoLibrary)]
        extern static int CVOpenGLESTextureCacheCreate (IntPtr alloc, IntPtr cache, IntPtr eaglContext, IntPtr textureAttr, out IntPtr cache);

        [DllImport (MonoTouch.Constants.CoreVideoLibrary)]
        extern static int CVOpenGLESTextureCacheCreateTextureFromImage (IntPtr alloc, IntPtr textureCache, IntPtr sourceImage, IntPtr textureAttr, EnableCap target, PixelFormat internalFormat, int width, int height, int format, DataType type, int planeIndex, out IntPtr texture);
        [DllImport (MonoTouch.Constants.CoreVideoLibrary)]
        extern static int CVOpenGLESTextureCacheFlush (IntPtr texture, int v);
        [DllImport (MonoTouch.Constants.CoreVideoLibrary)]
        extern static int CVOpenGLESTextureGetName (IntPtr texture);
        [DllImport (MonoTouch.Constants.CoreFoundationLibrary, CharSet=CharSet.Unicode)]
        internal extern static IntPtr CFRelease (IntPtr obj);

        IntPtr videoTextureCache;

        void Stuff (EAGLContext context)
        {
            int err;
            if (videoTextureCache == IntPtr.Zero) {
                err = CVOpenGLESTextureCacheCreate (IntPtr.Zero, IntPtr.Zero, context.Handle, IntPtr.Zero, out videoTextureCache);
                if (err != 0){
                    Console.WriteLine ("Error at CVOpenGLESTextureCacheCreate {0}", err);
                    return;
                }
            }
            CMSampleBuffer sampleBuffer = null; //videoFrameReader.SampleBuffer;
            CVPixelBuffer imageBuffer = (CVPixelBuffer) sampleBuffer.GetImageBuffer ();
            imageBuffer.Lock (CVOptionFlags.None);
            int frameWidth = imageBuffer.Width;
            int frameHeight = imageBuffer.Height;

            IntPtr texture = IntPtr.Zero;
            err = CVOpenGLESTextureCacheCreateTextureFromImage(IntPtr.Zero,
                                                                videoTextureCache,
                                                                imageBuffer.Handle,
                                                                IntPtr.Zero,
                                                                EnableCap.Texture2D,
                                                                PixelFormat.Rgba,
                                                                frameWidth,
                                                                frameHeight,
                                                                0x10B6,
                                                                DataType.UnsignedByte,
                                                                0,
                                                                out texture);
            if (texture == IntPtr.Zero || err != 0) {
                Console.WriteLine("CVOpenGLESTextureCacheCreateTextureFromImage failed (error: {0})", err);
                return;
            } else {
                Console.WriteLine ("Texture name: {0}", CVOpenGLESTextureGetName(texture));
            }

            CVOpenGLESTextureCacheFlush(videoTextureCache, 0);
            CFRelease(texture);

            imageBuffer.Unlock (CVOptionFlags.None);
        }

        public override bool FinishedLaunching (UIApplication app, NSDictionary options)
        {
            window = new UIWindow (UIScreen.MainScreen.Bounds);
            window.MakeKeyAndVisible ();


            return true;
        }
    }
}

Since you did not provide a couple of data points, I just had to leave them as either null values or printed out the value without assigning it to anything.