CODEWITHSUNDEEP
c#opencvunity-game-engineeye-detection
Nition
Nition

Reputation: 91

Eye detection using OpenCVSharp in Unity (fps issues)

I'm currently working on a project involving integrating OpenCVSharp into Unity, to allow eye tracking within a game environment. I've managed to get OpenCVSharp integrated into the Unity editor and currently have eye-detection (not tracking) working within a game. It can find your eyes within a webcam image, then display where its currently detected them on a texture, which I display within the scene.

However its causing a HUGE fps drop, mainly because every frame its converting a webcam texture into an IPLimage so that OpenCV can handle it. It then has to convert it back to a 2Dtexture to be displayed within the scene, after its done all the eye detection. So understandably its too much for the CPU to handle. (As far as I can tell its only using 1 core on my CPU).

Is there a way to do all the eye detection without converting the texture to an IPLimage? Or any other way to fix the fps drop. Some things that I've tried include:

I've included the code below, of if you would prefer to look at it in a code editor here is a link to the C# File. Any suggestions or help would be greatly appreciated!

For reference I used code from here (eye detection using opencvsharp).

using UnityEngine;
using System.Collections;
using System;
using System.IO;
using OpenCvSharp;
//using System.Xml;

//using OpenCvSharp.Extensions;
//using System.Windows.Media;
//using System.Windows.Media.Imaging;



public class CaptureScript : MonoBehaviour
{
    public GameObject planeObj;
    public WebCamTexture webcamTexture;     //Texture retrieved from the webcam
    public Texture2D texImage;              //Texture to apply to plane
    public string deviceName;

    private int devId = 1;
    private int imWidth = 640;              //camera width
    private int imHeight = 360;             //camera height
    private string errorMsg = "No errors found!";
    static IplImage matrix;                 //Ipl image of the converted webcam texture

    CvColor[] colors = new CvColor[]
    {
        new CvColor(0,0,255),
        new CvColor(0,128,255),
        new CvColor(0,255,255),
        new CvColor(0,255,0),
        new CvColor(255,128,0),
        new CvColor(255,255,0),
        new CvColor(255,0,0),
        new CvColor(255,0,255),
    };

    const double Scale = 1.25;
    const double ScaleFactor = 2.5;
    const int MinNeighbors = 2;


// Use this for initialization
    void Start ()
    {
            //Webcam initialisation
            WebCamDevice[] devices = WebCamTexture.devices;
            Debug.Log ("num:" + devices.Length);

            for (int i=0; i<devices.Length; i++) {
                    print (devices [i].name);
                    if (devices [i].name.CompareTo (deviceName) == 1) {
                            devId = i;
                    }
            }

            if (devId >= 0) {
                    planeObj = GameObject.Find ("Plane");
                    texImage = new Texture2D (imWidth, imHeight, TextureFormat.RGB24, false);
                    webcamTexture = new WebCamTexture (devices [devId].name, imWidth, imHeight, 30);
                    webcamTexture.Play ();

                    matrix = new IplImage (imWidth, imHeight, BitDepth.U8, 3);
            }


    }

    void Update ()
    {
        if (devId >= 0)
        {
                //Convert webcam texture to iplimage
                Texture2DtoIplImage();

            /*DO IMAGE MANIPULATION HERE*/

            //do eye detection on iplimage
            EyeDetection();


            /*END IMAGE MANIPULATION*/

            if (webcamTexture.didUpdateThisFrame) 
            {
                //convert iplimage to texture
                IplImageToTexture2D();
            }

        } 
        else 
        {
            Debug.Log ("Can't find camera!");
        }

    }

    void EyeDetection()
    {

        using(IplImage smallImg = new IplImage(new CvSize(Cv.Round (imWidth/Scale), Cv.Round(imHeight/Scale)),BitDepth.U8, 1))
        {
            using(IplImage gray = new IplImage(matrix.Size, BitDepth.U8, 1))
            {
                Cv.CvtColor (matrix, gray, ColorConversion.BgrToGray);
                Cv.Resize(gray, smallImg, Interpolation.Linear);
                Cv.EqualizeHist(smallImg, smallImg);
            }


            using(CvHaarClassifierCascade cascade = CvHaarClassifierCascade.FromFile (@"C:\Users\User\Documents\opencv\sources\data\haarcascades\haarcascade_eye.xml"))
            using(CvMemStorage storage = new CvMemStorage())
            {
                storage.Clear ();
                CvSeq<CvAvgComp> eyes = Cv.HaarDetectObjects(smallImg, cascade, storage, ScaleFactor, MinNeighbors, 0, new CvSize(30, 30));
                for(int i = 0; i < eyes.Total; i++)
                {
                    CvRect r = eyes[i].Value.Rect;
                    CvPoint center = new CvPoint{ X = Cv.Round ((r.X + r.Width * 0.5) * Scale), Y = Cv.Round((r.Y + r.Height * 0.5) * Scale) };
                    int radius = Cv.Round((r.Width + r.Height) * 0.25 * Scale);
                    matrix.Circle (center, radius, colors[i % 8], 3, LineType.AntiAlias, 0);
                }
            }

        }
    }

    void OnGUI ()
    {
            GUI.Label (new Rect (200, 200, 100, 90), errorMsg);
    }

    void IplImageToTexture2D ()
    {
            int jBackwards = imHeight;

            for (int i = 0; i < imHeight; i++) {
                    for (int j = 0; j < imWidth; j++) {
                            float b = (float)matrix [i, j].Val0;
                            float g = (float)matrix [i, j].Val1;
                            float r = (float)matrix [i, j].Val2;
                            Color color = new Color (r / 255.0f, g / 255.0f, b / 255.0f);


                            jBackwards = imHeight - i - 1; // notice it is jBackward and i
                            texImage.SetPixel (j, jBackwards, color);
                    }
            }
            texImage.Apply ();
            planeObj.renderer.material.mainTexture = texImage;

    }

    void Texture2DtoIplImage ()
    {
            int jBackwards = imHeight;

            for (int v=0; v<imHeight; ++v) {
                    for (int u=0; u<imWidth; ++u) {

                            CvScalar col = new CvScalar ();
                            col.Val0 = (double)webcamTexture.GetPixel (u, v).b * 255;
                            col.Val1 = (double)webcamTexture.GetPixel (u, v).g * 255;
                            col.Val2 = (double)webcamTexture.GetPixel (u, v).r * 255;

                            jBackwards = imHeight - v - 1;

                            matrix.Set2D (jBackwards, u, col);
                            //matrix [jBackwards, u] = col;
                    }
            }
    }
}

Upvotes: 3

Views: 7688

Answers (2)

zemmeister
zemmeister

Reputation: 11

You should also be able to gain some performance improvements if you use:

    Color32[] pixels;
    pixels = new Color32[webcamTexture.width * webcamTexture.height];
    webcamTexture.GetPixels32(pixels);

The Unity doco suggests that this can be quite a bit faster than calling "GetPixels" (and certainly faster than calling GetPixel for each pixel), and then you don't need to scale each RGB channel against 255 manually.

Upvotes: 1

Enkiduo
Enkiduo

Reputation: 11

You can move these out of the per frame update loop : 

using(CvHaarClassifierCascade cascade = CvHaarClassifierCascade.FromFile (@"C:\Users\User\Documents\opencv\sources\data\haarcascades\haarcascade_eye.xml"))
using(CvMemStorage storage = new CvMemStorage())

No reason to be building the recognizer graph each frame.

Threading is the logical way to go moving forward if you want real speed updates, unity itself is not threaded, but you can fold in other threads if your careful.

Do the texture -> ipl image on the main thread then trigger an event to fire off your thread. The thread can do all the CV work, probably construct the tex2d and then push back to main to render.

Upvotes: 1

Related Questions