Viola-Jones in Python with openCV, detection mouth and nose

Question

I have an algorithm Viola-Jones in Python. I'm using haarcascade xml, which I load from openCV root file. But there wasn't any xml file for mouth and nose in openCV, so I downloaded these files from EmguCV. Result for detection of face is OK, but detection of eye isn't good and nose with mouth is very bad. I tried to change parameters in face_cascade.detectMultiScale, but it didn't help at all.

---

My code:

import cv2
import sys

def facedet(img):
    face_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_frontalface_default.xml')
    eye_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_eye.xml')
    mouth_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_mcs_mouth.xml')
    nose_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_mcs_nose.xml')

    img = cv2.imread(img)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(gray, 1.3, 5)

    for (x,y,w,h) in faces:
        cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)
        roi_gray = gray[y:y+h, x:x+w]
        roi_color = img[y:y+h, x:x+w]
        eyes = eye_cascade.detectMultiScale(roi_gray)
        nose =  nose_cascade.detectMultiScale(roi_gray)
        mouth = mouth_cascade.detectMultiScale(roi_gray)

        for (ex,ey,ew,eh) in eyes:
            cv2.rectangle(roi_color, (ex,ey), (ex+ew, ey+eh), (0,255,0), 2)
        for (nx, ny, nw, nh) in nose:
            cv2.rectangle(roi_color, (nx, ny), (nx + nw, ny + nh), (0, 0, 255), 2)
        for (mx, my, mw, mh) in mouth:
            cv2.rectangle(roi_color, (mx, my), (mx + mw, my + mh), (0, 0, 0), 2)

    cv2.namedWindow('image', cv2.WINDOW_NORMAL)
    cv2.imshow('image',img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()


if __name__ == '__main__':
    #img = sys.argv[1]
    facedet(img)

---

My question

What am I doing wrong? Is there any simple solution, which will give me a better result?

---

Output:

Answer 1

This works really well for me.

I found that if you divide the face into 2 sections and have the eyes look for eyes in the top section, and the mouth in the lower section it works really well.

face
--------
| eyes |
|------|
|mouth |
--------

This is a rough illustration of what I did with the code below.

I am aware the the cascade i use is smile, but the mouth doesn't seem to work.

import cv2
import sys

mouthCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_smile.xml')
faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
eyeCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')
video_capture = cv2.VideoCapture(0)

while True:
    # Capture frame-by-frame
    ret, frame = video_capture.read()

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    mouth = mouthCascade.detectMultiScale(gray, 1.3, 5)
    faces = faceCascade.detectMultiScale(
                gray,
                scaleFactor=1.1,
                minNeighbors=5,
                minSize=(30, 30)
            )
            # Draw a rectangle around the faces
    for (x, y, w, h) in faces:
        cv2.rectangle(frame, (x, y), (x+w, y+h), (255, 0, 0), 2)
            # Draw a rectangle around the faces
        roi_gray_mouth = gray[y+(int(h/2)):y+h, x:x+w]
        roi_color_mouth = frame[y+(int(h/2)):y+h, x:x+w]

        roi_gray_eye = gray[y-(int(h/2)):y+h, x:x+w]
        roi_color_eye = frame[y-(int(h/2)):y+h, x:x+w]

        mouth = mouthCascade.detectMultiScale(roi_gray_mouth)
        eyes = eyeCascade.detectMultiScale(roi_gray_eye)
        for (ex,ey,ew,eh) in mouth:
            cv2.rectangle(roi_color_mouth, (ex, ey), (ex+ew, ey+eh), (0, 255, 0), 2)

        for (eex,eey,eew,eeh) in eyes:
            d = int(eew / 2)
            cv2.circle(roi_color_eye, (int(eex + eew / 4) + int(d / 2), int(eey + eeh / 4) + int(d / 2)), int(d) ,(0,0,255),2)

    # Display the resulting frame
    cv2.imshow('Video', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything is done, release the capture
video_capture.release()
cv2.destroyAllWindows()

Answer 2

import cv2 import sys

face_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_eye.xml')
mouth_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_mcs_mouth.xml')
nose_cascade = cv2.CascadeClassifier('/home/kattynka/opencv/data/haarcascades/haarcascade_mcs_nose.xml')

img = cv2.imread(img)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

faces = face_cascade.detectMultiScale(gray, 1.3, 5)

for (x,y,w,h) in faces:
    cv2.rectangle(img, (x,y), (x+w,y+h), (255,0,0), 2)
    roi_gray = gray[y:y+h, x:x+w]
    roi_color = img[y:y+h, x:x+w]
    eyes = eye_cascade.detectMultiScale(gray, 1.3, 5)
    nose =  nose_cascade.detectMultiScale(gray, 1.3, 5)
    mouth = mouth_cascade.detectMultiScale(gray, 1.7, 11)

    for (ex,ey,ew,eh) in eyes:
        cv2.rectangle(img, (ex,ey), (ex+ew, ey+eh), (0,255,0), 2)
    for (nx, ny, nw, nh) in nose:
        cv2.rectangle(img, (nx, ny), (nx + nw, ny + nh), (0, 0, 255), 2)
    for (mx, my, mw, mh) in mouth:
        cv2.rectangle(img, (mx, my), (mx + mw, my + mh), (0, 0, 0), 2)

cv2.namedWindow('image', cv2.WINDOW_NORMAL)
cv2.imshow('image',img)
cv2.waitKey(0)
cv2.destroyAllWindows()

---

you can try this code. It worked for me.

Answer 3

Haar cascades perform alright for faces but not so well for smaller individual parts. A better solution is to detect all the face landmarks together. A good algorithm for that is "One Millisecond Face Alignment with an Ensemble of Regression Trees by Vahid Kazemi and Josephine Sullivan, CVPR 2014" which is implemented in Dlib (http://dlib.net/face_landmark_detection.py.html).