Face recognition using Yunet model. problem when use Webcam the boxes are draw far from face, but if use MAC camera the boxes are draw correctly

Question

the code is for face recognition based on https://medium.com/@silkworm/sface-the-fastest-also-powerful-deep-learning-face-recognition-model-in-the-world-8c56e7d489bc

When I use the internal mac camera the boxes are draw correctly. But when I use Webcam, the boxes are draw displaced from face.

import os
import sys
import glob
import time
import math
import cv2
import numpy as np
from tqdm import tqdm

COSINE_THRESHOLD = 0.5

def match(recognizer, feature1, dictionary):
    max_score = 0.0
    sim_user_id = ""
    for user_id, feature2 in zip(dictionary.keys(), dictionary.values()):
        score = recognizer.match(
            feature1, feature2, cv2.FaceRecognizerSF_FR_COSINE)
        if score >= max_score:
            max_score = score
            sim_user_id = user_id
    if max_score < COSINE_THRESHOLD:
        return False, ("", 0.0)
    return True, (sim_user_id, max_score)

def recognize_face(image, face_detector, face_recognizer, file_name=None):
    channels = 1 if len(image.shape) == 2 else image.shape[2]
    if channels == 1:
        image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
    if channels == 4:
        image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)

    if image.shape[0] > 1000:
        image = cv2.resize(image, (0, 0),
                           fx=500 / image.shape[0], fy=500 / image.shape[0])

    height, width, _ = image.shape
    face_detector.setInputSize((width, height))
    try:
        dts = time.time()
        _, faces = face_detector.detect(image)
        if file_name is not None:
            assert len(faces) > 0, f'the file {file_name} has no face'

        faces = faces if faces is not None else []
        features = []
        print(f'time detection  = {time.time() - dts}')
        for face in faces:
            rts = time.time()

            aligned_face = face_recognizer.alignCrop(image, face)
            feat = face_recognizer.feature(aligned_face)
            print(f'time recognition  = {time.time() - rts}')

            features.append(feat)
        return features, faces
    except Exception as e:
        print(e)
        print(file_name)
        return None, None

def main():
    
    # contain npy for embedings and registration photos
    directory = '/Users/triplec/Sites/FRV/SFACEYUNET/'

    # Init models face detection & recognition
    weights = os.path.join(directory, "models",
                           "face_detection_yunet_2022mar.onnx")
    face_detector = cv2.FaceDetectorYN_create(weights, "", (0, 0))
    face_detector.setScoreThreshold(0.87)

    weights = os.path.join(directory, "models", "face_recognizer_fast.onnx")
    face_recognizer = cv2.FaceRecognizerSF_create(weights, "")

    # Get registered photos and return as npy files
    # File name = id name, embeddings of a photo is the representative for the id
    # If many files have the same name, an average embedding is used
    dictionary = {}
    # the tuple of file types, please ADD MORE if you want
    types = ('*.jpg', '*.png', '*.jpeg', '*.JPG', '*.PNG', '*.JPEG')
    files = []
    for a_type in types:
        files.extend(glob.glob(os.path.join(directory, 'images', a_type)))

    files = list(set(files))

    for file in tqdm(files):
        image = cv2.imread(file)
        feats, faces = recognize_face(
            image, face_detector, face_recognizer, file)
        if faces is None:
            continue
        user_id = os.path.splitext(os.path.basename(file))[0]
        dictionary[user_id] = feats[0]

    print(f'there are {len(dictionary)} ids')
    capture = cv2.VideoCapture(1)
    if not capture.isOpened():
        sys.exit()

    while True:
        start_hand = time.time()
        result, image = capture.read()
        if result is False:
            cv2.waitKey(0)
            break

        fetures, faces = recognize_face(image, face_detector, face_recognizer)
        if faces is None:
            continue

        for idx, (face, feature) in enumerate(zip(faces, fetures)):
            result, user = match(face_recognizer, feature, dictionary)
            box = list(map(int, face[:4]))
            color = (0, 255, 0) if result else (0, 0, 255)
            thickness = 2
            cv2.rectangle(image, box, color, thickness, cv2.LINE_AA)

            id_name, score = user if result else (f"unknown_{idx}", 0.0)
            text = "{0} ({1:.2f})".format(id_name, score)
            position = (box[0], box[1] - 10)
            font = cv2.FONT_HERSHEY_SIMPLEX
            scale = 0.6
            cv2.putText(image, text, position, font, scale,
                        color, thickness, cv2.LINE_AA)

        cv2.imshow("face recognition", image)
        key = cv2.waitKey(1)
        if key == ord('q'):
            break
        end_hand = time.time()
        print(f'speed of a loop = {end_hand - start_hand} means {1/(end_hand - start_hand)} frames per second')

    cv2.destroyAllWindows()


if __name__ == '__main__':
    main()

Originally I use Opencv 4.6.0.66, tried to update opencv without success to resolve the problem. Tried to serach online, but rher is no

Answer 1

This is old question, but for anyone who has this problem

The problem is in this line

if image.shape[0] > 1000:
    image = cv2.resize(image, (0, 0),
                       fx=500 / image.shape[0], fy=500 / image.shape[0])

This problem is due to the fact that you recognize the image in a smaller size, but display it in the original size, you should rescale and reposition faces area(box) before put them on the original image in correct ratio, your ratio for rescale and reposition is original size / smaller size

so change your percentage and rescale/reposition box code to this:

    if (image.shape[0] > 1000):
            percentage = image.shape[0] / 500
        else:
            percentage = 1
            
        # Apply the percentage increase to each dimension of the box
        box[0] = int(box[0] * percentage)
        box[2] = int(box[2] * percentage)
        box[1] = int(box[1] * percentage)
        box[3] = int(box[3] * percentage)

Answer 2

I read a lot of posts, and finally decided to empirical define the correct displacement:

for idx, (face, feature) in enumerate(zip(faces, fetures)):
            result, user = match(face_recognizer, feature, dictionary)
            box = list(map(int, face[:4]))

            percentage = 1.2
        
            # Apply the percentage increase to each dimension of the box
            box[0] = int(box[0] + box[0] * percentage)
            box[2] = int(box[2] + box[2] * percentage)
            box[1] = int(box[1] + box[1] * percentage)
            box[3] = int(box[3] + box[3] * percentage)

            color = (0, 255, 0) if result else (0, 0, 255)
            thickness = 2
            cv2.rectangle(image, box, color, thickness, cv2.LINE_AA)