AWS recognition face API: Crop teeth section (mouth) from face

Question

I'm cropping teeth section (mouth) from face based on x,y co-ordinates received from AWS recognition face API

this code is working and cropping the teeth section like

but I need only teeth section to be cropped.

AWS recognition API image response

 [
   {
     "BoundingBox": {
       "Width": 0.4604368805885315,
       "Height": 0.7760819792747498,
       "Left": 0.28602713346481323,
       "Top": 0.07381705939769745
     },
     "AgeRange": {
       "Low": 48,
       "High": 66
     },
     "Smile": {
       "Value": true,
       "Confidence": 99.91497802734375
     },
     "Eyeglasses": {
       "Value": false,
       "Confidence": 98.94174194335938
     },
     "Sunglasses": {
       "Value": false,
       "Confidence": 99.84471130371094
     },
     "Gender": {
       "Value": "Male",
       "Confidence": 99.57334899902344
     },
     "Beard": {
       "Value": false,
       "Confidence": 73.63420867919922
     },
     "Mustache": {
       "Value": false,
       "Confidence": 96.08769226074219
     },
     "EyesOpen": {
       "Value": true,
       "Confidence": 98.94685363769531
     },
     "MouthOpen": {
       "Value": true,
       "Confidence": 99.7721939086914
     },
     "Emotions": [
       {
         "Type": "HAPPY",
         "Confidence": 99.75701904296875
       },
       {
         "Type": "SURPRISED",
         "Confidence": 0.10713297128677368
       },
       {
         "Type": "CONFUSED",
         "Confidence": 0.056786004453897476
       },
       {
         "Type": "CALM",
         "Confidence": 0.02734198607504368
       },
       {
         "Type": "ANGRY",
         "Confidence": 0.020567195490002632
       },
       {
         "Type": "DISGUSTED",
         "Confidence": 0.01198340579867363
       },
       {
         "Type": "SAD",
         "Confidence": 0.011844608001410961
       },
       {
         "Type": "FEAR",
         "Confidence": 0.007329543586820364
       }
     ],
     "Landmarks": [
       {
         "Type": "eyeLeft",
         "X": 0.4020455777645111,
         "Y": 0.3627050220966339
       },
       {
         "Type": "eyeRight",
         "X": 0.6262026429176331,
         "Y": 0.379489928483963
       },
       {
         "Type": "mouthLeft",
         "X": 0.40419745445251465,
         "Y": 0.6104526519775391
       },
       {
         "Type": "mouthRight",
         "X": 0.5907381772994995,
         "Y": 0.6247860193252563
       },
       {
         "Type": "nose",
         "X": 0.49532997608184814,
         "Y": 0.48828810453414917
       },
       {
         "Type": "leftEyeBrowLeft",
         "X": 0.32399997115135193,
         "Y": 0.3045051097869873
       },
       {
         "Type": "leftEyeBrowRight",
         "X": 0.38662829995155334,
         "Y": 0.27300384640693665
       },
       {
         "Type": "leftEyeBrowUp",
         "X": 0.4492948651313782,
         "Y": 0.2880849540233612
       },
       {
         "Type": "rightEyeBrowLeft",
         "X": 0.578127920627594,
         "Y": 0.29742100834846497
       },
       {
         "Type": "rightEyeBrowRight",
         "X": 0.6459962725639343,
         "Y": 0.29183030128479004
       },
       {
         "Type": "rightEyeBrowUp",
         "X": 0.7144292593002319,
         "Y": 0.3330812454223633
       },
       {
         "Type": "leftEyeLeft",
         "X": 0.3629233241081238,
         "Y": 0.3603385388851166
       },
       {
         "Type": "leftEyeRight",
         "X": 0.4457237124443054,
         "Y": 0.36826738715171814
       },
       {
         "Type": "leftEyeUp",
         "X": 0.4013364613056183,
         "Y": 0.3494759500026703
       },
       {
         "Type": "leftEyeDown",
         "X": 0.40179359912872314,
         "Y": 0.37347349524497986
       },
       {
         "Type": "rightEyeLeft",
         "X": 0.5811655521392822,
         "Y": 0.3783351182937622
       },
       {
         "Type": "rightEyeRight",
         "X": 0.6668664813041687,
         "Y": 0.38298410177230835
       },
       {
         "Type": "rightEyeUp",
         "X": 0.6265660524368286,
         "Y": 0.36624279618263245
       },
       {
         "Type": "rightEyeDown",
         "X": 0.6238686442375183,
         "Y": 0.39007559418678284
       },
       {
         "Type": "noseLeft",
         "X": 0.4562915861606598,
         "Y": 0.5203639268875122
       },
       {
         "Type": "noseRight",
         "X": 0.5394821166992188,
         "Y": 0.5265129804611206
       },
       {
         "Type": "mouthUp",
         "X": 0.4932428300380707,
         "Y": 0.5806143283843994
       },
       {
         "Type": "mouthDown",
         "X": 0.48947831988334656,
         "Y": 0.6564671397209167
       },
       {
         "Type": "leftPupil",
         "X": 0.4020455777645111,
         "Y": 0.3627050220966339
       },
       {
         "Type": "rightPupil",
         "X": 0.6262026429176331,
         "Y": 0.379489928483963
       },
       {
         "Type": "upperJawlineLeft",
         "X": 0.28082960844039917,
         "Y": 0.37847602367401123
       },
       {
         "Type": "midJawlineLeft",
         "X": 0.3077985942363739,
         "Y": 0.6443988680839539
       },
       {
         "Type": "chinBottom",
         "X": 0.48529136180877686,
         "Y": 0.7894702553749084
       },
       {
         "Type": "midJawlineRight",
         "X": 0.7061411738395691,
         "Y": 0.6732134819030762
       },
       {
         "Type": "upperJawlineRight",
         "X": 0.77140212059021,
         "Y": 0.4138971269130707
       }
     ],
     "Pose": {
       "Roll": 3.0064163208007812,
       "Yaw": -2.569990634918213,
       "Pitch": 8.883845329284668
     },
     "Quality": {
       "Brightness": 76.55046844482422,
       "Sharpness": 94.08262634277344
     },
     "Confidence": 99.99818420410156
   }
 ]

Node Js code for crop using gm ImageMagick library

const init  = async () => {
try {
    console.info("Process Started");
    const size = await getImageSize("passport-photo.jpeg");
    console.info("get Image Size: ", size);
    const faceDetails = await getFaceDetailsFromImage();

    // Uploded image width height
    const imageWidth = size.width;
    const imageHeight = size.height;

    // Face detail width height
    const faceDetailWidth =  Math.round(faceDetails[0].BoundingBox.Width * imageWidth);
    const faceDetailHeight =  Math.round(faceDetails[0].BoundingBox.Height * imageHeight);

    // Coordinates for the mouth
    const faceDetailMouthLeft = faceDetails[0].Landmarks.filter(o => o.Type === "mouthLeft");
    const faceDetailMouthRight = faceDetails[0].Landmarks.filter(o => o.Type === "mouthRight");
    const faceDetailMouthUp = faceDetails[0].Landmarks.filter(o => o.Type === "mouthUp");
    const faceDetailMouthDown = faceDetails[0].Landmarks.filter(o => o.Type === "mouthDown");

    // Find x and y point from where the cropping needs to be started
    const xPoint = Math.round(faceDetailMouthLeft[0].X * imageWidth);
    const yPoint = Math.round(faceDetailMouthUp[0].Y * imageHeight);

    // Width height for which image needs to be cut from start index

    const width = ((faceDetailMouthRight[0].X - faceDetailMouthLeft[0].X) * imageWidth)
    const height = ((faceDetailMouthDown[0].Y - faceDetailMouthUp[0].Y) * imageHeight)

    console.log("xPoint:" + xPoint + ", yPoint:" + yPoint + ", faceDetailWidth:" + faceDetailWidth + ", faceDetailHeight:" + faceDetailHeight)
    gm('passport-photo.jpeg')

    // Invoke crop function 
    .crop(width, height, xPoint, yPoint, true) 

    // Process and Write the image 
    .write("crop5.png", function (err) {
        console.error(err);
        if (!err) console.log('done'); 
    }); 

} catch (error){
    console.error(error);
}
}
init();

original image

John Rotenstein · Accepted Answer

Your calculations for the mouth co-ordinates appear to be correct.

However, I see you are using:

.crop(width, height, xPoint, yPoint, true)

As you say, this "accepts the parameter as a percentage value", which probably isn't what you want.

AWS recognition face API: Crop teeth section (mouth) from face

Answers (1)

Related Questions