Unwanted Duplicate Triangle when selecting a Triangle from Mesh with Barycentric Selection. - Unity

Question

What I'm trying to do is select a specific triangle on any given mesh, so I'm using the Barycentric algorithm. However, as seen here, there are some strange behaviors at certain locations and always an unwanted second triangle. Best guess, it has something to do with inaccurate float precision of the Vector3.

---

Here is the Script I'm Currently Using:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class SpecificTriangleIndex : MonoBehaviour
{
    public Camera m_camera;
    public LayerMask layermask;

    public Vector3[] vertices;
    public Vector3[] normals;
    public int[] triangles;

    public List<int> triangleIndices = new List<int>(); ///Returned Triangle Indices -- What it thinks is selected.
    public List<Vector3> baryAllens = new List<Vector3>(); ///Position of Verticies of Returned Triangles

    private RaycastHit hit;
    private Transform objectHit;
    private Vector3 bary;

    void Update()
    {
        if (Input.GetMouseButton(0))
        {
            EobardThawne();
            Raycast();
        }
    }
    void Raycast()
    {
        Ray ray = m_camera.ScreenPointToRay(Input.mousePosition);

        if (Physics.Raycast(ray, out hit, layermask))
        {
            objectHit = hit.transform;
            MeshFilter meshFilter = hit.transform.gameObject.GetComponent<MeshFilter>();
            Mesh mesh = meshFilter.sharedMesh;
            vertices = mesh.vertices;
            normals = mesh.normals;
            triangles = mesh.triangles;

            Vector3 p = transform.InverseTransformPoint(hit.point);
            for (int i = 0; i < triangles.Length; i += 3)
            {
                Vector3 a = vertices[triangles[i]];
                Vector3 b = vertices[triangles[i + 1]];
                Vector3 c = vertices[triangles[i + 2]];

                bary = GetBarycentric(a, b, c, p);
                if (InTriangle(bary))
                {
                    triangleIndices.Add(i / 3);
                    baryAllens.Add(a + objectHit.position);
                    baryAllens.Add(b + objectHit.position);
                    baryAllens.Add(c + objectHit.position);
                }
            }
            for (int i = 0; i < baryAllens.Count; i += 3)
            {
                Vector3 v0 = baryAllens[i];
                Vector3 v1 = baryAllens[i + 1];
                Vector3 v2 = baryAllens[i + 2];

                Debug.DrawLine(v0, v1, Color.green);
                Debug.DrawLine(v1, v2, Color.green);
                Debug.DrawLine(v2, v0, Color.green);
            }
        }
    }

    Vector3 GetBarycentric(Vector2 v1, Vector2 v2, Vector2 v3, Vector2 p)
    {
        Vector3 B = new Vector3();
        B.x = ((v2.y - v3.y) * (p.x - v3.x) + (v3.x - v2.x) * (p.y - v3.y)) /
            ((v2.y - v3.y) * (v1.x - v3.x) + (v3.x - v2.x) * (v1.y - v3.y));
        B.y = ((v3.y - v1.y) * (p.x - v3.x) + (v1.x - v3.x) * (p.y - v3.y)) /
            ((v3.y - v1.y) * (v2.x - v3.x) + (v1.x - v3.x) * (v2.y - v3.y));
        B.z = 1 - B.x - B.y;
        return B;
    }

    bool InTriangle(Vector3 barycentric)
    {
        return (barycentric.x >= 0.0f) && (barycentric.x <= 1.0f)
            && (barycentric.y >= 0.0f) && (barycentric.y <= 1.0f)
            && (barycentric.z >= 0.0f); //(barycentric.z <= 1.0f)
    }

    private void OnDrawGizmos()
    {
        Gizmos.color = Color.yellow;
        Gizmos.DrawSphere(hit.point, .01f);

        Gizmos.color = Color.cyan;
        foreach (int i in triangleIndices)
        {
            Gizmos.DrawSphere(FindCenter(i * 3), .01f);
            Debug.DrawLine(hit.point, FindCenter(i * 3), Color.red);
        }
    }

    public void EobardThawne()
    {
        triangleIndices.Clear();
        baryAllens.Clear();
    }

    public Vector3 FindCenter(int i)
    {
        Vector3 v0 = transform.TransformPoint(vertices[triangles[i]]);
        Vector3 v1 = transform.TransformPoint(vertices[triangles[i + 1]]);
        Vector3 v2 = transform.TransformPoint(vertices[triangles[i + 2]]);
        Vector3 center = (v0 + v1 + v2) / 3;
        return center;
    }
}

Answer 1

Your triangles are duplicated because you fully ignore the Z axis!

You do all your triangle math assuming 2D space and only using Vector2!

=> Of course you get a second triangle since on your round objects there will always be two triangles overlapping along the Z axis!

---

You will have to use an actual 3D "point in triangle" test like e.g. the one from this thread

bool PointInTriangle(Vector3 a, Vector3 b, Vector3 c, Vector3 p)
{
    Vector3 d, e;
    double w1, w2;
    d = b - a;
    e = c - a;
 
    if (Mathf.Approximately(e.y, 0))
    {
        e.y = 0.0001f;
    }
     
    w1 = (e.x * (a.y - p.y) + e.y * (p.x - a.x)) / (d.x * e.y - d.y * e.x);
    w2 = (p.y - a.y - w1 * d.y) / e.y;
    return (w1 >= 0f) && (w2 >= 0.0) && ((w1 + w2) <= 1.0);
}

And then personally I would add some proper classes instead of arrays and do e.g.

public class SpecificTriangleIndex : MonoBehaviour
{
    public Camera m_camera;
    public LayerMask layermask;

    public class HitInfo
    {
        public Vector3 Point;
        public Triangle Triangle = new Triangle();
    }

    public class Triangle
    {
        public Vector3 A;
        public Vector3 B;
        public Vector3 C;

        public Vector3 Center => (A + B + C) / 3f;
    }

    private HitInfo currenHit;

    private void Update()
    {
        if (Input.GetMouseButton(0))
        {
            Raycast();
        }
    }

    private void Raycast()
    {
        var ray = m_camera.ScreenPointToRay(Input.mousePosition);

        if (Physics.Raycast(ray, out var hit, float.PositiveInfinity, layermask))
        {
            var meshFilter = hit.transform.gameObject.GetComponent<MeshFilter>();
            var mesh = meshFilter.sharedMesh;
            var vertices = mesh.vertices;
            var triangles = mesh.triangles;

            // Here you need to go into the local space of the hit not yourself!
            var p = hit.transform.InverseTransformPoint(hit.point);

            for (var i = 0; i < triangles.Length; i += 3)
            {
                var a = vertices[triangles[i]];
                var b = vertices[triangles[i + 1]];
                var c = vertices[triangles[i + 2]];

                if (PointInTriangle(a, b, c, p))
                {
                    if (currenHit == null)
                    {
                        currenHit = new HitInfo();
                    }

                    currenHit.Point = hit.point;

                    // as before you also want to convert back using the hit transform, not your own
                    currenHit.Triangle.A = hit.transform.TransformPoint(a);
                    currenHit.Triangle.B = hit.transform.TransformPoint(b);
                    currenHit.Triangle.C = hit.transform.TransformPoint(c);

                    // we only want one triangle anyway so we can skip the remaining triangles
                    break;
                }
            }
        }
        else
        {
            currenHit = null;
        }
    }

    private static bool PointInTriangle(Vector3 a, Vector3 b, Vector3 c, Vector3 p)
    {
        var d = b - a;
        var e = c - a;

        if (Mathf.Approximately(e.y, 0))
        {
            e.y = 0.0001f;
        }

        double w1 = (e.x * (a.y - p.y) + e.y * (p.x - a.x)) / (d.x * e.y - d.y * e.x);
        var w2 = (p.y - a.y - w1 * d.y) / e.y;
        return (w1 >= 0f) && (w2 >= 0.0) && ((w1 + w2) <= 1.0);
    }

    private void OnDrawGizmos()
    {
        // only draw if there is a hit
        if (currenHit == null) return;

        Gizmos.color = Color.yellow;
        Gizmos.DrawSphere(currenHit.Point, .01f);

        Gizmos.color = Color.cyan;
        Gizmos.DrawSphere(currenHit.Triangle.Center, .01f);

        Gizmos.color = Color.red;
        Gizmos.DrawLine(currenHit.Point, currenHit.Triangle.Center);

        Gizmos.color = Color.green;

        Gizmos.DrawLine(currenHit.Triangle.A, currenHit.Triangle.B);
        Gizmos.DrawLine(currenHit.Triangle.B, currenHit.Triangle.C);
        Gizmos.DrawLine(currenHit.Triangle.C, currenHit.Triangle.A);
    }
}

Maybe hard to see but this is how it looks like now