Projecting the ARKit face tracking 3D mesh to 2D image coordinates

Question

I am collecting face mesh 3D vertices using ARKit. I have read: Mapping image onto 3D face mesh and Tracking and Visualizing Faces.

---

I have the following struct:

 struct CaptureData {
        var vertices: [SIMD3<Float>]
        var verticesformatted: String {
            let verticesDescribed = vertices.map({ "\($0.x):\($0.y):\($0.z)" }).joined(separator: "~")
            return "<\(verticesDescribed)>"
        }
    }

---

I have a Strat button to capture vertices:

@IBAction private func startPressed() {
    captureData = [] // Clear data
    currentCaptureFrame = 0 //inital capture frame
    fpsTimer = Timer.scheduledTimer(withTimeInterval: 1/fps, repeats: true, block: {(timer) -> Void in self.recordData()})
}

---

 private var fpsTimer = Timer()
    private var captureData: [CaptureData] = [CaptureData]()
    private var currentCaptureFrame = 0

---

And a stop button to stop capturing (save the data):

 @IBAction private func stopPressed() {
        do {
            fpsTimer.invalidate() //turn off the timer
            let capturedData = captureData.map{$0.verticesformatted}.joined(separator:"")
            let dir: URL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).last! as URL
            let url = dir.appendingPathComponent("facedata.txt")
            try capturedData.appendLineToURL(fileURL: url as URL)
        }
        catch {
            print("Could not write to file")
        }
    }

---

Function for recoding data

 private func recordData() {
        guard let data = getFrameData() else { return }
        captureData.append(data)
        currentCaptureFrame += 1
    }

---

Function for get frame data

private func getFrameData() -> CaptureData? {
    let arFrame = sceneView?.session.currentFrame!
    guard let anchor = arFrame?.anchors[0] as? ARFaceAnchor else {return nil}
    let vertices = anchor.geometry.vertices
    let data = CaptureData(vertices: vertices)
    return data
}

---

ARSCN extension:

extension ViewController: ARSCNViewDelegate {
    
    func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
        guard let faceAnchor = anchor as? ARFaceAnchor else { return }
        currentFaceAnchor = faceAnchor
        if node.childNodes.isEmpty, let contentNode = selectedContentController.renderer(renderer, nodeFor: faceAnchor) {
            node.addChildNode(contentNode)
        }
        selectedContentController.session = sceneView?.session
        selectedContentController.sceneView = sceneView
    }
    
    /// - Tag: ARFaceGeometryUpdate
    func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
        guard anchor == currentFaceAnchor,
            let contentNode = selectedContentController.contentNode,
            contentNode.parent == node
            else { return }
        selectedContentController.session = sceneView?.session
        selectedContentController.sceneView = sceneView
        selectedContentController.renderer(renderer, didUpdate: contentNode, for: anchor)
    }
}

---

I am trying to use the example code from Tracking and Visualizing Faces:

// Transform the vertex to the camera coordinate system.
float4 vertexCamera = scn_node.modelViewTransform * _geometry.position;

// Camera projection and perspective divide to get normalized viewport coordinates (clip space).
float4 vertexClipSpace = scn_frame.projectionTransform * vertexCamera;
vertexClipSpace /= vertexClipSpace.w;

// XY in clip space is [-1,1]x[-1,1], so adjust to UV texture coordinates: [0,1]x[0,1].
// Image coordinates are Y-flipped (upper-left origin).
float4 vertexImageSpace = float4(vertexClipSpace.xy * 0.5 + 0.5, 0.0, 1.0);
vertexImageSpace.y = 1.0 - vertexImageSpace.y;

// Apply ARKit's display transform (device orientation * front-facing camera flip).
float4 transformedVertex = displayTransform * vertexImageSpace;

// Output as texture coordinates for use in later rendering stages.
_geometry.texcoords[0] = transformedVertex.xy;

---

I aslo read about projection point (but not sure which one is more applicable still):

func projectPoint(_ point: SCNVector3) -> SCNVector3

My question is how to use the example code above and transform the collected 3D face mesh vertices to 2D image coordinates??

I would like to get the 3D mesh vertices together with their corresponding 2D coordinates.

Currently, I can capture the face mesh points like so: <mesh_x: mesh_ y: mesh_ z:...>

I would to convert my mesh points to the image coordinates and show them together like so:

Expected result: <mesh_x: mesh_ y: mesh_ z:img_x: img_y...>

Any suggestions? Thanks in advance!

Answer 1

Maybe you can use the projectPoint function of the SCNSceneRenderer.

extension ARFaceAnchor{
    // struct to store the 3d vertex and the 2d projection point
    struct VerticesAndProjection {
        var vertex: SIMD3<Float>
        var projected: CGPoint
    }
    
    // return a struct with vertices and projection
    func verticeAndProjection(to view: ARSCNView) -> [VerticesAndProjection]{
        
        let points = geometry.vertices.compactMap({ (vertex) -> VerticesAndProjection? in

            let col = SIMD4<Float>(SCNVector4())
            let pos = SIMD4<Float>(SCNVector4(vertex.x, vertex.y, vertex.z, 1))
            
            let pworld = transform * simd_float4x4(col, col, col, pos)
            
            let vect = view.projectPoint(SCNVector3(pworld.position.x, pworld.position.y, pworld.position.z))

            let p = CGPoint(x: CGFloat(vect.x), y: CGFloat(vect.y))
            return VerticesAndProjection(vertex:vertex, projected: p)
            })
        
        return points
    }
}

Here is a convenient way to get the position:

extension matrix_float4x4 {
    
    /// Get the position of the transform matrix.
    public var position: SCNVector3 {
        get{
            return SCNVector3(self[3][0], self[3][1], self[3][2])
        }
    }
}

If you want to check that the projection is ok, add a debug subview to the ARSCNView instance, then, with a couple of others extensions to draw the 2d points on a view such as:

extension UIView{
    
    private struct drawCircleProperty{
        static let circleFillColor = UIColor.green
        static let circleStrokeColor = UIColor.black
        static let circleRadius: CGFloat = 3.0
    }
    
    func drawCircle(point: CGPoint) {
    
        let circlePath = UIBezierPath(arcCenter: point, radius: drawCircleProperty.circleRadius, startAngle: CGFloat(0), endAngle: CGFloat(Double.pi * 2.0), clockwise: true)
        let shapeLayer = CAShapeLayer()
        shapeLayer.path = circlePath.cgPath
        shapeLayer.fillColor = drawCircleProperty.circleFillColor.cgColor
        shapeLayer.strokeColor = drawCircleProperty.circleStrokeColor.cgColor
        
        self.layer.addSublayer(shapeLayer)
    }
    
    func drawCircles(points: [CGPoint]){
        
        self.clearLayers()
        
        for point in points{
            self.drawCircle(point: point)
        }
    }
    
    func clearLayers(){
        if let subLayers = self.layer.sublayers {
            for subLayer in subLayers {
                subLayer.removeFromSuperlayer()
            }
        }
    }

You can compute the projection, and draw the points with:

let points:[ARFaceAnchor.VerticesAndProjection] = faceAnchor.verticeAndProjection(to: sceneView)
     
// keep only the projected points
let projected = points.map{ $0.projected}
// draw the points !
self.debugView?.drawCircles(points: projected)

I can see all the 3d vertices projected on the 2d screen (picture generated by https://thispersondoesnotexist.com).

I added this code to the Apple demo project, available here https://github.com/hugoliv/projectvertices.git