RealityKit – Visualizing Grid on detected planes

Question

I want to draw the mesh point on the detected plane as shown in the ARCore video link and I don't know how to achieve it.

Can you help me out to achieve this? Thanks in advance.

Answer 1

I improved Andy's code, to solve some crashes ( when trying to approach [0] on array), or creating a texture with invalid name, and also update it (extent is no longer in use):

Grid:

class Grid: Entity, HasModel, HasAnchoring {
    var planeAnchor: ARPlaneAnchor
    var planeGeometry: MeshResource!
    
    init(planeAnchor: ARPlaneAnchor) {
        self.planeAnchor = planeAnchor
        super.init()
        self.didSetup()
    }
        
    fileprivate func didSetup() {
      self.planeGeometry = .generatePlane(width: planeAnchor.transform.columns.3.x,
                                            depth: planeAnchor.transform.columns.3.z)
        var material = UnlitMaterial()
        guard let texture = try? TextureResource.load(named: "grid", in: nil)
        else{
          print("<Grid> ERROR: cannot create texutre!")
          return
        }
      
        let textureMat = MaterialParameters.Texture(texture)
        material.color = .init(tint: .white.withAlphaComponent(0.75),
                            texture: textureMat)
        let model = ModelEntity(mesh: planeGeometry, materials: [material])
        model.position = [planeAnchor.center.x, planeAnchor.center.y, planeAnchor.center.z]
        self.addChild(model)
    }
    
    fileprivate func didUpdate(anchor: ARPlaneAnchor) {
        self.planeGeometry = .generatePlane(width: anchor.transform.columns.3.x,
                                            depth: anchor.transform.columns.3.z)
        let pose: SIMD3<Float> = [anchor.center.x, anchor.center.y, anchor.center.z]
        let model = self.children.first as? ModelEntity
        model?.position = pose
    }
    required init() { fatalError("Hasn't been implemented yet") }
}

ARSessionDelegate:

  func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
for anchor in anchors{
  if let plane = anchor as? ARPlaneAnchor{
        let grid = Grid(planeAnchor: plane)
        grid.transform.matrix = plane.transform
        self.arView?.scene.anchors.append(grid)
        self.grids.append(grid)
  }


   }
  }

  func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
    for anchor in anchors{
      if let plane = anchor as? ARPlaneAnchor{
        let grid: Grid? = grids.filter { grd in
        grd.planeAnchor.identifier == plane.identifier }.first
        guard let updatedGrid: Grid = grid else { return }
        updatedGrid.transform.matrix = plane.transform
        updatedGrid.didUpdate(anchor: plane)
      }
    }
  }

ViewController (which I didn't change)

class ViewController: UIViewController {

@IBOutlet var arView: ARView!
var grids = [Grid]()

override func viewWillAppear(_ animated: Bool) {
    super.viewWillAppear(animated)
    arView.session.delegate = self
    
        let config = ARWorldTrackingConfiguration()
        config.planeDetection = [.horizontal, .vertical]
        arView.session.run(config)
    }
}

Answer 2

Expanding on Andy's answer:

As I really wanted to use a Metal texture and also the answer provided by Andy sadly didn't update the plane bounds for me, I came up with this solution:

Firstly, I used a custom Metal Surface Shader:

#include <metal_stdlib>
#include <RealityKit/RealityKit.h>

using namespace metal;
using namespace realitykit;

[[visible]]
void PlaneGridShader(surface_parameters params)
{
    // Constants for grid parameters. bigGrid is for the thicker lines, smallGrid for the finer grid. The bigger these values are, the more grid rows you get.
    constexpr float bigGridSize = 2.0;
    constexpr float smallGridSize = 10.0;
    constexpr float gridThickness = 0.005;
    constexpr float borderThickness = 0.003;
    
    //uv1 gets a global reference, whereas uv0 gets a local UV representation. This means that uv0 has coordinates from 0 to 1 for any grid size and is useful to create a border around the plane. uv1 is dependant on the grid size and offers a great way to create evenly spaced grid rows for differently sized planes
    float2 uv = params.geometry().uv1();
    float2 absoluteUV = params.geometry().uv0();
    
    //convenience values for grid size
    float bigU = uv.x * bigGridSize;
    float bigV = uv.y * bigGridSize;
    
    float smallU = uv.x * smallGridSize;
    float smallV = uv.y * smallGridSize;
    
    //Grid color in rgb values
    //half3 gridColor = half3(1,0.71,0);
    half3 gridColor = half3(1,1,1);
    
    //Make material appear unlit. If you use an unlit material, all grid lines appear black, so you need to dial all the pbr properties down
    params.surface().set_metallic(0);
    params.surface().set_specular(0);
    params.surface().set_clearcoat(0);
    params.surface().set_ambient_occlusion(1);
    
    //Sets the grid color. Can be moved down into if statements if multiple colors are needed for different parts of the grid
    params.surface().set_emissive_color(gridColor);
    params.surface().set_base_color(gridColor);
    
    //Inverse of Outer Border selection
    if (absoluteUV.x > borderThickness && absoluteUV.x < (1.0 - borderThickness) && absoluteUV.y > borderThickness && absoluteUV.y < (1.0 - borderThickness)) {
        //Thicker inner Grid (big)
        if (fract(bigU) < gridThickness || fract(bigV) < gridThickness || fract(bigU) > 1.0 - gridThickness || fract(bigV) > 1.0 - gridThickness) {
            params.surface().set_opacity(0.7);
        }
            else{
                //Small grid
                if (fract(smallU) < gridThickness || fract(smallV) < gridThickness || fract(smallU) > 1.0 - gridThickness || fract(smallV) > 1.0 - gridThickness) {
                    params.surface().set_opacity(0.25);
                }
                //Discard all fragments that do not meet grid criteria
                else{
                    discard_fragment();
                }
            }
    }
    //Constant Outer Border
    else{
        params.surface().set_opacity(0.8);
    }
    
}

Then, I created a Plane similarly to Andy, so thanks for that part :) Notably, it has some changes to the didUpdate() function and the way the plane extent is retrieved, as Andy's code is using functions that Apple has since deprecated. Also, I renamed it to "ARGrid", so it wouldn't conflict with SwiftUI's Grid().

import ARKit
import RealityKit

class ARGrid: Entity, HasModel, HasAnchoring {
    //Use identifier to later match the ARPlaneAnchor with the Grid
    var identifier: UUID
    var planeAnchor: ARPlaneAnchor
    var planeGeometry: MeshResource!
    
    
    init(id: UUID, planeAnchor: ARPlaneAnchor) {
        self.identifier = id
        self.planeAnchor = planeAnchor
        super.init()
        self.didSetup()
    }
    
    func didSetup() {
        self.planeGeometry = .generatePlane(width: planeAnchor.planeExtent.width, height: planeAnchor.planeExtent.height)
        
        var model = ModelEntity(mesh: planeGeometry, materials: [gridMaterial])
        model.position = [planeAnchor.center.x, 0, planeAnchor.center.z]
        // Rotate 90 degrees around the X-axis so plane is aligned to surface
        model.orientation = simd_quatf(angle: -Float.pi / 2, axis: [1, 0, 0])
        self.addChild(model)
    }
    
    func didUpdate(anchor: ARPlaneAnchor) {
        self.planeGeometry = .generatePlane(width: anchor.planeExtent.width, height: anchor.planeExtent.height)
        let pose: SIMD3<Float> = [anchor.center.x, 0, anchor.center.z]
        let model = self.children[0] as! ModelEntity
        model.model?.mesh = self.planeGeometry
        model.position = pose
    }
    required init() { fatalError("Hasn't been implemented yet") }
}

Lastly, I implemented a few changes in my ARView Delegate Functions:

 func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
        let filtered = anchors.compactMap({$0 as? ARPlaneAnchor})
        for anchor in filtered{
            let grid = ARGrid(id: anchor.identifier, planeAnchor: anchor)
            grid.transform.matrix = anchor.transform
            self.arView.scene.anchors.append(grid)
        }
    }
    
    func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
        let filtered = anchors.compactMap({$0 as? ARPlaneAnchor})
        let gridAnchors = arView.scene.anchors.compactMap({$0 as? ARGrid})
        for anchor in filtered{
            let updateAnchor = gridAnchors.first(where: {$0.identifier == anchor.identifier})!
            updateAnchor.transform.matrix = anchor.transform
            updateAnchor.didUpdate(anchor: anchor)
        }
    }

I'm still very new to Metal, this being my first project. So I'd be more than happy about suggestions for improving this, if I messed up somewhere. Hope this helps

Answer 3

Classic grid visualization in ARKit|RealityKit

In this post, I want to show you how to enable the "classic" visualization of the plane detection process with ARKit/RealityKit frameworks. But in this case, I don't promise you to visualize a grid in such a way that is visually pleasing, as it's implemented in Google ARCore.

However, in order to visualize the process of detecting horizontal and vertical planes similar to the behavior you can see in ARCore, you should use the scene reconstruction methodology, i.e. using a LiDAR scanner. But that's a different story, because you'll have to use Metal to implement a procedural texture with a soft mask for the front edge.

Here's a code:

import ARKit
import RealityKit

class Grid: Entity, HasModel, HasAnchoring {        
    var planeAnchor: ARPlaneAnchor
    var planeGeometry: MeshResource!
    
    init(planeAnchor: ARPlaneAnchor) {
        self.planeAnchor = planeAnchor
        super.init()
        self.didSetup()
    }
        
    fileprivate func didSetup() {            
        self.planeGeometry = .generatePlane(width: planeAnchor.extent.x,
                                            depth: planeAnchor.extent.z)
        var material = UnlitMaterial()
        material.color = .init(tint: .white.withAlphaComponent(0.999),
                            texture: .init(try! .load(named: "grid.png")))
        let model = ModelEntity(mesh: planeGeometry, materials: [material])
        model.position = [planeAnchor.center.x, 0, planeAnchor.center.z]
        self.addChild(model)
    }
    
    fileprivate func didUpdate(anchor: ARPlaneAnchor) {            
        self.planeGeometry = .generatePlane(width: anchor.extent.x,
                                            depth: anchor.extent.z)
        let pose: SIMD3<Float> = [anchor.center.x, 0, anchor.center.z]
        let model = self.children[0] as! ModelEntity
        model.position = pose
    }        
    required init() { fatalError("Hasn't been implemented yet") }
}

ViewController.swift

class ViewController: UIViewController {
    
    @IBOutlet var arView: ARView!
    var grids = [Grid]()

    override func viewWillAppear(_ animated: Bool) {
        super.viewWillAppear(animated)
        arView.session.delegate = self
        
        let config = ARWorldTrackingConfiguration()
        config.planeDetection = [.horizontal, .vertical]
        arView.session.run(config)
    }
}

ARSessionDelegate

extension ViewController: ARSessionDelegate {
    
    func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
        
        guard let planeAnchor = anchors.first as? ARPlaneAnchor else { return }
        let grid = Grid(planeAnchor: planeAnchor)
        grid.transform.matrix = planeAnchor.transform
        self.arView.scene.anchors.append(grid)
        self.grids.append(grid)
    }
    
    func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
        
        guard let planeAnchor = anchors[0] as? ARPlaneAnchor else { return }
        let grid: Grid? = grids.filter { grd in
            grd.planeAnchor.identifier == planeAnchor.identifier }[0]
        guard let updatedGrid: Grid = grid else { return }
        updatedGrid.transform.matrix = planeAnchor.transform
        updatedGrid.didUpdate(anchor: planeAnchor)
    }
}

Only coplanar detected planes may be updated.

P. S.

If you're interested in visualizing the Canonical Face Mask in RealityKit, take a look at this post.