How to implement a Billboard effect (LookAt camera) in RealityKit?

Question

I want to achieve the billboard effect in RealityKit (the plane always look at the camera), I used the Entity.Look() method, but the result is weird, I can't even see the plane, the scripts I used as below, so, what is the problem?

struct ARViewContainer: UIViewRepresentable {
    
    func makeUIView(context: Context) -> ARView {
        
        let arView = ARView(frame: .zero)
        let config = ARWorldTrackingConfiguration()
        config.planeDetection = .horizontal
        arView.session.run(config, options:[ ])
        arView.session.delegate = arView
        arView.createPlane()
        return arView        
    }
    
    func updateUIView(_ uiView: ARView, context: Context) { }
}

var planeMesh = MeshResource.generatePlane(width: 0.15, height: 0.15)
var planeMaterial = SimpleMaterial(color:.white,isMetallic: false)
var planeEntity = ModelEntity(mesh:planeMesh,materials:[planeMaterial])
var arCameraPostion : SIMD3<Float>!
var isPlaced = false

extension ARView : ARSessionDelegate{
    func createPlane(){
        let planeAnchor = AnchorEntity(plane:.horizontal)
        planeAnchor.addChild(planeEntity)
        self.scene.addAnchor(planeAnchor)
        //planeAnchor.transform.rotation = simd_quatf(angle: .pi, axis: [0,1,0])

    }
  
    public func session(_ session: ARSession, didUpdate frame: ARFrame){
        guard let arCamera = session.currentFrame?.camera else { return }
        if isPlaced {
            arCameraPostion = SIMD3(arCamera.transform.columns.3.x,0,arCamera.transform.columns.3.z)
            planeEntity.look(at: arCameraPostion, from: planeEntity.position, upVector: [0, 1, 0],relativeTo: nil)
        }
    }
   
    public func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
        isPlaced = true
    }
}

Answer 1

Answer for visionOS 2.0. It's now built in and can be enabled with 1 line!

Now you can simply do this:

if #available(visionOS 2.0, *) {
  myEntity.components.set(BillboardComponent())
}

🎉

or if your app requires a minimum version of visionOS 2.0 you can simply do this:

myEntity.components.set(BillboardComponent())

In these examples, myEntity is a reference to a RealityKit Entity.

Answer 2

in visionOS, you can try Apple's approach and apply BillboardComponent.

The below code comes from Swift Splash

BillboardComponent.swift

import Foundation
import RealityKit

/// The component that marks an entity as a billboard object which will always face the camera.
public struct BillboardComponent: Component, Codable {
    public init() {
    }
}

and another file BillboardSystem

import ARKit
import Foundation
import OSLog
import RealityKit
import simd
import SwiftUI

/// An ECS system that points all entities containing a billboard component at the camera.
public struct BillboardSystem: System {
    
    static let query = EntityQuery(where: .has(SwiftSplashTrackPieces.BillboardComponent.self))
    
    private let arkitSession = ARKitSession()
    private let worldTrackingProvider = WorldTrackingProvider()
    
    public init(scene: RealityKit.Scene) {
        setUpSession()
    }
    
    func setUpSession() {
        
        Task {
            do {
                try await arkitSession.run([worldTrackingProvider])
            } catch {
                os_log(.info, "Error: \(error)")
            }
        }
    }
    
    public func update(context: SceneUpdateContext) {
        
        let entities = context.scene.performQuery(Self.query).map({ $0 })
        
        guard !entities.isEmpty,
              let pose = worldTrackingProvider.queryDeviceAnchor(atTimestamp: CACurrentMediaTime()) else { return }
        
        let cameraTransform = Transform(matrix: pose.originFromAnchorTransform)
        
        for entity in entities {
            entity.look(at: cameraTransform.translation,
                        from: entity.scenePosition,
                        relativeTo: nil,
                        forward: .positiveZ)
        }
    }
}

Altough, keep in mind that you must then use it in RealityKitComposer Pro and also register it in app initilizer.

Answer 3

session(_:didUpdate:) method

Try the following logic to implement a "billboard" behavior for RealityKit camera. You can use this code as a starting point. It generates a rotation of the model around its local Y axis based on camera position.

import RealityKit
import ARKit

class ViewController: UIViewController {

    @IBOutlet var arView: ARView!
    var model = Entity()

    override func viewDidLoad() {
        super.viewDidLoad()

        arView.session.delegate = self
        
        let config = ARWorldTrackingConfiguration()
        arView.session.run(config)

        self.model = try! ModelEntity.load(named: "drummer")
        let anchor = AnchorEntity(world: [0, 0, 0])
        anchor.addChild(self.model)
        arView.scene.anchors.append(anchor)
    }
}

A pivot point of the model must be in the center of it (not at some distance from the model).

extension ViewController: ARSessionDelegate {

    func session(_ session: ARSession, didUpdate frame: ARFrame) {

        let camTransform: float4x4 = arView.cameraTransform.matrix

        let alongXZPlane: simd_float4 = camTransform.columns.3

        let yaw: Float = atan2(alongXZPlane.x - model.position.x,
                               alongXZPlane.z - model.position.z)    
        print(yaw)

        // Identity matrix 4x4
        var positionAndScale = float4x4()
        
        // position
        positionAndScale.columns.3.z = -0.25
        
        // scale
        positionAndScale.columns.0.x = 0.01
        positionAndScale.columns.1.y = 0.01
        positionAndScale.columns.2.z = 0.01
        
        // orientation matrix
        let orientation = Transform(pitch: 0, yaw: yaw, roll: 0).matrix

        // matrices multiplication
        let transform = simd_mul(positionAndScale, orientation)   
        
        self.model.transform.matrix = transform
    }
}

subscribe(to:on:_:) method

Alternatively, you can implement a subscription to the event stream.

import RealityKit
import Combine

class ViewController: UIViewController {

    @IBOutlet var arView: ARView!
    var model = Entity()
    var subs: [AnyCancellable] = []

    override func viewDidLoad() {
        super.viewDidLoad()    
        self.model = try! ModelEntity.load(named: "drummer")
        let anchor = AnchorEntity(world: [0, 0, 0])
        anchor.addChild(self.model)
        arView.scene.anchors.append(anchor)
        
        arView.scene.subscribe(to: SceneEvents.Update.self) { _ in
            let camTransform: float4x4 = self.arView.cameraTransform.matrix
            let alongXZPlane: simd_float4 = camTransform.columns.3    
            let yaw: Float = atan2(alongXZPlane.x - self.model.position.x,
                                   alongXZPlane.z - self.model.position.z)
    
            var positionAndScale = float4x4()    
            positionAndScale.columns.3.z = -0.25    
            positionAndScale.columns.0.x = 0.01
            positionAndScale.columns.1.y = 0.01
            positionAndScale.columns.2.z = 0.01    
            let orientation = Transform(pitch: 0, yaw: yaw, roll: 0).matrix
            let transform = simd_mul(positionAndScale, orientation)    
            self.model.transform.matrix = transform               
        }.store(in: &subs)
    }
}