How to Send and Receive Point Cloud Data Using Photon PUN2 in Unity?

Question

Question:

I am working on a Unity project where I need to capture point cloud data using a Kinect sensor on a PC and send this data to a VR user using Photon PUN2. The goal is to display the point cloud data in real-time in the VR environment. The PC and VR projects are separate Unity projects.

What I have done

1. Setup

• PC side: Kinect device is connected, and point cloud data is captured successfully.
• VR side: Intended to receive and display the point cloud data.

2. Unity and Photon Setup

• I have integrated Photon PUN2 for network communication.
• Both the PC and VR sides connect to the Photon server and join the same room.

Code PC Side(Kinect Script):

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Microsoft.Azure.Kinect.Sensor;
using System.Threading.Tasks;
using Photon.Pun;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;

public class KinectScript : MonoBehaviourPun
{
    Device kinect;
    int num;
    Vector3[] vertices;
    Color32[] colors;
    Transformation transformation;
    Mesh mesh;
    int[] indices;

    void Start()
    {
        Debug.Log("KinectScript Start");

        InitKinect();
        InitMesh();
    }

    void Update()
    {
        Task t = KinectLoop();
    }

    private void InitKinect()
    {
        Debug.Log("InitKinect Start");

        kinect = Device.Open(0);
        kinect.StartCameras(new DeviceConfiguration
        {
            ColorFormat = ImageFormat.ColorBGRA32,
            ColorResolution = ColorResolution.R720p,
            DepthMode = DepthMode.NFOV_2x2Binned,
            SynchronizedImagesOnly = true,
            CameraFPS = FPS.FPS30
        });
        transformation = kinect.GetCalibration().CreateTransformation();

        Debug.Log("InitKinect Complete");
    }

    private void InitMesh()
    {
        Debug.Log("InitMesh Start");

        int width = kinect.GetCalibration().DepthCameraCalibration.ResolutionWidth;
        int height = kinect.GetCalibration().DepthCameraCalibration.ResolutionHeight;
        num = width * height;

        vertices = new Vector3[num];
        colors = new Color32[num];
        indices = new int[num];

        for (int i = 0; i < num; i++)
        {
            indices[i] = i;
        }

        mesh = new Mesh();
        mesh.indexFormat = UnityEngine.Rendering.IndexFormat.UInt32;
        GetComponent<MeshFilter>().mesh = mesh;

        Debug.Log("InitMesh Complete");
    }

    private async Task KinectLoop()
    {
        Debug.Log("KinectLoop Start");

        while (true)
        {
            using (Capture capture = await Task.Run(() => kinect.GetCapture()).ConfigureAwait(true))
            {
                if (capture != null)
                {
                    Image colorImage = transformation.ColorImageToDepthCamera(capture);
                    if (colorImage != null)
                    {
                        BGRA[] colorArray = colorImage.GetPixels<BGRA>().ToArray();

                        Image xyzImage = transformation.DepthImageToPointCloud(capture.Depth);
                        if (xyzImage != null)
                        {
                            Short3[] xyzArray = xyzImage.GetPixels<Short3>().ToArray();

                            for (int i = 0; i < num; i++)
                            {
                                vertices[i].x = xyzArray[i].X * 0.001f;
                                vertices[i].y = -xyzArray[i].Y * 0.001f;
                                vertices[i].z = xyzArray[i].Z * 0.001f;
                                colors[i].b = colorArray[i].B;
                                colors[i].g = colorArray[i].G;
                                colors[i].r = colorArray[i].R;
                                colors[i].a = 255;
                            }

                            Debug.Log("First vertex position: " + vertices[0].ToString());
                            Debug.Log("First vertex color: " + colors[0].ToString());

                            for (int i = 0; i < 10; i++)
                            {
                                Debug.Log("Vertex " + i + " position: " + vertices[i].ToString());
                                Debug.Log("Vertex " + i + " color: " + colors[i].ToString());
                            }

                            // Update Mesh
                            mesh.vertices = vertices;
                            mesh.colors32 = colors;
                            mesh.SetIndices(indices, MeshTopology.Points, 0);
                            mesh.RecalculateBounds();

                            SendPointCloudData(vertices, colors);
                        }
                        else
                        {
                            Debug.LogError("XYZ image is null");
                        }
                    }
                    else
                    {
                        Debug.LogError("Color image is null");
                    }
                }
                else
                {
                    Debug.LogError("Capture is null");
                }
            }
        }
    }

    private void SendPointCloudData(Vector3[] vertices, Color32[] colors)
    {
        PointCloudData data = new PointCloudData(vertices, colors);
        byte[] serializedData = SerializePointCloudData(data);
        Debug.Log("Sending point cloud data...");
        photonView.RPC("ReceivePointCloudData", RpcTarget.All, serializedData);
    }

    private byte[] SerializePointCloudData(PointCloudData data)
    {
        using (MemoryStream stream = new MemoryStream())
        {
            BinaryFormatter formatter = new BinaryFormatter();
            formatter.Serialize(stream, data);
            return stream.ToArray();
        }
    }

    private void OnDestroy()
    {
        kinect.StopCameras();
    }
}

[System.Serializable]
public class PointCloudData
{
    public Vector3[] vertices;
    public Color32[] colors;

    public PointCloudData(Vector3[] vertices, Color32[] colors)
    {
        this.vertices = vertices;
        this.colors = colors;
    }
}

VR Side(PointCloudReceiver):

using UnityEngine;
using Photon.Pun;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;

public class PointCloudReceiver : MonoBehaviourPun
{
    public GameObject pointCloudObject;
    private Mesh mesh;
    private Vector3[] vertices;
    private Color32[] colors;
    private int num;
    private int[] indices;

    void Start()
    {
        InitMesh();
    }

    private void InitMesh()
    {
        mesh = new Mesh();
        mesh.indexFormat = UnityEngine.Rendering.IndexFormat.UInt32;
        pointCloudObject.GetComponent<MeshFilter>().mesh = mesh;
    }

    [PunRPC]
    void ReceivePointCloudData(byte[] serializedData)
    {
        Debug.Log("ReceivePointCloudData called");

        PointCloudData data = DeserializePointCloudData(serializedData);

        Debug.Log("Received point cloud data");
        Debug.Log("First vertex position: " + data.vertices[0].ToString());
        Debug.Log("First vertex color: " + data.colors[0].ToString());

        for (int i = 0; i < 10; i++)
        {
            Debug.Log("Received vertex " + i + " position: " + data.vertices[i].ToString());
            Debug.Log("Received vertex " + i + " color: " + data.colors[i].ToString());
        }

        UpdateMesh(data);
    }

    private PointCloudData DeserializePointCloudData(byte[] data)
    {
        using (MemoryStream stream = new MemoryStream(data))
        {
            BinaryFormatter formatter = new BinaryFormatter();
            return (PointCloudData)formatter.Deserialize(stream);
        }
    }

    private void UpdateMesh(PointCloudData data)
    {
        vertices = data.vertices;
        colors = data.colors;
        num = vertices.Length;
        indices = new int[num];

        for (int i = 0; i < num; i++)
        {
            indices[i] = i;
        }

        mesh.vertices = vertices;
        mesh.colors32 = colors;
        mesh.SetIndices(indices, MeshTopology.Points, 0);
        mesh.RecalculateBounds();

        Debug.Log("Mesh updated with " + num + " vertices.");
    }
}

[System.Serializable]
public class PointCloudData
{
    public Vector3[] vertices;
    public Color32[] colors;

    public PointCloudData(Vector3[] vertices, Color32[] colors)
    {
        this.vertices = vertices;
        this.colors = colors;
    }
}

Problem The point cloud data is successfully captured and displayed on the PC side, as confirmed by the debug logs and visual output. However, the VR side does not receive any data. No Debug.Log messages from the ReceivePointCloudData method are displayed, indicating that the RPC call might not be working correctly.

I have verified that:

1. Both the PC and VR sides are connected to the Photon server and are in the same room.
2. The PhotonView components are correctly attached and configured on both sides.
3. The PhotonView components observe the appropriate scripts (KinectScript and PointCloudReceiver).

Despite these steps, the VR side still does not receive the point cloud data.

Questions

1. How can I ensure that the RPC call ReceivePointCloudData is correctly received and processed on the VR side?
2. Are there any additional steps or configurations required to correctly send and receive complex data types like point cloud data using Photon PUN2?

Thank you for your help!