How to disable infinity sky ray in ray tracing?

Question

I am learning how to render voxels using Unity. I created a simple shader using the 3D DDA algorithm. When I look at the ground I have 80-200 fps depending on the landscape and distances. But when I look at the sky my fps drops to 20-30. I understand that the ray is going, it doesn’t encounter any obstacles and is simply wasting GPU time. But I don't understand at all how to fix this moment, how to determine the sky? How do other ray tracers work, why in other games with ray tracing I don't get this effect? ​​How to optimize this DDA?

Renderig voxels

Rendeding sky

Shader "Custom/TerrainShader"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
        _VoxelTex ("VoxTexture", 3D) = "white" {}

    }
    SubShader
    {
        // No culling or depth
        Cull Off ZWrite Off ZTest Always

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #pragma target 3.0
            #include "UnityCG.cginc"

            sampler2D _MainTex;
            sampler2D _CameraDepthTexture;
            sampler3D _VoxelTex;

            uniform float4x4 _CamFrustum, _CamToWorld;


            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                float4 vertex : SV_POSITION;
                float3 ray : TEXTCOORD1;
            };

            v2f vert(appdata v)
            {
                v2f o;
                half index = v.vertex.z;
                v.vertex.z = 0;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = v.uv;

                o.ray = _CamFrustum[(int)index].xyz;
                o.ray /= abs(o.ray.z);

                o.ray = mul(_CamToWorld, o.ray);

                return o;
            }


            fixed4 frag(v2f i) : SV_Target
            {
                half3 rayStart = _WorldSpaceCameraPos;
                half3 rayDir = normalize(i.ray.xyz);

                float infinity = 99999999;

                half3 rayUnitStepSize = half3(
                    rayDir.x != 0.0 ? abs(1.0 / rayDir.x) : infinity,
                    rayDir.y != 0.0 ? abs(1.0 / rayDir.y) : infinity,
                    rayDir.z != 0.0 ? abs(1.0 / rayDir.z) : infinity
                );


                half3 mapCheck = floor(rayStart);
                half3 rayLen;
                half3 step;


                if (rayDir.x < 0)
                {
                    step.x = -1;
                    rayLen.x = (rayStart.x - (mapCheck.x)) * rayUnitStepSize.x;
                }
                else
                {
                    step.x = 1;
                    rayLen.x = ((mapCheck.x + 1) - rayStart.x) * rayUnitStepSize.x;
                }

                if (rayDir.z < 0)
                {
                    step.z = -1;
                    rayLen.z = (rayStart.z - (mapCheck.z)) * rayUnitStepSize.z;
                }
                else
                {
                    step.z = 1;
                    rayLen.z = ((mapCheck.z + 1) - rayStart.z) * rayUnitStepSize.z;
                }

                if (rayDir.y < 0)
                {
                    step.y = -1;
                    rayLen.y = (rayStart.y - (mapCheck.y)) * rayUnitStepSize.y;
                }
                else
                {
                    step.y = 1;
                    rayLen.y = ((mapCheck.y + 1) - rayStart.y) * rayUnitStepSize.y;
                }

                float4 targetColor;

                bool isTileFound = false;
                half maxDist = 256.0f;
                half dist = 0.0f;
                targetColor = float4(1, 1, 1, 1);
                [loop] while (!isTileFound && dist < maxDist)
                {
                    if (rayLen.x < rayLen.z && rayLen.x < rayLen.y)
                    {
                        mapCheck.x += step.x;
                        dist = rayLen.x;
                        rayLen.x += rayUnitStepSize.x;
                    }
                    else if (rayLen.z < rayLen.y)
                    {
                        mapCheck.z += step.z;
                        dist = rayLen.z;
                        rayLen.z += rayUnitStepSize.z;
                    }
                    else
                    {
                        mapCheck.y += step.y;
                        dist = rayLen.y;
                        rayLen.y += rayUnitStepSize.y;
                    }

                    float4 color = tex3D(_VoxelTex, float3(mapCheck.x, mapCheck.y, mapCheck.z) / 255.0);
                    if (color.a != 0)
                    {
                        return color;
                    }
                }

                return targetColor;
            }
            ENDCG
        }
    }
}

Thanks!

I tried setting the upper limits of the sky, counting the tracer's steps and similar primitive things, but they did not give a strong increase in fps and also continued to significantly reduce fps.