Agent stuck in loop MLAgents in Unity

Question

Goal: The agent need to shoot red dots. The agent shoots a ball that when it hits a red dot, it removes the dot and himself but when it hits an obstacle, only the ball disappear. (There's no bounce)

Example Level 1 The agent on the first level does really good, the problem is when i train on the other levels.

Example Level 2 Example Level 3 (Obstacles are moving left and right) The agent does some success but after many episodes it gets stuck on loop, going left and right.

public class Capsule_ML : Agent
{
    public GameObject sfera;
    private int num, num_rossi; //tiri, numero rossi
    public bool activate_shoot = false;
    public Text num_sfere, punteggio, tempo_Rimanente;
    //private float time_Left;
    //private bool timer_Active;
    public override void OnEpisodeBegin()
    {
        Episode_Debug.Episode += 1;
        this.transform.rotation = Quaternion.Euler(Vector3.zero);
        activate_shoot = false;
        //timer_Active = true;
        //time_Left = 60;

        Active(this.transform.parent);
        SearchRossiInParent(this.transform.parent);
        punteggio.text = "#ROSSI: " + num_rossi;
        num_sfere.text = "#SFERE: " + num;
    }

    //private void Update()
    //{
    //    if (timer_Active)
    //    {
    //        time_Left -= Time.deltaTime;
    //        tempo_Rimanente.text = "Tempo Rimanente: " + time_Left + "s";
    //        if (time_Left <= 0)
    //        {
    //            timer_Active = false;
    //            TimeError();
    //        }
    //    }



    //}


    public void TimeError()
    {
        AddReward(-1.0f);
        Episode_Debug.Fail += 1;
        EndEpisode();
    }

    public override void CollectObservations(VectorSensor sensor)
    {
        
    }

    public override void OnActionReceived(float[] action)
    {
        this.transform.Rotate(Vector3.back * action[0] * 60 * Time.deltaTime);
        this.transform.Rotate(-Vector3.back * action[1] * 60 * Time.deltaTime);
        

        if(UnityEditor.TransformUtils.GetInspectorRotation(this.transform).z < -90 || UnityEditor.TransformUtils.GetInspectorRotation(this.transform).z > 90)
        {
            AddReward(-1.0f);
            Episode_Debug.Fail += 1;
            EndEpisode();
        }

        RaycastHit2D hit = Physics2D.Raycast(this.transform.position, transform.TransformDirection(Vector3.down), Mathf.Infinity);
        if (activate_shoot == false && hit.collider.tag == "red")
        {
            Debug.Log("Raycast colpisce " + hit.collider.name);
            activate_shoot = true;
            Debug.Log(activate_shoot);

            Shoot();
            //AddReward(1.0f);
        }


        //if (activate_shoot == false && action[2] == 1)
        //{
        //    activate_shoot = true;
        //    Shoot();
        //}


        if ((num_rossi == 0 && num == 0) || (num_rossi == 0 && num > 0))
        {
            Episode_Debug.Success += 1;
            AddReward(1.0f);
            EndEpisode();
        }

        //if (num_rossi > 0 && num == 0)
        //{
        //    Episode_Debug.Fail += 1;
        //    AddReward(-0.3f);
        //    EndEpisode();
        //}

        if (num < num_rossi)
        {
            Episode_Debug.Fail += 1;
            AddReward(-1.0f);
            EndEpisode();
        }


        Episode_Debug.ScreenText();
    }

    public void Shoot()
    {
        if (num > 0)
        {
            num -= 1;
            num_sfere.text = "#SFERE: " + num;
            GameObject clone = GameObject.Instantiate(sfera);
            clone.transform.position = transform.position - transform.up * 1.0f;
            clone.GetComponent<Rigidbody2D>().AddForce(-transform.up * 15.0f, ForceMode2D.Impulse);
        }
        
    }

    public int GetCountRossi() //Numero Rossi Text
    {
        return num_rossi;
    }

    public void SetCountRossi(int x) //Numero Rossi Text
    {
        
        num_rossi = x;
    }

    //public void Shoot_Rosso() //check rosso colpito
    //{
    //    AddReward(0.5f);
    //}

    public void Shoot_Error()
    {
        AddReward(-0.5f);
    }

    public void SearchRossiInParent(Transform parent) //Numero rossi iniziali nel livello e numero tiri
    {
        num_rossi = 0;
        foreach(Transform red in parent)
        {
            if (red.gameObject.tag == "red")
                num_rossi++;
        }
        num = num_rossi + 1;
    }

    public void Active(Transform parent)
    {
        foreach (Transform sphere in parent)
        {
            if (sphere.gameObject.tag == "red" || sphere.gameObject.tag == "blu")
                sphere.gameObject.SetActive(true);
        }
    }

This is the code of the agent.

This is the .yaml config

behaviors: 
 Capsule_ML:
    trainer_type: ppo
    hyperparameters:
      batch_size: 1024
      buffer_size: 10240
      learning_rate: 0.0004
      beta: 0.005
      epsilon: 0.2
      lambd: 0.95
      num_epoch: 5
      learning_rate_schedule: linear
    network_settings:
      normalize: false
      hidden_units: 128
      num_layers: 2
      vis_encode_type: simple
      memory:
        use_recurrent: true
        sequence_length: 64
        memory_size: 256
      goal_conditioning_type: hyper
      deterministic: false
    reward_signals:
      extrinsic:
        gamma: 0.99
        strength: 1.0
    init_path: null
    keep_checkpoints: 5
    checkpoint_interval: 500000
    max_steps: 500000
    time_horizon: 64
    summary_freq: 50000
    threaded: false
    self_play: null
    behavioral_cloning: null
env_settings:
  env_path: null
  env_args: null
  base_port: 5005
  num_envs: 1
  seed: -1
engine_settings:
  width: 84
  height: 84
  quality_level: 5
  time_scale: 20
  target_frame_rate: -1
  capture_frame_rate: 60
  no_graphics: false
environment_parameters: null
checkpoint_settings:
  run_id: TL1.1
  initialize_from: null
  load_model: false
  resume: false
  force: false
  train_model: false
  inference: false
debug: false

I'm using the RayPerception with these settings: Settings

Tried to use a timer to end the episodes that gets stuck, but the agent is always in loop, even after many and many steps