Unexpected rotation angle from 4x4 transformation matrix (libgdx)

Question

This is the gist of it, you can see the rotation angle go from 0 - 236 then jump 119 - 0, weird.

I would like to get the angle of a model in degrees (0-360) in relation to its y-axis. However using libgdx thus far I have been unable to get an expected result using:

angle = modelInstance.transform.getRotation(new Quaternion()).getAxisAngle(Vector3.Zero)

Assuming I am only rotating around the y-axis, is this the correct way to get the angle of rotation from the transformation matrix in libgdx?

I have included a full example class to demonstrate that the angles I am receiving from getting rotations in such a way are not what is expected, instead of 0 - 360 for a full revolution I get something like 0 - 117 then jumps to 243.

public class RotateExperiment extends InputAdapter implements ApplicationListener {

    private Environment environment;
    private ModelBatch modelBatch;
    private Camera camera;
    private ModelInstance player;

    @Override
    public void create() {
        this.player = new ModelInstance(new ModelBuilder().createCone(10, 10,10, 10, new Material(ColorAttribute.createDiffuse(Color.GRAY)), VertexAttributes.Usage.Position | VertexAttributes.Usage.Normal));

        int w = Gdx.graphics.getWidth();
        int h = Gdx.graphics.getHeight();

        modelBatch = new ModelBatch();
        camera = new PerspectiveCamera(67, w, h);
        camera.position.set(10f, 10f, 10f);
        camera.lookAt(0,0,0);
        camera.near = 0.1f;
        camera.far = 300f;
        camera.update();

        environment = new Environment();
        environment.set(new ColorAttribute(ColorAttribute.AmbientLight, 0.4f, 0.4f, 0.4f, 1f));
        environment.add(new DirectionalLight().set(0.8f, 0.8f, 0.8f, -1f, -0.8f, -0.2f));
    }

    @Override
    public void dispose() {}

    private void updateModel(){
        Vector3 directionVector = new Vector3(0,0,0);
        if(Gdx.input.isKeyPressed(Input.Keys.S)){
            directionVector.z = -1;
        }
        if(Gdx.input.isKeyPressed(Input.Keys.W)){
            directionVector.z = 1;
        }
        if(Gdx.input.isKeyPressed(Input.Keys.A)){
            if(Gdx.input.isButtonPressed(Input.Buttons.RIGHT)){
                directionVector.y = 1;
            } else {
                player.transform.rotate(Vector3.Y, 90 * Gdx.graphics.getDeltaTime());
            }
        }
        if(Gdx.input.isKeyPressed(Input.Keys.D)){
            if(Gdx.input.isButtonPressed(Input.Buttons.RIGHT)){
                directionVector.y = -1;
            } else {
                player.transform.rotate(Vector3.Y, -90 * Gdx.graphics.getDeltaTime());
            }
        }
        if(directionVector.z != 0 || directionVector.y != 0){
            player.transform.translate(directionVector.nor().scl(20 * Gdx.graphics.getDeltaTime()));
        }
    }

    public void update() {
        if(player != null){
            updateModel();
            //update angle text -> actual test code
            outputText.setText(String.valueOf(player.transform.getRotation(new Quaternion()).getAxisAngle(Vector3.Zero)));
        }
    }

    @Override
    public void render() {
        update();
        Gdx.gl.glViewport(0, 0, Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
        Gdx.gl.glClear(GL10.GL_COLOR_BUFFER_BIT | GL10.GL_DEPTH_BUFFER_BIT);

        modelBatch.begin(camera);
        modelBatch.render(player, environment);
        modelBatch.end();
    }

    @Override
    public void resize(int width, int height) {
        Gdx.gl.glViewport(0, 0, width, height);
    }

    @Override
    public void pause() {}

    @Override
    public void resume() {}

    private final static JLabel outputText = new JLabel("test");

    public static void main(String[] args) {
        LwjglApplicationConfiguration cfg = new LwjglApplicationConfiguration();
        cfg.title = "TestRotation";
        cfg.useGL20 = true;
        Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize();
        cfg.width = screenSize.width;
        cfg.height = screenSize.height;

        new LwjglApplication(new RotateExperiment(), cfg);

        JFrame outputWindow = new JFrame();
        JPanel outputPanel = new JPanel();

        outputPanel.add(outputText);
        outputWindow.add(outputPanel);
        outputWindow.setAlwaysOnTop(true);
        outputWindow.setVisible(true);
        outputWindow.pack();
        outputWindow.toFront();
    }
}

If you are running the example, click on the libgdx render window once it has loaded, and model rotations can be controlled with "A" and "D". The resulting rotation using the above method are also output in a separate JFrame.

Edit: I believe the suspect code may be in the libgdx method for taking a 4x4 transformation matrix to a quaternion: Quaternion.setFromAxes. When you call transform.getRotation() this calls setFromAxes and passes in the matrix axes. I have seen the open issue, however as you can see in he example my cube model is not scaled, so this can't be what's happening. The setFromAxes code is as follows:

            // the trace is the sum of the diagonal elements; see
    // http://mathworld.wolfram.com/MatrixTrace.html
    final float m00 = xx, m01 = xy, m02 = xz;
    final float m10 = yx, m11 = yy, m12 = yz;
    final float m20 = zx, m21 = zy, m22 = zz;
    final float t = m00 + m11 + m22;

    // we protect the division by s by ensuring that s>=1
    double x, y, z, w;
    if (t >= 0) { // |w| >= .5
        double s = Math.sqrt(t + 1); // |s|>=1 ...
        w = 0.5 * s;
        s = 0.5 / s; // so this division isn't bad
        x = (m21 - m12) * s;
        y = (m02 - m20) * s;
        z = (m10 - m01) * s;
    } else if ((m00 > m11) && (m00 > m22)) {
        double s = Math.sqrt(1.0 + m00 - m11 - m22); // |s|>=1
        x = s * 0.5; // |x| >= .5
        s = 0.5 / s;
        y = (m10 + m01) * s;
        z = (m02 + m20) * s;
        w = (m21 - m12) * s;
    } else if (m11 > m22) {
        double s = Math.sqrt(1.0 + m11 - m00 - m22); // |s|>=1
        y = s * 0.5; // |y| >= .5
        s = 0.5 / s;
        x = (m10 + m01) * s;
        z = (m21 + m12) * s;
        w = (m02 - m20) * s;
    } else {
        double s = Math.sqrt(1.0 + m22 - m00 - m11); // |s|>=1
        z = s * 0.5; // |z| >= .5
        s = 0.5 / s;
        x = (m02 + m20) * s;
        y = (m21 + m12) * s;
        w = (m10 - m01) * s;
    }

    return set((float)x, (float)y, (float)z, (float)w);

Answer 1

So assuming you have a non scaled model, and you only performed a rotation around the y-axis, the following code will get you an angle 0 -360 only from the transformation matrix of your model in libgdx:

        Vector3 axisVec = new Vector3();
        int angle = (int) (player.transform.getRotation(new Quaternion()).getAxisAngle(axisVec) * axisVec.nor().y);
        angle = angle < 0 ? angle + 360 : angle; //convert <0 values

If you have a scaled model the current solution as per the open issue is to normalize the axes in the setFromAxes code, I have replaced my setFromAxes with the following until it is updated:

private Quaternion setFromAxes(float xx, float xy, float xz, float yx, float yy, float yz, float zx, float zy, float zz){

    //normalise axis
    Vector3 xAxis = new Vector3(xx, xy, xz).nor();
    Vector3 yAxis = new Vector3(yx, yy, yz).nor();
    Vector3 zAxis = new Vector3(zx, zy, zz).nor();

    xx = xAxis.x;
    xy = xAxis.y;
    xz = xAxis.z;

    yx = yAxis.x;
    yy = yAxis.y;
    yz = yAxis.z;

    zx = zAxis.x;
    zy = zAxis.y;
    zz = zAxis.z;

    // the trace is the sum of the diagonal elements; see
    // http://mathworld.wolfram.com/MatrixTrace.html
    final float m00 = xx, m01 = xy, m02 = xz;
    final float m10 = yx, m11 = yy, m12 = yz;
    final float m20 = zx, m21 = zy, m22 = zz;
    final float t = m00 + m11 + m22;

    // we protect the division by s by ensuring that s>=1
    double x, y, z, w;
    if (t >= 0) { // |w| >= .5
        double s = Math.sqrt(t + 1); // |s|>=1 ...
        w = 0.5 * s;
        s = 0.5 / s; // so this division isn't bad
        x = (m21 - m12) * s;
        y = (m02 - m20) * s;
        z = (m10 - m01) * s;
    } else if ((m00 > m11) && (m00 > m22)) {
        double s = Math.sqrt(1.0 + m00 - m11 - m22); // |s|>=1
        x = s * 0.5; // |x| >= .5
        s = 0.5 / s;
        y = (m10 + m01) * s;
        z = (m02 + m20) * s;
        w = (m21 - m12) * s;
    } else if (m11 > m22) {
        double s = Math.sqrt(1.0 + m11 - m00 - m22); // |s|>=1
        y = s * 0.5; // |y| >= .5
        s = 0.5 / s;
        x = (m10 + m01) * s;
        z = (m21 + m12) * s;
        w = (m02 - m20) * s;
    } else {
        double s = Math.sqrt(1.0 + m22 - m00 - m11); // |s|>=1
        z = s * 0.5; // |z| >= .5
        s = 0.5 / s;
        x = (m02 + m20) * s;
        y = (m21 + m12) * s;
        w = (m10 - m01) * s;
    }

    return new Quaternion((float)x, (float)y, (float)z, (float)w);

}