glReadPixels returning incorrect values

Question

I am trying to implement picking via following an opengl tutorial, I have a mesh with 6000 vertices and I wish to pick particular ones; I have chosen to do by redering uniquely coloured boxes at each vertex point, read the pixel on my mouse click at that point and that should return to me the ID of the closest vertex. The background is rendered as white so if I miss I get nothing.

However I have a problem, it only works most of the time; there are certain areas that if I click there I get white returned even though its clearly a vertex, and when rendering the colorized scene clearly has a redish box at the point I clicked.

Then there are white areas near the mesh, to the bottom left and at some random point away from it, returns me a hit.

I do not understand at all why this is happening, it should work.

void Display() {
Controls->setVector(indexed_vertices);


if (Controls->getPicking()) {
    // Clear the screen in white
    glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    for ( int i=0; i< indexed_vertices.size(); i++) {
        // use shaders
        glUseProgram(pickingProgramID);
        // Get a handle for our "MVP" uniform
        GLuint PickingMatrixID = glGetUniformLocation(pickingProgramID, "MVP");

        glm::mat4 RotationMatrix = glm::toMat4(orientations);
        glm::mat4 btTranslationMatrix = glm::translate(glm::mat4(1.0f), indexed_vertices[i]);
        glm::mat4 myModelMatrix = ModelMatrix * Controls->getTranslationMatrix() * Controls->getRotationMatrix() * btTranslationMatrix;
        MVP = ProjectionMatrix * ViewMatrix * myModelMatrix;
        Controls->setCntrlsViewMatrix(ViewMatrix);
        Controls->setCntrlsProjectionMatrix(ProjectionMatrix);

        glUniformMatrix4fv(PickingMatrixID, 1, GL_FALSE, &MVP[0][0]);


        // Convert "i", the integer mesh ID, into an RGB color
        int r = (i & 0x000000FF) >>  0;
        int g = (i & 0x0000FF00) >>  8;
        int b = (i & 0x00FF0000) >> 16;

        // OpenGL expects colors to be in [0,1], so divide by 255.
        glUniform4f(pickingColorID, r/255.0f, g/255.0f, b/255.0f, 1.0f);

        // 1rst attribute buffer : vertices
        glEnableVertexAttribArray(0);
        glBindBuffer(GL_ARRAY_BUFFER, gvertexbuffer);
        glVertexAttribPointer(
            0,                  // attribute. No particular reason for 0, but must match the layout in the shader.
            3,                  // size
            GL_FLOAT,           // type
            GL_FALSE,           // normalized?
            0,                  // stride
            (void*)0            // array buffer offset
        );

        // Index buffer
        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, elementbuffer);

        // Draw the triangles !
        glDrawElements(
            GL_TRIANGLES,      // mode
            indices.size(),    // count
            GL_UNSIGNED_SHORT,   // type
            (void*)0           // element array buffer offset
        );
        // OpenGL expects colors to be in [0,1], so divide by 255.

    }
    glDisableVertexAttribArray(0);

    glFlush();
    glFinish(); 


    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
    // Read the pixel at the center of the screen.
    // You can also use glfwGetMousePos().
    // Ultra-mega-over slow too, even for 1 pixel, 
    // because the framebuffer is on the GPU.
    unsigned char data[4];
    glReadPixels(Controls->get_mx_cur(), Controls->get_my_cur(),1,1, GL_RGBA, GL_UNSIGNED_BYTE, data);
    std::cout << "MX: " << Controls->get_mx_cur() << " MY: " << Controls->get_my_cur() << std::endl;

    // Convert the color back to an integer ID
    int pickedID = 
        data[0] + 
        data[1] * 256 +
        data[2] * 256*256;

    //std::cout << std::hex << pickedID << std::dec<<std::endl;
    if (pickedID == 0x00ffffff) { // Full white, must be the background !
        printf("Miss\n");
    } 
    else {
        std::cout << "mesh " << pickedID << std::endl;
    }

    // Uncomment these lines to see the picking shader in effect
    glutSwapBuffers();
    skip = true;
    Controls->setPicking(false);
}
if (!skip) {
// White background
glClearColor(0.2f, 0.25f, 0.5f, 0.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glEnable(GL_DEPTH_TEST);
//glEnable(GL_CULL_FACE);

glUseProgram(ShaderIDs[0]);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, Texture);
glUniform1i(TextureID, 0);


glm::mat4 myModelMatrix = ModelMatrix * Controls->getTranslationMatrix() * Controls->getRotationMatrix();
MVP = ProjectionMatrix * ViewMatrix * myModelMatrix;

// The inverse transpose of the View Model Matrix will re-normalize the normals if there's
// been any scaling. Otherwise you don't need it.
glm::mat3 NormalMatrix = glm::mat3( glm::transpose(glm::inverse(ViewMatrix * myModelMatrix)));

Controls->setCntrlsViewMatrix(ViewMatrix);
Controls->setCntrlsProjectionMatrix(ProjectionMatrix);

glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP[0][0]);
glUniformMatrix4fv(ViewMatrixID, 1, GL_FALSE, &ViewMatrix[0][0]);
glUniformMatrix4fv(ModelMatrixID, 1, GL_FALSE, &myModelMatrix[0][0]);
    // Notice we're passing a 3 by 3 matrix here.
glUniformMatrix3fv(NormalMatrixID, 1, GL_FALSE, &NormalMatrix[0][0]);
glUniform3f(CameraID, cameraLoc.x, cameraLoc.y, cameraLoc.z);
glUniform3f(LightPosID, lightPosition.x, lightPosition.y, lightPosition.z);

// VBO buffer: vertices
// 1rst attribute buffer : vertices
glEnableVertexAttribArray(0);
glBindBuffer(GL_ARRAY_BUFFER, vertexbuffer);
glVertexAttribPointer(
    0,                  // attribute
    3,                  // size
    GL_FLOAT,           // type
    GL_FALSE,           // normalized?
    0,                  // stride
    (void*)0            // array buffer offset
    );

// 2nd attribute buffer : UVs
glEnableVertexAttribArray(1);
glBindBuffer(GL_ARRAY_BUFFER, uvbuffer);
glVertexAttribPointer(
    1,                                // attribute
    2,                                // size
    GL_FLOAT,                         // type
    GL_FALSE,                         // normalized?
    0,                                // stride
    (void*)0                          // array buffer offset
    );

// 2rd attribute buffer : normals
glEnableVertexAttribArray(2);
glBindBuffer(GL_ARRAY_BUFFER, normalbuffer);
glVertexAttribPointer(
    2,                                // attribute
    3,                                // size
    GL_FLOAT,                         // type
    GL_FALSE,                         // normalized?
    0,                                // stride
    (void*)0                          // array buffer offset
    );

// Index buffer
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, elementbuffer);

if ( Controls->getRenderingMode() == 0 ) {
    glDisable(GL_CULL_FACE);
    glEnable(GL_DEPTH_TEST);
    glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
    //glDisable(GL_POLYGON_OFFSET_FILL);
} 
else if (Controls->getRenderingMode() == 1 ) {

    glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
    glDisable(GL_CULL_FACE);
    //glDisable(GL_POLYGON_OFFSET_FILL);
    glUseProgram(ShaderIDs[1]);
    glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP[0][0]);
    glUniformMatrix4fv(ViewMatrixID, 1, GL_FALSE, &ViewMatrix[0][0]);
    glUniformMatrix4fv(ModelMatrixID, 1, GL_FALSE, &myModelMatrix[0][0]);
        // Notice we're passing a 3 by 3 matrix here.
    glUniformMatrix3fv(NormalMatrixID, 1, GL_FALSE, &NormalMatrix[0][0]);
    glUniform3f(CameraID, cameraLoc.x, cameraLoc.y, cameraLoc.z);
    glUniform3f(LightPosID, lightPosition.x, lightPosition.y, lightPosition.z);
} 
else if (Controls->getRenderingMode() == 2 ) {
    glUseProgram(ShaderIDs[1]);
    // 
    glm::mat4 MyOffsetMatrix = glm::scale(glm::mat4(1.0f), glm::vec3(1.025,1.025,1.025));
    MyOffsetMatrix = glm::mat4(1.0f);
    glm::mat4 myModelMatrix2 = ModelMatrix * Controls->getTranslationMatrix() * 
        Controls->getRotationMatrix()*MyOffsetMatrix;
    glm::mat3 NormalMatrix2 = glm::mat3( glm::transpose(glm::inverse(ViewMatrix * 
        myModelMatrix2)));

    glm::mat4 MVP2 = ProjectionMatrix * ViewMatrix * myModelMatrix2;

    glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP2[0][0]);
    glUniformMatrix4fv(ViewMatrixID, 1, GL_FALSE, &ViewMatrix[0][0]);
    glUniformMatrix4fv(ModelMatrixID, 1, GL_FALSE, &myModelMatrix2[0][0]);
        // Notice we're passing a 3 by 3 matrix here.
    glUniformMatrix3fv(NormalMatrixID, 1, GL_FALSE, &NormalMatrix2[0][0]);
    glUniform3f(CameraID, cameraLoc.x, cameraLoc.y, cameraLoc.z);
    glUniform3f(LightPosID, lightPosition.x, lightPosition.y, lightPosition.z);


    // The rest is exactly the same as the first object

    // 1rst attribute buffer : vertices
    glEnableVertexAttribArray(0);
    glBindBuffer(GL_ARRAY_BUFFER, vertexbuffer);
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, (void*)0);

    // 2nd attribute buffer : UVs
    glEnableVertexAttribArray(1);
    glBindBuffer(GL_ARRAY_BUFFER, uvbuffer);
    glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, (void*)0);

    // 3rd attribute buffer : normals
    glEnableVertexAttribArray(2);
    glBindBuffer(GL_ARRAY_BUFFER, normalbuffer);
    glVertexAttribPointer(2, 3, GL_FLOAT, GL_FALSE, 0, (void*)0);

    // Index buffer
    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, elementbuffer);

    glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
    glDisable(GL_CULL_FACE);
    glEnable(GL_POLYGON_OFFSET_FILL);
    // Draw the triangles !
    glDrawElements(GL_TRIANGLES, indices.size(), GL_UNSIGNED_SHORT, (void*)0);

    glEnable(GL_POLYGON_OFFSET_FILL);
    glDisable(GL_CULL_FACE);
    glEnable(GL_DEPTH_TEST);
    glPolygonOffset(2.0f, 2.0f);
    glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
    //glDisable(GL_POLYGON_OFFSET_FILL);

    glUseProgram(ShaderIDs[0]);
}
//glUseProgram(ShaderIDs[1]);
// Draw the triangles !
glDrawElements(
    GL_TRIANGLES,      // mode
    indices.size(),    // count
    GL_UNSIGNED_SHORT,   // type
    (void*)0           // element array buffer offset
    );

glDisableVertexAttribArray(0);
glDisableVertexAttribArray(1);
glDisableVertexAttribArray(2);
glutSwapBuffers();
}

}

Answer 1

The problem was two-fold: Crazy numbers returned is a result of improper multisampling enabled when I wanted specific colour values (how to have my cake and eat it might need a bit of work but right now I don't care), and secondly because glReadPixels() inverts the Y axis and needed to do Height - Current_Mouse_Position for the Y value.

Perhaps glPoints would be faster means of doing what I'm doing, I'll need to look into it.