Converting mouse coordinates to OpenGL is erroneously offset

Question

I am trying to achieve picking of a vertex on the surface of sphere using the mouse. I am using freeglut and OpenGL 4.5.

Rendering

Display Function:

void display(void) {

    // Clear display port
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    glLoadIdentity();

    // Reset camera
    gluLookAt(
        0.0, 0.0, 5.0,
        0.0, 0.0, 0.0,
        0.0, 1.0, 0.0
    );

    // Rotate
    glRotatef(-theta, 1.0, 0.0, 0.0);
    glRotatef(phi, 0.0, 0.0, 1.0);

    // Zoom
    glScalef(zoom, zoom, zoom);

    // Render sphere
    glPushMatrix();
    glTranslatef(0, 0, 0);
    glColor3f(1, 0, 0);
    glutWireSphere(1, 32, 32);
    glPopMatrix();

    .
    .
    .

    // Render selection point
    glPushMatrix();
    pointIsOnSphere ? glColor3f(0, 1, 0) : glColor3f(0, 0, 1);
    pointIsOnSphere ? glPointSize(5.0f) : glPointSize(2.5f);
    glBegin(GL_POINTS);
    glVertex3f(clickPosition.x, clickPosition.y, clickPosition.z);
    glEnd();
    glPopMatrix();

    // Swap buffers
    glutSwapBuffers();
}

Reshape Function:

void reshape(GLsizei width, GLsizei height) {
    if (height == 0) {
        height = 1;
    }
    float ratio = 1.0 * width / height;
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    glViewport(0, 0, width, height);
    gluPerspective(45, ratio, 0.5, 5);
    glMatrixMode(GL_MODELVIEW);
}

Mouse Handling

// Handles mouse click for point selection.
void mouse(int button, int state, int x, int y) {

    mouse_x = x;
    mouse_y = y;

    if (!pointIsOnSphere)
        return;

    if (button == GLUT_LEFT_BUTTON && state == GLUT_UP)
        controlPoints.push_back(clickPosition);

    if (button == GLUT_RIGHT_BUTTON && state == GLUT_DOWN)
        RIGHT_BUTTON_DOWN = true;

    glutPostRedisplay();
}

// Handles mouse movement for rotation.
void motion(int x, int y) {
    if (RIGHT_BUTTON_DOWN) {
        phi += (float)(x - mouse_x) / 4.0; 
        theta += (float)(mouse_y - y) / 4.0;
    }

    mouse_x = x;
    mouse_y = y;
    glutPostRedisplay();
}

    // Handles mouse movement for point selection.
void passiveMotion(int x, int y) {
    // Get position of click.
    clickPosition = GetOGLPos(x, y);
    // Set click position's z position to camera's z position.
    clickPosition.z = 1;
    // Create directional vector pointing into the screen.
    glm::vec3 into_screen = glm::vec3(0, 0, -1);
    // Create ray.
    ray r = ray(
        clickPosition,
        into_screen
    );

    mousePosition = clickPosition;
    float t = hit_sphere(glm::vec3(0), 1, r);
    if (t != -1) {
        pointIsOnSphere = true;
        clickPosition += t * into_screen;
    }
    else {
        pointIsOnSphere = false;
    }

    glutPostRedisplay();
}

// Handles scroll input for zoom.
void mouseWheel(int button, int state, int x, int y) {
    if (state == 1) {
        zoom = zoom + zoom_sensitivity >= zoom_max ? zoom_max : zoom + zoom_sensitivity;
    }
    else if (state == -1) {
        zoom = zoom - zoom_sensitivity <= zoom_min ? zoom_min : zoom - zoom_sensitivity;
    }
    glutPostRedisplay();
}

3D Picking

// Get position of click in 3-d space
glm::vec3 GetOGLPos(int x, int y)
{
    GLint viewport[4];
    GLdouble modelview[16];
    GLdouble projection[16];
    GLfloat winX, winY, winZ;
    GLdouble posX, posY, posZ;

    glGetDoublev(GL_MODELVIEW_MATRIX, modelview);
    glGetDoublev(GL_PROJECTION_MATRIX, projection);
    glGetIntegerv(GL_VIEWPORT, viewport);

    winX = (float)x;
    winY = (float)viewport[3] - (float)y;
    glReadPixels(x, int(winY), 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &winZ);

    gluUnProject(winX, winY, winZ, modelview, projection, viewport, &posX, &posY, &posZ);
    return glm::vec3(posX, posY, posZ);
}

float hit_sphere(const glm::vec3& center, float radius, const ray& r) {
    glm::vec3 oc = r.origin() - center;
    float a = dot(r.direction(), r.direction());
    float b = 2.0 * glm::dot(oc, r.direction());
    float c = glm::dot(oc, oc) - radius * radius;
    float discriminant = b * b - 4 * a*c;

    if (discriminant < 0.0) {
        return -1.0;
    }
    else {
        float numerator = -b - sqrt(discriminant);
        if (numerator > 0.0) {
            return numerator / (2.0 * a);
        }

        numerator = -b + sqrt(discriminant);
        if (numerator > 0.0) {
            return numerator / (2.0 * a);
        }
        else {
            return -1;
        }
    }
}

Visual Result

As you can see, the actual point of selection is offset from the cursor's position.

Here's a video showing the error:

https://gfycat.com/graciousantiquechafer

Clearly, there's something wrong with the z-value and I can see that the error gets bigger as you go away from the center axis. But I can't seem to figure out where in the code the exact problem lies.

New Error

Now, the points appear away from the surface of the sphere.

Answer 1

At Perspective Projection the projection matrix describes the mapping from 3D points in the world as they are seen from of a pinhole camera, to 2D points of the viewport.
The viewing volume is a Frustum (a truncated pyramid), where the top of the pyramid is the viewer's position.
If you start a ray from the camera position, then all the points on the ray have the same xy window coordinate (and xy normalized device coordinate), the points have just a different "depth" (z coordinate). The projection of the view ray onto the viewport is a point.

This means your assumption is wrong, the direction "into the screen" is not (0, 0, -1):

// Create directional vector pointing into the screen.
glm::vec3 into_screen = glm::vec3(0, 0, -1);

Note, that would be correct for Orthographic Projection, but it is wrong for Perspective Projection.

The direction depends on the window coordinate. Luckily it does not depend on the depth and the window coordinate is givens by the mouse position.
To find the ray which "hits" the current mouse position you've to compute 2 points on the ray.
Find the intersection of the ray with the near plane (depth = 0.0) and the far plane (depth). Since all = 1.0)

This means the window coordinates of 2 points on a ray, that starts at the camera position and goes through the mouse cursor are:

point 1: (mouseX, height-mouseY, 0.0)
point 2: (mouseX, height-mouseY, 1.0)

Adapt GetOGLPos:

glm::vec3 GetOGLPos(int x, int y, float depth)
{
    GLint viewport[4];
    GLdouble modelview[16];
    GLdouble projection[16];
    GLfloat winX, winY, winZ;
    GLdouble posX, posY, posZ;

    glGetDoublev(GL_MODELVIEW_MATRIX, modelview);
    glGetDoublev(GL_PROJECTION_MATRIX, projection);
    glGetIntegerv(GL_VIEWPORT, viewport);

    winX = (float)x;
    winY = (float)viewport[3] - (float)y;
    winZ = depth;
    //glReadPixels(x, int(winY), 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &winZ);

    gluUnProject(winX, winY, winZ, modelview, projection, viewport, &posX, &posY, &posZ);

    return glm::vec3(posX, posY, posZ);
}

Define the ray

void passiveMotion(int x, int y)
{
    glm::vec3 clickPositionNear = GetOGLPos(x, y, 0.0);
    glm::vec3 clickPositionFar = GetOGLPos(x, y, 1.0);

    glm::vec3 into_screen = glm::normalize(clickPositionFar - clickPositionNear);

    ray r = ray(
        clickPositionNear,
        into_screen
    );

    // [...]
}