How to find PyGame Window Coordinates of an OpenGL Vertice?

Question

I am trying to figure out the coordinates of the vertices of two rectangles in a pygame window that is using OpenGL to create the 3D objects.

import pygame
from pygame.locals import *
import random
from OpenGL.GL import *
from OpenGL.GLU import *

rect1 = [(-5.125,0,-40),(-3.125,0,-40),(-3.125,5,-40),(-5.125,5,-40),]
rect2 = [(3.125,0,-40),(5.125,0,-40),(5.125,5,-40),(3.125,5,-40)]
edges = ((0,1),(1,2),(2,3),(3,0))

#This draws the rectangles edges
def Target():
    glBegin(GL_LINES)
    for edge in edges:
        for vertex in edge:
            glVertex3fv(rect1[vertex])
    glEnd()
    glBegin(GL_LINES)
    for edge in edges:
        for vertex in edge:
            glVertex3fv(rect2[vertex])
    glEnd()

def main():
    try:
        pygame.init()
        display = (320,240)
        pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
        gluPerspective(45, (display[0]/display[1]), .1, 1000)

        while True:
            #iterates through events to check for quits
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    quit()        
            Target()
            pygame.display.flip()
            pygame.time.wait(100)
            glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)         
    except Exception as e:
        print (e)
main()

How do I grab the coordinates on the pygame window(320,240) of the object?

Answer 1

The projection matrix describes the mapping from 3D points of a scene, to 2D points of the viewport. It transforms from eye space to the clip space, and the coordinates in the clip space are transformed to the normalized device coordinates (NDC) by dividing with the w component of the clip coordinates. The NDC are in range (-1,-1,-1) to (1,1,1).

At Perspective Projection the projection matrix describes the mapping from 3D points in the world as they are seen from of a pinhole camera, to 2D points of the viewport.
The eye space coordinates in the camera frustum (a truncated pyramid) are mapped to a cube (the normalized device coordinates).

Perspective Projection Matrix:

r = right, l = left, b = bottom, t = top, n = near, f = far

2*n/(r-l)      0              0                0
0              2*n/(t-b)      0                0
(r+l)/(r-l)    (t+b)/(t-b)    -(f+n)/(f-n)    -1    
0              0              -2*f*n/(f-n)     0

where :

aspect = w / h
tanFov = tan( fov_y / 2 );

2 * n / (r-l) = 1 / (tanFov * aspect)
2 * n / (t-b) = 1 / tanFov

Since the projection matrix is defined by the field of view and the aspect ratio it is possible to recover the viewport position with the field of view and the aspect ratio. Provided that it is a symmetrical perspective projection, where the field of view is not dispalced (as in your case).

First you have to transform the mose position to normalized device coordianates:

w = with of the viewport
h = height of the viewport
x = X position of the mouse
y = Y position ot the mouse

ndc_x = 2.0 * x/w - 1.0;
ndc_y = 1.0 - 2.0 * y/h; // invert Y axis

Then you have to converte the normalized device coordinates to view coordinates:

z = z coodinate of the geometry in view space

viewPos.x = -z * ndc_x * aspect * tanFov;
viewPos.y = -z * ndc_y * tanFov;  

If you want to check if the mouse hovers over your rectangles, then the code may look like this:

mpos = pygame.mouse.get_pos()
z = 40

ndc = [ 2.0 * mpos[0]/width - 1.0, 1.0 - 2.0 * mpos[1]/height ]
tanFov = math.tan( fov_y * 0.5 * math.pi / 180 )
aspect = width / height 
viewPos = [z * ndc[0] * aspect * tanFov, z * ndc[1] * tanFov ]

onRect1 = 1 if (viewPos[0]>=rect1[0][0] and viewPos[0]<=rect1[1][0] and viewPos[1]>=rect1[0][1] and viewPos[1]<=rect1[2][1] ) else 0
onRect2 = 1 if (viewPos[0]>=rect2[0][0] and viewPos[0]<=rect2[1][0] and viewPos[1]>=rect2[0][1] and viewPos[1]<=rect2[2][1] ) else 0

See further:

• How to recover view space position given view space depth value and ndc xy
• Is it possble get which surface of cube will be click in OpenGL?
• OpenGL - Mouse coordinates to Space coordinates

In the following I added the algorithm to your example. If the mouse hovers over an rectangle, then the rectangle is colored in red.

import pygame
from pygame.locals import *
import random
from OpenGL.GL import *
from OpenGL.GLU import *
import math

rect1 = [(-5.125,0,-40),(-3.125,0,-40),(-3.125,5,-40),(-5.125,5,-40),]
rect2 = [(3.125,0,-40),(5.125,0,-40),(5.125,5,-40),(3.125,5,-40)]
edges = ((0,1),(1,2),(2,3),(3,0))

fov_y = 45
width = 320
height = 200

#This draws the rectangles edges
def Target():
    mpos = pygame.mouse.get_pos()
    z = 40

    ndc = [ 2.0 * mpos[0]/width - 1.0, 1.0 - 2.0 * mpos[1]/height ]
    tanFov = math.tan( fov_y * 0.5 * math.pi / 180 )
    aspect = width / height 
    viewPos = [z * ndc[0] * aspect * tanFov, z * ndc[1] * tanFov ]

    onRect1 = 1 if (viewPos[0]>=rect1[0][0] and viewPos[0]<=rect1[1][0] and viewPos[1]>=rect1[0][1] and viewPos[1]<=rect1[2][1] ) else 0
    onRect2 = 1 if (viewPos[0]>=rect2[0][0] and viewPos[0]<=rect2[1][0] and viewPos[1]>=rect2[0][1] and viewPos[1]<=rect2[2][1] ) else 0

    glColor3f( 1, 1-onRect1, 1-onRect1 )
    glBegin(GL_LINES)
    for edge in edges:
        for vertex in edge:
            glVertex3fv(rect1[vertex])
    glEnd()

    glColor3f( 1, 1-onRect2, 1-onRect2 )
    glBegin(GL_LINES)
    for edge in edges:
        for vertex in edge:
            glVertex3fv(rect2[vertex])
    glEnd()

def main():
    try:
        pygame.init()
        display = (width,height)
        pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
        glMatrixMode(GL_PROJECTION)
        gluPerspective(fov_y, (display[0]/display[1]), .1, 1000)
        glMatrixMode(GL_MODELVIEW)

        while True:
            #iterates through events to check for quits
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    quit()        
            Target()
            pygame.display.flip()
            pygame.time.wait(100)
            glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)         
    except Exception as e:
        print (e)
main()

Extension to the answer

Of course you can also do it the other way around. You can transform the corner points of the rectangle to normalized device coordinates and compare them to the mouse position, in normalized device coordinates.

For this you have to read the projection matrix by glGetFloatv(GL_PROJECTION_MATRIX):

prjMat = (GLfloat * 16)()
glGetFloatv(GL_PROJECTION_MATRIX, prjMat)

And you need a function which transform a 3 dimensional cartesian vector by a projection matrix. This is done by multiplying the vector by the projection matrix, which gives homogeneous clip space coordinates. The normalized device coordinates are calculated by dividing the x, y, and z component by the w component.

def TransformVec3(vecA,mat44):
    vecB = [0, 0, 0, 0]
    for i0 in range(0, 4):
        vecB[i0] = vecA[0] * mat44[0*4+i0] + vecA[1] * mat44[1*4+i0] + vecA[2] * mat44[2*4+i0] + mat44[3*4+i0]
    return [vecB[0]/vecB[3], vecB[1]/vecB[3], vecB[2]/vecB[3]]

The following function tests if the mouse position is in an rectangle defined by a lower left and a upper right point (the corner points have to be in view space coordinates):

def TestRec(prjMat, mpos, ll, tr):
    ll_ndc = TransformVec3(ll, prjMat)
    tr_ndc = TransformVec3(tr, prjMat)
    ndc = [ 2.0 * mpos[0]/width - 1.0, 1.0 - 2.0 * mpos[1]/height ]
    inRect = 1 if (ndc[0]>=ll_ndc[0] and ndc[0]<=tr_ndc[0] and ndc[1]>=ll_ndc[1] and ndc[1]<=tr_ndc[1] ) else 0
    return inRect

Again I added the algorithm to your example. If the mouse hovers over an rectangle, then the rectangle is colored in red.

import pygame
from pygame.locals import *
import random
from OpenGL.GL import *
from OpenGL.GLU import *
import math

rect1 = [(-5.125,0,-40),(-3.125,0,-40),(-3.125,5,-40),(-5.125,5,-40),]
rect2 = [(3.125,0,-40),(5.125,0,-40),(5.125,5,-40),(3.125,5,-40)]
edges = ((0,1),(1,2),(2,3),(3,0))

fov_y = 45
width = 320
height = 200

def TransformVec3(vecA,mat44):
    vecB = [0, 0, 0, 0]
    for i0 in range(0, 4):
        vecB[i0] = vecA[0] * mat44[0*4+i0] + vecA[1] * mat44[1*4+i0] + vecA[2] * mat44[2*4+i0] + mat44[3*4+i0]
    return [vecB[0]/vecB[3], vecB[1]/vecB[3], vecB[2]/vecB[3]]

def TestRec(prjMat, mpos, ll, tr):
    ll_ndc = TransformVec3(ll, prjMat)
    tr_ndc = TransformVec3(tr, prjMat)
    ndc = [ 2.0 * mpos[0]/width - 1.0, 1.0 - 2.0 * mpos[1]/height ]
    inRect = 1 if (ndc[0]>=ll_ndc[0] and ndc[0]<=tr_ndc[0] and ndc[1]>=ll_ndc[1] and ndc[1]<=tr_ndc[1] ) else 0
    return inRect


#This draws the rectangles edges
def Target():

    prjMat = (GLfloat * 16)()
    glGetFloatv(GL_PROJECTION_MATRIX, prjMat)

    mpos = pygame.mouse.get_pos()
    onRect1 = TestRec(prjMat, mpos, rect1[0], rect1[2])
    onRect2 = TestRec(prjMat, mpos, rect2[0], rect2[2])

    glColor3f( 1, 1-onRect1, 1-onRect1 )
    glBegin(GL_LINES)
    for edge in edges:
        for vertex in edge:
            glVertex3fv(rect1[vertex])
    glEnd()

    glColor3f( 1, 1-onRect2, 1-onRect2 )
    glBegin(GL_LINES)
    for edge in edges:
        for vertex in edge:
            glVertex3fv(rect2[vertex])
    glEnd()

def main():
    try:
        pygame.init()
        display = (width,height)
        pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
        glMatrixMode(GL_PROJECTION)
        gluPerspective(fov_y, (display[0]/display[1]), .1, 1000)
        glMatrixMode(GL_MODELVIEW)

        while True:
            #iterates through events to check for quits
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    quit()        
            Target()
            pygame.display.flip()
            pygame.time.wait(100)
            glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)         
    except Exception as e:
        print (e)
main()