OpenGL (Python) - applying an STMap (UVmap) to an image?

Question

i have the following code that loads and displays (using PyQt) an image on the screen:

gVShader = """
              attribute vec4 position;
              attribute vec2 texture_coordinates;
              varying vec2 v_texture_coordinates;
              void main() {
                  v_texture_coordinates = texture_coordinates;
                  v_texture_coordinates.y = 1.0 - v_texture_coordinates.y;
                  gl_Position = position;
              }"""

gFShader = """
              uniform sampler2D texture1;
              uniform sampler2D texture2;
              varying vec2 v_texture_coordinates;

              void main() {
                  gl_FragColor = texture2D(texture1, texture2.rg);
              }"""


class ProjectiveGLViewer(QtOpenGL.QGLWidget):

    def __init__(self, parent=None):
        super(ProjectiveGLViewer, self).__init__(parent)

    def initializeGL(self):

        vshader = QtOpenGL.QGLShader(QtOpenGL.QGLShader.Vertex, self)
        if not vshader.compileSourceCode(gVShader):
            print vshader.log()

        fshader = QtOpenGL.QGLShader(QtOpenGL.QGLShader.Fragment, self)
        if not fshader.compileSourceCode(gFShader):
            print fshader.log()

        self._program = QtOpenGL.QGLShaderProgram()
        self._program.addShader(vshader)
        self._program.addShader(fshader)
        self._program.link()
        self._program.bind()

        # data array (2 [position], 4 [color])

        data = np.array([-1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0], dtype=np.float32)

        self.bufferID = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, self.bufferID)
        glBufferData(GL_ARRAY_BUFFER, data.nbytes, data, GL_DYNAMIC_DRAW)

        loc = self._program.attributeLocation("position")
        glEnableVertexAttribArray(loc)
        glVertexAttribPointer(loc, 2, GL_FLOAT, False, 16, ctypes.c_void_p(0))

        loc = self._program.attributeLocation("texture_coordinates")
        glEnableVertexAttribArray(loc)
        glVertexAttribPointer(loc, 2, GL_FLOAT, False, 16, ctypes.c_void_p(8))

        image = QtGui.QImage("image.jpg")
        ptr = image.bits()
        ptr.setsize(image.byteCount())
        image_data = np.asarray(ptr).reshape(image.width(), image.height(), 4)

        self._imageTextureID = glGenTextures(1)

        glBindTexture(GL_TEXTURE_2D,  self._imageTextureID)
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 0, GL_BGRA, GL_UNSIGNED_BYTE, image_data)

        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)

        glActiveTexture(GL_TEXTURE0)
        self._program.setUniformValue('texture1', 0)


    image2 = QtGui.QImage("st_map.tif")
    ptr2 = image2.bits()
    ptr2.setsize(image2.byteCount())
    image_data2 = np.asarray(ptr).reshape(image2.width(), image2.height(), 4)

    self._stTextureID = glGenTextures(1)

    glBindTexture(GL_TEXTURE_2D,  self._stTextureID)
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image2.width(), image2.height(), 0, GL_BGRA, GL_UNSIGNED_BYTE, image_data2)

    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)

    self._program.setUniformValue('texture2', 1)

    def paintGL(self):
        glBindBuffer(GL_ARRAY_BUFFER, self.bufferID)
        glClearColor(0, 0.2, 0.3, 1.0)
        glClearDepth(1.0)
        glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)
        glDrawArrays(GL_TRIANGLE_STRIP, 0, 4)

    def resizeGL(self, w, h):
        glViewport(0, 0, w, h)

Now, i want to use an STMap (UVmap) to remap the pixels of that image. For those of you who aren't familiar with it, an STMap looks like this: link But could have many shapes. It's the same size as my original image, and contains only Red and Green channels (Blue is empty). The Red represents the X axis and the Green represents the Y. So for example, to get the new coordinates of the pixel that's at position [100,100] in my original image, i would go to that same position in the STMap and look for that pixel's Red channel value (that would be the new X value) and Green value (new Y value).

So how can i do that? I'm not too sure.. I tried adding the STMap texture but it wouldn't let me use it's Red and Green channel values (see fragment shader).Thanks!

EDIT:

I've edited my code with something that i tried and didn't work..

Answer 1

Since I do not have all resources I cannot run you code here.

From the shader point of view, you can modify texture1 coordinates by using texture2 coordinates. There are many ways to do that, for example you could just translate them by adding UVMap values to it. Here, texture2 contains UVMap data:

 uniform sampler2D texture1;
 uniform sampler2D texture2;
 varying vec2 v_texture_coordinates;

 vec2 my_new_coords;
 vec4 uvmap;
 void main() {
     // uv map data at current position
     uvmap = texture2D(texture2, v_texture_coordinates);

     // map current position by using uv map data
     my_new_coords.x = v_texture_coordinates.x + uvmap.r;
     my_new_coords.y = v_texture_coordinates.y + uvmap.g;

     // sample texture 2 with new mapped coordinates
     gl_FragColor = texture2D(texture1, my_new_coords);
 }

Or if you just want to use the coordinates from UVMap you can do

 gl_FragColor = texture2D(texture1, uvmap);