GLSL Geometry Shader problems

Question

I have been pulling my hair out for 2 days over this.

First.. the code.. All of it.

//Leaf Vertex Shader
#version 330 compatibility 

out vec4 VertexColor;
void main(void)
{
    gl_Position = gl_ModelViewMatrix *gl_Vertex;
    gl_TexCoord[0].st = gl_MultiTexCoord0.st;
    gl_TexCoord[1].st  = gl_MultiTexCoord1.st;
    gl_TexCoord[2].st  = gl_MultiTexCoord2.st;
    gl_TexCoord[3].st  = gl_MultiTexCoord3.st;
    gl_TexCoord[4].st = gl_MultiTexCoord4.st; // this is out size in x.y
    VertexColor = gl_Color;
}

Geo shader

#version 330 compatibility 
#extension GL_EXT_geometry_shader4 : enable         

//uniform float uNormalsLength;      
 out vec2 texCoord;
out vec4 color;
layout(points) in;
layout(triangle_strip, max_vertices = 4) out;
void main(void)
{
    vec4 vVertex = gl_in[0].gl_Position;
    vec2 uvs[4];
    uvs[0] = gl_TexCoord[0].st;
    uvs[1] = gl_TexCoord[1].st;
    uvs[2] = gl_TexCoord[2].st;
    uvs[3] = gl_TexCoord[3].st;
    vec2 scale = gl_TexCoord[4].st;
    float vx[4];
    vx[0] = -1.0;
    vx[1] = -1.0;
    vx[2] = 1.0;
    vx[3] = 1.0;
    float vy[4];
    vy[0]= -1.0;
    vy[1]= 1.0;
    vy[2]= 1.0;
    vy[3]= -1.0;
    int indi[4];
    indi[0] =  0;
    indi[1] =  3;
    indi[2] =  1;
    indi[3] =  2;
    texCoord = uvs[0];
    vec4 oVertex = gl_PositionIn[0];
    color = vec4(0.5, 0.5, 0.5, 1.0);
    for(int i = 0; i < 4; ++i)
        {
        oVertex.xyz = vVertex.xyz;
        color = vec4(0.0, 0.0, 0.5, 1.0);
        oVertex.x += (vx[ indi[i] ] * scale.x);
        oVertex.y += (vx[ indi[i] ] * scale.y);
        texCoord.xy = uvs[ indi[i] ].xy;

        gl_Position = gl_ModelViewProjectionMatrix * oVertex;
        EmitVertex();
    }


         EndPrimitive();
}

Frag shader

// leaf fragment shader
#version 330 compatibility 
uniform sampler2D colorMap;
in vec2 texCoord;
in vec4 color;
out vec4 FragColor;
void main (void)
{
    vec2 uv = texCoord.xy;
    uv.y *= -1.0;
    vec4 t = texture2D(colorMap,uv);
    FragColor = t;
    FragColor.rgb += color.rgb;
}

When I send this a point, I get nothing out but a point. I have tried sending triangle_stips.. It makes no difference.

When I check the int32 returned gl_getuniformlocation for colorMap, its -1.

It is almost like this program does NOT running when I call gl_useprogram. It has not effect.. I can't even effect the color and.. getting back -1 for colorMap makes me think the fragment shader never gets any input. I have tried hard coding a vertex in the vertex shader and it makes no difference.

I'm not sure about the transform but I can't test this as I cant see ANYTHING but a point. Like I said.. Its like its never being loaded. If I send the vertices as a triangle_strip, I see a triangle_strip on the screen. Thanks for any one that takes the time to read this.

Sorry about all the code but maybe someone can spot something I'm missing. Also.. There is a lot of deprecated things in there but it has never been a problem in my other shaders.

Answer 1

OK, I'll try to point out all of the wrong things you do. But since there's so much that's wrong with this code, I probably won't get them all.

#extension GL_EXT_geometry_shader4 : enable 

The interactions between core Geometry Shaders in GL 3.2 and the EXT_geometry_shader4 functionality is not well-defined. Therefore, exactly what this line will do is unknown.

This line, and any code that relates to it, should be removed. For example, the input variable gl_PositionIn is defined by EXT_geometry_shader4, but not core GS. Oddly, you do use that value, but you immediately overwrite it in the first step of the loop, for some reason.

uvs[0] = gl_TexCoord[0].st;
uvs[1] = gl_TexCoord[1].st;
uvs[2] = gl_TexCoord[2].st;
uvs[3] = gl_TexCoord[3].st;
vec2 scale = gl_TexCoord[4].st;

No matter what shader stage you put it in, gl_TexCoord always refers to the output of that stage. Never the input. So you're copying the value of a shader stage output variable that you never write to.

The correct input variable name would be gl_in[0].gl_TexCoord[x].

The easiest way to check to see if you're reading an input variable in a GS is this: do you have to prefix/suffix it with an array index? If you don't, then it's not an input variable. All Geometry Shader input variables are arrayed.

gl_Position = gl_ModelViewProjectionMatrix * oVertex;

In your vertex shader, you transformed the output position by gl_ModelViewMatrix. Which transforms the positions from model space to camera/eye/view space (whatever you want to call it). So those positions are now camera-relative.

The gl_ModelViewProjectionMatrix transforms positions from model space to projection space. If your positions are in camera space (which, as previously established, they are), then this transformation is nonsensical.

The correct matrix to use here would be gl_ProjectionMatrix, which transforms from camera space to projection space.

---

This is what functioning code would look like. Code that uses the OpenGL 3.3 core profile.

Vertex Shader:

#version 330 core 
in vec4 position;
in vec4 color;
in vec2 texCoords[4]; //Yes, that's perfectly legal.
in vec2 scaleFactor;

out VS
{
    out vec4 color;
    out vec2 texCoords[4]; //Yes, this is legal too.
    out vec2 scaleFactor;
} dest;

//The transformation from model space to camera space.
uniform mat4 modelToCamera;

void main(void)
{
    gl_Position = modelToCamera * position;
    dest.color = color;
    for(int i = 0; i < texCoord.length(); ++i)
        dest.texCoord[i] = texCoord[i];
    dest.scaleFactor = scaleFactor;
}

Geometry Shader:

#version 330 core
layout(points) in;
layout(triangle_strip, points) out;

in VS
{
    vec4 color;
    vec2 texCoords[4]; //Still legal.
    vec2 scaleFactor;
} source[]; //One for each input vertex, but since we're using points, it's just one.

out FS
{
    vec4 color;
    vec2 texCoord;
} dest;

const vec2 offsets[4] = vec2[4](
    vec2(-1.0, -1.0),
    vec2(-1.0,  1.0),
    vec2( 1.0,  1.0),
    vec2( 1.0, -1.0));

//The transformation from camera to projection.
uniform mat4 ProjectionTM;

void main(void)
{
    for(int i = 0; i < offsets.length(); ++i)
    {
        dest.color = source[0].color;
        dest.texCoord = source[0].texCoord[i];

        vec2 scale = offsets[i] * source[0].scaleFactor;
        vec4 pos = gl_in[0].gl_Position;
        pos.xy += scale;
        gl_Position = ProjectionTM * pos;
        EmitVertex();
    }

    EndPrimitive();
}

Fragment Shader:

#version 330 core 
in FS
{
    vec4 color;
    vec2 texCoord;
};

layout(location = 0) out vec4 FragColor;

uniform sampler2D colorMap;

void main (void)
{
    vec4 t = texture2D(colorMap, texCoord.xy * vec2(1.0, -1.0));
    FragColor = t;
    FragColor.rgb += color.rgb;
}