Adding Shadows to Parallax Occlusion Map

Question

I've implemented Parallax Occlusion Mapping through LearnOpengl, and now I want to add self-shadows so that the fragment extrusions throw shadows on the surface. I've read a few papers on the topic, but I admit it's a bit advanced for me. From what I understand, it's the same process as parallax occlusion mapping, but from the light's direction instead of the view direction. I tried modifying the fragment shader, but the shadows still don't show up.

This is how I want it to look like. http://www.cs.utah.edu/~sujin/courses/reports/cs6610/project-report/images/pom.png

This is the result of the modified fragment shader. Nothing changed from when it was just the parallax occlusion map.

Here's the modified fragment shader. I've marked that parts I added to the original parallax tutorial code.

#version 330 core
in vec2 o_texCoord;
in vec3 o_worldPos;
in vec3 world_normal;
in vec3 world_tangent;

out vec4 fragColor;

uniform vec3 light_pos;
uniform sampler2D diffuseMap;
uniform sampler2D normalMap;
uniform sampler2D heightMap;
uniform vec3 viewPosition;
uniform float heightScale;

vec2 ParallaxMapping (vec2 texCoord, vec3 viewDir)
{
    float minLayers = 0;
    float maxLayers = 32;
    float numLayers = mix(maxLayers, minLayers, abs(dot(vec3(0.0, 0.0, 1.0), viewDir)));
    float layerDepth = 1.0 / numLayers;

    float currentLayerDepth = 0;

    vec2 P = viewDir.xy / viewDir.z * heightScale;

    vec2 deltaTexCoords = P / numLayers;

    vec2 currentTexCoords = texCoord;

    float currentDepthMapValue = texture(heightMap, currentTexCoords).r;

    while (currentLayerDepth < currentDepthMapValue)
    {
        currentTexCoords -= deltaTexCoords;
        currentDepthMapValue = texture(heightMap, currentTexCoords).r;
        currentLayerDepth += layerDepth;
    }

    vec2 prevTexCoords = currentTexCoords + deltaTexCoords;
    float afterDepth = currentDepthMapValue - currentLayerDepth;
    float beforeDepth = texture(heightMap, prevTexCoords).r - currentLayerDepth + layerDepth;

    float weight = afterDepth / (afterDepth - beforeDepth);

    vec2 finalTexCoords = prevTexCoords * weight + currentTexCoords * (1.0 - weight);

    return finalTexCoords;
}

// FUNCTION I ADDED FOR SHADOW CALCULATION
float ShadowCalc(vec2 texCoord, vec3 lightDir)
{
    float minLayers = 0;
    float maxLayers = 32;
    float numLayers = mix(maxLayers, minLayers, abs(dot(vec3(0.0, 0.0, 1.0), lightDir)));

    float layerDepth = 1.0 / numLayers;

    float currentLayerDepth = 0;

    vec2 P = lightDir.xy / lightDir.z * heightScale;

    vec2 deltaTexCoords = P / numLayers;

    vec2 currentTexCoords = texCoord;

    float currentDepthMapValue = texture(heightMap, currentTexCoords).r;

    while (currentLayerDepth < currentDepthMapValue)
    {
        currentTexCoords -= deltaTexCoords;
        currentDepthMapValue = texture(heightMap, currentTexCoords).r;
        currentLayerDepth += layerDepth;
    }

    float r = currentDepthMapValue > currentLayerDepth ? 0.0 : 1.0;

    return r;
}

void main()
{
    mat3 TBN_norm = transpose(mat3(normalize(world_tangent),
                                   normalize(cross(world_normal, world_tangent)),
                                   normalize(world_normal)));
    vec3 viewDir = TBN_norm * normalize(o_worldPos - viewPosition);
    vec2 currentTex = ParallaxMapping(o_texCoord, viewDir);
    if (currentTex.x > 1.0 || currentTex.y > 1.0 || currentTex.x < 0.0 || currentTex.y < 0.0)
    {
        discard;
    }
    vec3 normal = texture(normalMap, currentTex).rgb;
    normal = normalize(normal * 2.0 - 1.0);
    vec3 lightDir = normalize(TBN_norm * light_pos - TBN_norm * o_worldPos);
    float dc = max(0.0, dot(lightDir, normal));

    // STUFF I ADDED FOR SHADOWS
    float shadow = 0;
    if (dc > 0)
    {
        shadow = ShadowCalc(currentTex, lightDir);
    }

    fragColor = shadow * dc * texture(diffuseMap, currentTex);
}

Answer 1

First of all the direction of the light source to the fragment in texture space is:

vec3 lightDir = TBN_norm * normalize(o_worldPos - light_pos);
float dc = max(0.0, dot(-lightDir, normal));

To check if a fragment is in self-shadow, you have to track the ray to the light source from starting a the the "parallax" texel.

float shadow = dc > 0.0 ? ShadowCalc(currentTex, lightDir) : 0.0;

The initial height (currentLayerDepth) is the height of the current fragment:

float currentDepthMapValue = texture(heightMap, currentTexCoords).r;
float currentLayerDepth = currentDepthMapValue;

Since the depth mao is an inverse depth map (1.0 is low), a fragment is in shadow if any layer depth (currentLayerDepth) is less or equal the current height (currentDepthMapValue). The sampling has to be aborted if the maximum depth (minimum value of 0.0) is reached.
Note, that the depth is decremented (currentLayerDepth -= layerDepth) and the texture samples are taken in the opposite direction (currentTexCoords += deltaTexCoords) in compare the the ParallaxMapping algorithm:

while (currentLayerDepth <= currentDepthMapValue && currentLayerDepth > 0.0)
{
    currentTexCoords += deltaTexCoords;
    currentDepthMapValue = texture(heightMap, currentTexCoords).r;
    currentLayerDepth -= layerDepth;
}
float r = currentLayerDepth > currentDepthMapValue ? 0.0 : 1.0;

Because of the division by the z-component in (P = lightDir.xy / lightDir.z), P and thus deltaTexCoords, points always to the light source (of course in the projection to the texture).
If the z-component of lightDir is grater than 0.0, the the surface is lit from the back. This results in a condition for an early abort:

if ( lightDir.z >= 0.0 )
    return 0.0;

The full function ShadowCalc function may look like this:

float ShadowCalc(vec2 texCoord, vec3 lightDir)
{
    if ( lightDir.z >= 0.0 )
        return 0.0;

    float minLayers = 0;
    float maxLayers = 32;
    float numLayers = mix(maxLayers, minLayers, abs(dot(vec3(0.0, 0.0, 1.0), lightDir)));

    vec2 currentTexCoords = texCoord;
    float currentDepthMapValue = texture(heightMap, currentTexCoords).r;
    float currentLayerDepth = currentDepthMapValue;

    float layerDepth = 1.0 / numLayers;
    vec2 P = lightDir.xy / lightDir.z * heightScale;
    vec2 deltaTexCoords = P / numLayers;

    while (currentLayerDepth <= currentDepthMapValue && currentLayerDepth > 0.0)
    {
        currentTexCoords += deltaTexCoords;
        currentDepthMapValue = texture(heightMap, currentTexCoords).r;
        currentLayerDepth -= layerDepth;
    }

    float r = currentLayerDepth > currentDepthMapValue ? 0.0 : 1.0;
    return r;
}