Nested Stencil - OpenGL ES

Question

I have a 2D node scene graph that I'm trying to 'nest' stencil clipping in.

I was thinking what I could do is when drawing the stencil, increment any pixel it writes to by 1, and keep track of what the current 'layer' is that I'm on.

Then when drawing, only write pixel data to the color buffer if the value of the stencil at that pixel is >= the current layer #.

This is the code I have now. It doesn't quite work. Where am I messing up?

First I call SetupStencilForMask(). Then draw stencil primitives. Next, call SetupStencilForDraw(). Now draw actual imagery When done with a layer, call DisableStencil().

Edit: Updated with solution. It doesn't work for individual items on the same layer, but otherwise is fine. Found a great article on how to actually pull this off, although it's fairly limited. http://cranialburnout.blogspot.com/2014/03/nesting-and-overlapping-translucent.html

// glClear(GL_STENICL_BIT) at start of each draw frame
static int stencilLayer = 0;

void SetupStencilForMask(void)
{
    if (stencilLayer == 0)
        glEnable(GL_STENCIL_TEST);

    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
    glStencilFunc(GL_LESS, stencilLayer, 0xff);
    glStencilOp(GL_INCR, GL_KEEP, GL_KEEP);
    glStencilMask(0xff);

    if (stencilLayer == 0)
        glClear(GL_STENCIL_BUFFER_BIT);

    stencilLayer++;
}

void SetupStencilForDraw()
{   
    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
    glStencilFunc(GL_EQUAL, stencilLayer, 0xff);
    glStencilMask(0x00);
}

void DisableStencil(void)
{
    if (--stencilLayer == 0)
        glDisable(GL_STENCIL_TEST);
}

Answer 1

I have figured out the way to do this in libgdx. I'm not sure you still need this, but for future reference here is the code:

/**
 * Start cropping
 * 
 * @param cropMask
 *            Mask plane
 */
public void startCropping(Plane cropMask) {

    // Check if there is active masking group
    if (activeCropMaskGroup == null) {

        // Create new one
        activeCropMaskGroup = new CropMaskingGroupDescriptor(cropMask);
    } else {

        // Increase hierarchy level
        activeCropMaskGroup.increaseHierachy(cropMask);
    }

}

/** End cropping */
public void endCropping() throws IllegalStateException {

    // Check if there is active group mask
    if (activeCropMaskGroup == null) {
        throw new IllegalStateException("Call start cropping before this!");
    }

    if (activeCropMaskGroup.getHierachy() > 0) {

        activeCropMaskGroup.decreaseHierachy();
    } else {

        // Finish setup of crop data
        cropMaskGroups.add(activeCropMaskGroup);
        activeCropMaskGroup = null;
    }

}

/** Crop registered planes for cropping */
private void cropRender(CropMaskingGroupDescriptor cropMaskGroupDescriptor) {

    // Draw mask to stencil buffer
    Gdx.gl.glClear(GL20.GL_STENCIL_BUFFER_BIT);

    // setup drawing to stencil buffer
    Gdx.gl20.glEnable(GL20.GL_STENCIL_TEST);

    // Number of registered hierarchy levels
    int hierarchyLevels = cropMaskGroupDescriptor.getRegisteredBatch().size();

    // Loop trough hierarchy
    for (int hierarchyLevel = 0; hierarchyLevel < hierarchyLevels; hierarchyLevel++) {

        Gdx.gl20.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
        Gdx.gl20.glStencilOp(GL20.GL_INCR, GL20.GL_INCR, GL20.GL_INCR);
        Gdx.gl20.glColorMask(false, false, false, false);
        Gdx.gl20.glDepthMask(false);

        // Draw mask with decal batch
        cropMaskBatch.add(((NativePlane) cropMaskGroupDescriptor.getCroppingMasks().get(hierarchyLevel)).getPlane());
        cropMaskBatch.flush();

        // fix stencil buffer, enable color buffer
        Gdx.gl20.glColorMask(true, true, true, true);
        Gdx.gl20.glDepthMask(true);
        Gdx.gl20.glStencilOp(GL20.GL_KEEP, GL20.GL_KEEP, GL20.GL_KEEP);

        // draw where pattern has been drawn
        Gdx.gl20.glStencilFunc(GL20.GL_LEQUAL, hierarchyLevel + 1, 0xffffffff);

        // Loop trough registered masked layers and found which one belongs
        // to
        // current hierarchy level
        for (int i = 0; i < cropMaskGroupDescriptor.getMaskedLayers().size(); i++) {

            if (cropMaskGroupDescriptor.getMaskedLayers().get(i).getHierarchyId() == hierarchyLevel) {

                Plane plane = cropMaskGroupDescriptor.getMaskedLayers().get(i).getMaskedPlane();
                cropMaskGroupDescriptor.getRegisteredBatch().get(hierarchyLevel).add(((NativePlane) plane).getPlane());
            }

        }

        cropMaskGroupDescriptor.getRegisteredBatch().get(hierarchyLevel).flush();

    }

    Gdx.gl20.glDisable(GL20.GL_STENCIL_TEST);

}

And inner class inside renderer module.

/** * Cropped layer descriptor * * @author Veljko Ilkic * */ private class CropMaskLayerDescriptor {

    /** Layer that needs to be masked */
    private Plane maskedPlane;

    /** Hierarchy level in which belongs */
    private int hierarchyId = 0;

    /** Constructor 1 */
    public CropMaskLayerDescriptor(Plane maskedPlane, int hierarchyId) {
        this.maskedPlane = maskedPlane;
        this.hierarchyId = hierarchyId;
    }

    public Plane getMaskedPlane() {
        return maskedPlane;
    }

    public int getHierarchyId() {
        return hierarchyId;
    }

}

/**
 * Crop masking group descriptor class
 * 
 * @author Veljko Ilkic
 * 
 */
private class CropMaskingGroupDescriptor {

    /** Crop mask */
    private ArrayList<Plane> croppingMasks = new ArrayList<Plane>();

    /** Planes that will be masked by crop mask */
    private ArrayList<CropMaskLayerDescriptor> maskedLayers = new ArrayList<Renderer.CropMaskLayerDescriptor>();

    /** Batch for drawing masked planes */
    private ArrayList<DecalBatch> hierarchyBatches = new ArrayList<DecalBatch>();

    private int activeHierarchyLayerId = 0;

    /** Constructor 1 */
    public CropMaskingGroupDescriptor(Plane topLevelCropMask) {

        // Create batch for top level hierarchy
        hierarchyBatches.add(new DecalBatch(new CameraGroupStrategy(perspectiveCamera)));

        // Register top level crop mask
        croppingMasks.add(topLevelCropMask);
    }

    /** Increase hierarchy level of the group */
    public void increaseHierachy(Plane hierarchyCropMask) {
        activeHierarchyLayerId++;

        // Create individual batch for hierarchy level
        hierarchyBatches.add(new DecalBatch(new CameraGroupStrategy(perspectiveCamera)));

        // Register crop mask for current hierarchy level
        croppingMasks.add(hierarchyCropMask);
    }

    /** Decrease hierarchy group */
    public void decreaseHierachy() {
        activeHierarchyLayerId--;
    }

    /** Get current hierarchy level */
    public int getHierachy() {
        return activeHierarchyLayerId;
    }

    /** Register plane for masking */
    public void registerLayer(Plane maskedPlane) {
        hierarchyBatches.get(activeHierarchyLayerId).add(((NativePlane) maskedPlane).getPlane());
        maskedLayers.add(new CropMaskLayerDescriptor(maskedPlane, activeHierarchyLayerId));
    }

    /** Get all registered batched */
    public ArrayList<DecalBatch> getRegisteredBatch() {
        return hierarchyBatches;
    }

    /** Get registered cropping masks */
    public ArrayList<Plane> getCroppingMasks() {
        return croppingMasks;
    }

    /** Get layer that should be masked */
    public ArrayList<CropMaskLayerDescriptor> getMaskedLayers() {
        return maskedLayers;
    }

    /** Dispose */
    public void dispose() {
        for (int i = 0; i < hierarchyBatches.size(); i++) {
            hierarchyBatches.get(i).dispose();
            hierarchyBatches.set(i, null);
        }

        hierarchyBatches.clear();
    }
}

Hope this helps.