Make transparency not show what is behind the window in opengl with c++

Question

I am making a 2 dimensional image in opengl with C++, and am running into an interesting issue. Whenever I try to draw a partially transparent polygon on my image, it makes the window itself partially transparent where the polygon is. For example, I can see whatever is behind my window (e.g. my code) when I am running the program (which I don't want). I can also see the image behind the polygon (which I do want). Is there any way I can turn the "transparent window" behavior off? I have included what I feel to be relevant portions of the code below:

glClearColor(0.0f, 0.0f, 0.0f, 0.0f); // I have tried 1.0f for the alpha value too
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST);
glHint(GL_POLYGON_SMOOTH_HINT, GL_NICEST);
glEnable(GL_BLEND);
glEnable(GL_LINE_SMOOTH);
glEnable(GL_POLYGON_SMOOTH);
glPolygonMode (GL_FRONT_AND_BACK, GL_FILL);
glHint(GL_POINT_SMOOTH_HINT, GL_FASTEST);
glDisable(GL_POINT_SMOOTH);

glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

glMatrixMode (GL_MODELVIEW);
glLoadIdentity ();

    // other code to draw my opaque "background" object

    // Draw my partially transparent quad (note: this is where the window itself becomes partially transparent)
glBegin(GL_QUADS); // Begin drawing quads
    glColor4f(1.0,1.0,1.0,0.5); // Make a white quad with .5 alpha
    glVertex2f(-0.5, 0.5);
    glVertex2f(0.5, .05);
    glVertex2f(0.5, -0.5);
    glVertex2f(-0.5, -0.5);
    glEnd();

Other relevant information:

• I am running CentOS 6
• I am fairly new to opengl, and am working on the code after a prior developer, so I could be missing something trivial
• It is using the X Windows system

Here is the X Window creation code further debug, the problem is likely here rather than the opengl code above.

/*  The simplest possible Linux OpenGL program? Maybe...
 Modification for creating a RGBA window (transparency with compositors)
 by Wolfgang 'datenwolf' Draxinger

 (c) 2002 by FTB. See me in comp.graphics.api.opengl

 (c) 2011 Wolfgang Draxinger. See me in comp.graphics.api.opengl and on StackOverflow

 License agreement: This source code is provided "as is". You
 can use this source code however you want for your own personal
 use. If you give this source code to anybody else then you must
 leave this message in it.

 --
 <\___/>
 / O O \
 \_____/  FTB.

 --
 datenwolf

 ------------------------------------------------------------------------*/
static void createTheWindow() {
XEvent event;
int x, y, attr_mask;
XSizeHints hints;
XWMHints *StartupState;
XTextProperty textprop;
XSetWindowAttributes attr;
static char *title = "Fix me";

/* Connect to the X server */
Xdisplay = XOpenDisplay(NULL);
if (!Xdisplay)
{
    fatalError("Couldn't connect to X server\n");
}
Xscreen = DefaultScreen(Xdisplay);
Xroot = RootWindow(Xdisplay, Xscreen) ;

fbconfigs = glXChooseFBConfig(Xdisplay, Xscreen, VisData, &numfbconfigs);
for (int i = 0; i < numfbconfigs; i++)
{
    visual = (XVisualInfo_CPP*) glXGetVisualFromFBConfig(Xdisplay,
            fbconfigs[i]);
    if (!visual)
        continue;

    pictFormat = XRenderFindVisualFormat(Xdisplay, visual->visual);
    if (!pictFormat)
        continue;

    if (pictFormat->direct.alphaMask > 0)
    {
        fbconfig = fbconfigs[i];
        break;
    }
}

/* Create a colormap - only needed on some X clients, eg. IRIX */
cmap = XCreateColormap(Xdisplay, Xroot, visual->visual, AllocNone);

/* Prepare the attributes for our window */
attr.colormap = cmap;

attr.border_pixel = 0;
attr.event_mask = StructureNotifyMask | EnterWindowMask | LeaveWindowMask
        | ExposureMask | ButtonPressMask | ButtonReleaseMask
        | OwnerGrabButtonMask | KeyPressMask | KeyReleaseMask;

attr.background_pixmap = None;

attr_mask = CWBackPixmap | CWColormap | CWBorderPixel | CWEventMask; /* What's in the attr data */

width = DisplayWidth(Xdisplay, DefaultScreen(Xdisplay)) ;
height = DisplayHeight(Xdisplay, DefaultScreen(Xdisplay)) ;
x = width / 2, y = height / 2;
// x=0, y=10;

/* Create the window */
attr.do_not_propagate_mask = NoEventMask;
WindowHandle = XCreateWindow(Xdisplay, /* Screen */
Xroot, /* Parent */
x, y, width, height,/* Position */
1,/* Border */
visual->depth,/* Color depth*/
InputOutput,/* klass */
visual->visual,/* Visual */
attr_mask, &attr);/* Attributes*/

if (!WindowHandle)
{
    fatalError("Couldn't create the window\n");
}

/* Configure it...  (ok, ok, this next bit isn't "minimal") */
textprop.value = (unsigned char*) title;
textprop.encoding = XA_STRING;
textprop.format = 8;
textprop.nitems = strlen(title);

hints.x = x;
hints.y = y;
hints.width = width;
hints.height = height;
hints.flags = USPosition | USSize;

StartupState = XAllocWMHints();
StartupState->initial_state = NormalState;
StartupState->flags = StateHint;

XSetWMProperties(Xdisplay, WindowHandle, &textprop, &textprop,/* Window title/icon title*/
NULL, 0,/* Argv[], argc for program*/
&hints, /* Start position/size*/
StartupState,/* Iconised/not flag   */
NULL);

XFree(StartupState);

/* Open it, wait for it to appear */
int event_base, error_base = 0;

XMapWindow(Xdisplay, WindowHandle);
//   }

XIfEvent(Xdisplay, &event, WaitForMapNotify, (char*) &WindowHandle);

/* Set the kill atom so we get a message when the user tries to close the window */
if ((del_atom = XInternAtom(Xdisplay, "WM_DELETE_WINDOW", 0)) != None)
{
    XSetWMProtocols(Xdisplay, WindowHandle, &del_atom, 1);
}

}

Here are the settings for VisData:

static int VisData[] = { GLX_RENDER_TYPE, GLX_RGBA_BIT, GLX_DRAWABLE_TYPE,
    GLX_WINDOW_BIT, GLX_DOUBLEBUFFER, True, GLX_RED_SIZE, 1, GLX_GREEN_SIZE,
    1, GLX_BLUE_SIZE, 1, GLX_ALPHA_SIZE, 1, GLX_DEPTH_SIZE, 1,
    None
};

Here is where the rendering context is created:

static void createTheRenderContext() {
/* See if we can do OpenGL on this visual */
int dummy;
if (!glXQueryExtension(Xdisplay, &dummy, &dummy))
{
    fatalError("OpenGL not supported by X server\n");
}

/* Create the OpenGL rendering context */
RenderContext = glXCreateNewContext(Xdisplay, fbconfig, GLX_RGBA_TYPE, 0,
        True);
if (!RenderContext)
{
    fatalError("Failed to create a GL context\n");
}

GLXWindowHandle = glXCreateWindow(Xdisplay, fbconfig, WindowHandle, NULL);

/* Make it current */
if (!glXMakeContextCurrent(Xdisplay, GLXWindowHandle, GLXWindowHandle,
        RenderContext))
{
    fatalError("glXMakeCurrent failed for window\n");
}
}

Answer 1

What ratchet freak suggestet (Aero Glass effect in Windows) does not happen by accident, because one has to manually enable DWM transparency for this to happen.

However in X11/GLX it is perfectly possible to end up with a visual mode that has an Alpha Channel by default. If you want to get realiably a window that does or does not have an alpha channel the code gets a bit more complex than what most toolkits do.

The code you're using looks strikingly familiar. To be specific it seems to originate from a codesample I wrote about how to create a transparent window (you see where this is going), namely this code:

https://github.com/datenwolf/codesamples/blob/master/samples/OpenGL/x11argb_opengl/x11argb_opengl.c

The key sequence is this:

   fbconfigs = glXChooseFBConfig(Xdisplay, Xscreen, VisData, &numfbconfigs);
   fbconfig = 0;
   for(int i = 0; i<numfbconfigs; i++) {
    visual = (XVisualInfo*) glXGetVisualFromFBConfig(Xdisplay, fbconfigs[i]);
    if(!visual)
        continue;

    pict_format = XRenderFindVisualFormat(Xdisplay, visual->visual);
    if(!pict_format)
        continue;

    fbconfig = fbconfigs[i];
    if(pict_format->direct.alphaMask > 0) {
        break;
    }
}

What this does is, it selects an X11 Visual that matches one of the previously selected FBConfigs that also contains an alpha mask.

If I had to make a bet I suspect that the VisData array you passed to glXChooseFBConfig does not specify an alpha channel. So what happens is, that you may end up with a window that has an X11 alpha mask, but not an alpha channel accessible to OpenGL.

Since I never intended that code to be used for windows that don't have an alpha channel this code does only whats originally intended if VisData does select for an alpha channel.

You have now two options:

• implement a complementary test if(pict_format->direct.alphaMask == 0 && no_alpha_in(VisData)) break;

• select for an alpha channel in VisData and clear the alpha channel to 1.0 with OpenGL glClearColor(…,…,…,1.0f);

Answer 2

This is not a opengl problem, but rather the kind of window you are creating. I suspect you running a window manager with supports transparency effects. Either way, what probably is happening is that, when you render the transparent poly, the window canvas ends up with some alpha, and your window manager assumes that you want the background transparent. Turn off all advanced effects of your window manager to check.

I am not familiar with window creation code using xlib, but it probably has to do with the kind of window you are creating.