OpenGL and AWT/Swing User Interface

Question

I am currently developing 3d viewer with OpenGL (LWJGL) for an embedded system. Without going into much detail, there is an application written on Java/Swing that currently has fully implemented UI and logic on Java+Swing. But it was decided that instead of 2d top-down picture it'd be cool to have 3d view as well, and that's where I come in.

I will say that I have to use GLES 3.0 or Opengl 2.1, and I prefer GLES because it has more features compared to 2.1 I also have a standalone application implemented with GLFW windowing system (a default for LWJGL) that runs fine on the device - it doesn't lag and gives decent FPS. Again, when running in a GLFW window.

But now I need to attach it to JFrame preferably, and that's where my problems start. Basically, what I need is to have my 3D view render as a background, and then have Swing buttons/panels and additional windows (for example with options menu) display on top of it.

I've implemented a basic algorhythm of simply reading the FrameBuffer and drawing it on a canvas as a raster image. But that's SLOW. I get like 10 FPS with that.

private void render()
{
    logic.Render(window);      //GLFW window
    window.update();

    ByteBuffer nativeBuffer = BufferUtils.createByteBuffer(GlobalSettings.WINDOW_WIDTH*GlobalSettings.WINDOW_HEIGHT*4);
    BufferedImage image = new BufferedImage(GlobalSettings.WINDOW_WIDTH,GlobalSettings.WINDOW_HEIGHT, BufferedImage.TYPE_4BYTE_ABGR);
    GLES30.glReadPixels(0, 0, GlobalSettings.WINDOW_WIDTH,GlobalSettings.WINDOW_HEIGHT, GLES20.GL_RGBA, GLES30.GL_UNSIGNED_BYTE, nativeBuffer);
    byte[] imgData = ((DataBufferByte)image.getRaster().getDataBuffer()).getData();
    nativeBuffer.get(imgData);

    Graphics g = canvas.getGraphics();
    g.drawImage(image, 0,0, GlobalSettings.WINDOW_WIDTH,GlobalSettings.WINDOW_HEIGHT, null);

}

The other thing I've tried is using this library https://github.com/LWJGLX/lwjgl3-awt that seems to be recommended a lot. The problem is, it uses some OpenGL 3.1 functionality, and getting it to work AT ALL under 2.1 context was a huge chore. But as for GLES - I couldn't make it work at all, and I obviously can't have it create context for 2.1 and then use GLES functionality further on, so it basically breaks my entire rendering code. Maybe I'm just not doing it right, but anyway - I couldn't make it work for me.

And that's pretty much where I stand. I thought I'd ask for help here. To me, at this point, it seems likely that I'll have to write the entire interface for OpenGL/GLFW entirely, and forgo Swing completely. Which is not ideal at all - I even wonder if we have such time at all.

Please, point me in the right direction if there are ways to make my thing work with AWT.

Answer 1

I will post an answer myself. It works well both on Linux desktop and on the embedded system, with good performance so far as I can see.

The idea is taking an EGL context and GLES, and setting AWT canvas surface as a direct target for the context. You don't need to read buffer and then paint it on the canvas. Below is my Window class initialization.

private Canvas canvas;
private JAWTDrawingSurface ds;
public long display;
private long eglDisplay;
public long drawable;
private long surface;

public static final JAWT awt;
static {
    awt = JAWT.calloc();
    awt.version(JAWT_VERSION_1_4);
    if (!JAWT_GetAWT(awt))
        throw new AssertionError("GetAWT failed");
}

public void lock() throws AWTException {
    int lock = JAWT_DrawingSurface_Lock(ds, ds.Lock());
    if ((lock & JAWT_LOCK_ERROR) != 0)
        throw new AWTException("JAWT_DrawingSurface_Lock() failed");
}

public void unlock() throws AWTException {
    JAWT_DrawingSurface_Unlock(ds, ds.Unlock());
}

public void Init2()
{
    frame = new JFrame("AWT test");
    frame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
    frame.setLayout(new BorderLayout());
    frame.setPreferredSize(new Dimension(width, height));

    canvas = new Canvas();
    frame.add(canvas);

    frame.pack();
    frame.setVisible(true);
    frame.transferFocus();

    int error;

    System.out.println("Window init2() started");

    this.ds = JAWT_GetDrawingSurface(canvas, awt.GetDrawingSurface());
    //JAWTDrawingSurface ds = JAWT_GetDrawingSurface(canvas, awt.GetDrawingSurface());
    try
    {
        lock();
        try
        {
            JAWTDrawingSurfaceInfo dsi = JAWT_DrawingSurface_GetDrawingSurfaceInfo(ds, ds.GetDrawingSurfaceInfo());

            JAWTX11DrawingSurfaceInfo dsiWin = JAWTX11DrawingSurfaceInfo.create(dsi.platformInfo());

            int depth = dsiWin.depth();
            this.display = dsiWin.display();
            this.drawable = dsiWin.drawable();

            System.out.printf("EGL Display %d, drawable: \n", display, drawable);

            eglDisplay = eglGetDisplay(display);

            EGLCapabilities egl;
            try (MemoryStack stack = stackPush()) {
                IntBuffer major = stack.mallocInt(1);
                IntBuffer minor = stack.mallocInt(1);

                if (!eglInitialize(eglDisplay, major, minor)) {
                    throw new IllegalStateException(String.format("Failed to initialize EGL [0x%X]", eglGetError()));
                }

                egl = EGL.createDisplayCapabilities(eglDisplay, major.get(0), minor.get(0));
            }
            System.out.println("EGL caps created");

            IntBuffer attrib_list = BufferUtils.createIntBuffer(18);
            attrib_list.put(EGL_CONFORMANT).put(EGL_OPENGL_ES2_BIT);
            //attrib_list.put(EGL_ALPHA_MASK_SIZE).put(4);
            //attrib_list.put(EGL_ALPHA_SIZE).put(4);
            //attrib_list.put(EGL_RED_SIZE).put(5);
            //attrib_list.put(EGL_GREEN_SIZE).put(6);
            //attrib_list.put(EGL_BLUE_SIZE).put(5);
            //attrib_list.put(EGL_DEPTH_SIZE).put(4);
            //attrib_list.put(EGL_SURFACE_TYPE).put(EGL_WINDOW_BIT);
            attrib_list.put(EGL_NONE);
            attrib_list.flip();

            PointerBuffer fbConfigs = BufferUtils.createPointerBuffer(1);
            IntBuffer numConfigs = BufferUtils.createIntBuffer(1);

            boolean test2 = eglChooseConfig(eglDisplay, attrib_list, fbConfigs,numConfigs);

            if (fbConfigs == null || fbConfigs.capacity() == 0) {
                // No framebuffer configurations supported!
                System.out.println("No supported framebuffer configurations found");
            }

            long test = numConfigs.get(0);

            IntBuffer context_attrib_list = BufferUtils.createIntBuffer(18);
            context_attrib_list.put(EGL_CONTEXT_MAJOR_VERSION).put(3);
            context_attrib_list.put(EGL_CONTEXT_MINOR_VERSION).put(0);
            context_attrib_list.put(EGL_NONE);
            context_attrib_list.flip();

            long context = eglCreateContext(eglDisplay,fbConfigs.get(0),EGL_NO_CONTEXT,context_attrib_list);

            error = eglGetError();

            surface = eglCreateWindowSurface(eglDisplay,fbConfigs.get(0),drawable,(int[])null);

            error = eglGetError();

            eglMakeCurrent(eglDisplay,surface,surface,context);

            error = eglGetError();

            GLESCapabilities gles = GLES.createCapabilities();
            System.out.println("CLES caps created");
        }
        finally
        {
            unlock();
        }

    }
    catch (Exception e)
    {
        System.out.println("JAWT Failed");
    }

    // Render with OpenGL ES
    glClearColor(0f,0f,0f,1f);
    glfwSwapInterval(vSync);
    glEnable(GL_DEPTH_TEST);

    int[] range = new int[2];
    int[] precision = new int[2];
    glGetShaderPrecisionFormat(GL_FRAGMENT_SHADER, GL_HIGH_FLOAT, range, precision);

    System.out.println("Window context Initialized");
}

Note that attribute lists can be whatever you want or nothing at all. I actually ran into GLES problem though when I didn't specify 3.0 as version compatibility for the context - it tried to use OpenGl ES-CL 1.1 for some reason and failed. Anyway, simply specifying minor and major context version fixed it. Also note that it's the initialization only. JFrame and Canvas are created elsewhere and Canvas then passed to the Window class constructor. Another important thing is the lock() / unlock() functions. You will get lots of XCB errors and crashes without them - because other threads will try to update display while you're drawing, causing conflicts and XCB will crash.

Also, you need to lock and unlock when you swap buffers (i.e. when you actually draw the framebuffer.

public void update()
{
    try
    {
        lock();
        eglSwapBuffers(eglDisplay, surface);
        unlock();
    }
    catch (Exception e)
    {
        System.out.println("Swap buffers failed");
    }
}

Don't mind that my current exception handling is lacking - I am editing my answer as soon as I found the solution lest my previous version confuses people.

I would also like to give credit to lwjgl-awt project, for giving me ideas. It does not support EGL as is, so I had to modify it a bit, but I took parts from there, so credit where credit is due. https://github.com/LWJGLX/lwjgl3-awt

Just for comparison sake, here is the GLFW version. It's the initialization of the same class, basically, but it simply does other things. Here, however, the window is created directly inside the method.

public void Init()
{
    System.out.println("Window init() started");

    GLFWErrorCallback.createPrint().set();
    if (!glfwInit()) {
        throw new IllegalStateException("Unable to initialize glfw");
    }

    glfwDefaultWindowHints();
    glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
    glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);

    // GLFW setup for EGL & OpenGL ES
    glfwWindowHint(GLFW_CONTEXT_CREATION_API, GLFW_EGL_CONTEXT_API);
    glfwWindowHint(GLFW_CLIENT_API, GLFW_OPENGL_ES_API);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0);

    windowHandle = glfwCreateWindow(width, height, title, NULL, NULL);
    if (windowHandle == NULL) {
        throw new RuntimeException("Failed to create the GLFW window");
    }

    System.out.printf("Window handle created %d\n", windowHandle);

    SetCallbacks();

    // EGL capabilities
    displayHandle = glfwGetEGLDisplay();

    System.out.printf("EGL DisplayHandle %d\n", displayHandle);

    try (MemoryStack stack = stackPush()) {
        IntBuffer major = stack.mallocInt(1);
        IntBuffer minor = stack.mallocInt(1);

        if (!eglInitialize(displayHandle, major, minor)) {
            throw new IllegalStateException(String.format("Failed to initialize EGL [0x%X]", eglGetError()));
        }

        EGLCapabilities egl = EGL.createDisplayCapabilities(displayHandle, major.get(0), minor.get(0));
    }
    System.out.println("EGL caps created");

    // OpenGL ES capabilities
    glfwMakeContextCurrent(windowHandle);
    System.out.printf("Current context: %d.%d rev %d, Client_Context: %d\n",glfwGetWindowAttrib(windowHandle,GLFW_CONTEXT_VERSION_MAJOR),
            glfwGetWindowAttrib(windowHandle,GLFW_CONTEXT_VERSION_MINOR), glfwGetWindowAttrib(windowHandle,GLFW_CONTEXT_REVISION),
            glfwGetWindowAttrib(windowHandle,GLFW_CLIENT_API));

    GLESCapabilities gles = GLES.createCapabilities();
    System.out.println("CLES caps created");

    // Render with OpenGL ES
    //glfwShowWindow(windowHandle);
    glClearColor(0f,0f,0f,1f);
    glfwSwapInterval(vSync);
    glEnable(GL_DEPTH_TEST);

    int[] range = new int[2];
    int[] precision = new int[2];
    glGetShaderPrecisionFormat(GL_FRAGMENT_SHADER, GL_HIGH_FLOAT, range, precision);

    System.out.printf("Current context: %d.%d\n",glfwGetWindowAttrib(windowHandle,GLFW_CONTEXT_VERSION_MAJOR),
                                            glfwGetWindowAttrib(windowHandle,GLFW_CONTEXT_VERSION_MINOR));
}