How to write text or put image on top of 3D cube in OpenGL java in android (Object is used in Augmenting Purpose)

Question

I'm developing an Augmented Reality based application using EasyAR SDK in android. It rendering cube on the top of image target by default. I want to print something on the top of that cube. So what should I do? I'm new to OpenGL, please help.

If I can put an image on top of that cube then it's also fine! I just want to display "Loading" on that cube, whether it is image or text, that doesn't really matter!

This is the current situation:

And I need something like this:

Here is my code that renders box on top of image target.

import android.opengl.GLES20;
import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
import java.nio.ShortBuffer;
import cn.easyar.Vec2F;
import cn.easyar.Matrix44F;

public class BoxRenderer {
    private int program_box;
    private int pos_coord_box;
    private int pos_color_box;
    private int pos_trans_box;
    private int pos_proj_box;
    private int vbo_coord_box;
    private int vbo_color_box;
    private int vbo_color_box_2;
    private int vbo_faces_box;

    private String box_vert = "uniform mat4 trans;\n"
            + "uniform mat4 proj;\n"
            + "attribute vec4 coord;\n"
            + "attribute vec4 color;\n"
            + "varying vec4 vcolor;\n"
            + "\n"
            + "void main(void)\n"
            + "{\n"
            + "    vcolor = color;\n"
            + "    gl_Position = proj*trans*coord;\n"
            + "}\n"
            + "\n";

    private String box_frag = "#ifdef GL_ES\n"
            + "precision highp float;\n"
            + "#endif\n"
            + "varying vec4 vcolor;\n"
            + "\n"
            + "void main(void)\n"
            + "{\n"
            + "    gl_FragColor = vcolor;\n"
            + "}\n"
            + "\n";

    private float[] flatten(float[][] a) {
        int size = 0;
        for (int k = 0; k < a.length; k += 1) {
            size += a[k].length;
        }
        float[] l = new float[size];
        int offset = 0;
        for (int k = 0; k < a.length; k += 1) {
            System.arraycopy(a[k], 0, l, offset, a[k].length);
            offset += a[k].length;
        }
        return l;
    }

    private int[] flatten(int[][] a) {
        int size = 0;
        for (int k = 0; k < a.length; k += 1) {
            size += a[k].length;
        }
        int[] l = new int[size];
        int offset = 0;
        for (int k = 0; k < a.length; k += 1) {
            System.arraycopy(a[k], 0, l, offset, a[k].length);
            offset += a[k].length;
        }
        return l;
    }

    private short[] flatten(short[][] a) {
        int size = 0;
        for (int k = 0; k < a.length; k += 1) {
            size += a[k].length;
        }
        short[] l = new short[size];
        int offset = 0;
        for (int k = 0; k < a.length; k += 1) {
            System.arraycopy(a[k], 0, l, offset, a[k].length);
            offset += a[k].length;
        }
        return l;
    }

    private byte[] flatten(byte[][] a) {
        int size = 0;
        for (int k = 0; k < a.length; k += 1) {
            size += a[k].length;
        }
        byte[] l = new byte[size];
        int offset = 0;
        for (int k = 0; k < a.length; k += 1) {
            System.arraycopy(a[k], 0, l, offset, a[k].length);
            offset += a[k].length;
        }
        return l;
    }

    private byte[] byteArrayFromIntArray(int[] a) {
        byte[] l = new byte[a.length];
        for (int k = 0; k < a.length; k += 1) {
            l[k] = (byte) (a[k] & 0xFF);
        }
        return l;
    }

    private int generateOneBuffer() {
        int[] buffer = {0};
        GLES20.glGenBuffers(1, buffer, 0);
        return buffer[0];
    }

    public void init() {
        program_box = GLES20.glCreateProgram();
        int vertShader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
        GLES20.glShaderSource(vertShader, box_vert);
        GLES20.glCompileShader(vertShader);
        int fragShader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
        GLES20.glShaderSource(fragShader, box_frag);
        GLES20.glCompileShader(fragShader);
        GLES20.glAttachShader(program_box, vertShader);
        GLES20.glAttachShader(program_box, fragShader);
        GLES20.glLinkProgram(program_box);
        GLES20.glUseProgram(program_box);
        pos_coord_box = GLES20.glGetAttribLocation(program_box, "coord");
        pos_color_box = GLES20.glGetAttribLocation(program_box, "color");
        pos_trans_box = GLES20.glGetUniformLocation(program_box, "trans");
        pos_proj_box = GLES20.glGetUniformLocation(program_box, "proj");

        vbo_coord_box = generateOneBuffer();
        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_coord_box);
        float cube_vertices[][] = {
                /* +z */{1.0f / 2, 1.0f / 2, 0.01f / 2}, {1.0f / 2, -1.0f / 2, 0.01f / 2}, {-1.0f / 2, -1.0f / 2, 0.01f / 2}, {-1.0f / 2, 1.0f / 2, 0.01f / 2},
                /* -z */{1.0f / 2, 1.0f / 2, -0.01f / 2}, {1.0f / 2, -1.0f / 2, -0.01f / 2}, {-1.0f / 2, -1.0f / 2, -0.01f / 2}, {-1.0f / 2, 1.0f / 2, -0.01f / 2}
        };
        FloatBuffer cube_vertices_buffer = FloatBuffer.wrap(flatten(cube_vertices));
        GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, cube_vertices_buffer.limit() * 4, cube_vertices_buffer, GLES20.GL_DYNAMIC_DRAW);

        vbo_color_box = generateOneBuffer();
        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_color_box);
        int cube_vertex_colors[][] = {
                {255, 0, 0, 128}, {0, 255, 0, 128}, {0, 0, 255, 128}, {0, 0, 0, 128},
                {0, 255, 255, 128}, {255, 0, 255, 128}, {255, 255, 0, 128}, {255, 255, 255, 128}};
        ByteBuffer cube_vertex_colors_buffer = ByteBuffer.wrap(byteArrayFromIntArray(flatten(cube_vertex_colors)));
        GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, cube_vertex_colors_buffer.limit(), cube_vertex_colors_buffer, GLES20.GL_STATIC_DRAW);

        vbo_color_box_2 = generateOneBuffer();
        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_color_box_2);
        int cube_vertex_colors_2[][] = {
                {255, 0, 0, 255}, {255, 255, 0, 255}, {0, 255, 0, 255}, {255, 0, 255, 255},
                {255, 0, 255, 255}, {255, 255, 255, 255}, {0, 255, 255, 255}, {255, 0, 255, 255}};
        ByteBuffer cube_vertex_colors_2_buffer = ByteBuffer.wrap(byteArrayFromIntArray(flatten(cube_vertex_colors_2)));
        GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, cube_vertex_colors_2_buffer.limit(), cube_vertex_colors_2_buffer, GLES20.GL_STATIC_DRAW);

        vbo_faces_box = generateOneBuffer();
        GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, vbo_faces_box);
        short cube_faces[][] = {
                /* +z */{3, 2, 1, 0}, /* -y */{2, 3, 7, 6}, /* +y */{0, 1, 5, 4},
                /* -x */{3, 0, 4, 7}, /* +x */{1, 2, 6, 5}, /* -z */{4, 5, 6, 7}};
        ShortBuffer cube_faces_buffer = ShortBuffer.wrap(flatten(cube_faces));
        GLES20.glBufferData(GLES20.GL_ELEMENT_ARRAY_BUFFER, cube_faces_buffer.limit() * 2, cube_faces_buffer, GLES20.GL_STATIC_DRAW);
    }

    public void render(Matrix44F projectionMatrix, Matrix44F cameraview, Vec2F size) {
        float size0 = size.data[0];
        float size1 = size.data[1];

        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_coord_box);
        float height = size0 / 1000;
        float cube_vertices[][] = {
                /* +z */{size0 / 2, size1 / 2, height / 2}, {size0 / 2, -size1 / 2, height / 2}, {-size0 / 2, -size1 / 2, height / 2}, {-size0 / 2, size1 / 2, height / 2},
                /* -z */{size0 / 2, size1 / 2, 0}, {size0 / 2, -size1 / 2, 0}, {-size0 / 2, -size1 / 2, 0}, {-size0 / 2, size1 / 2, 0}};
        FloatBuffer cube_vertices_buffer = FloatBuffer.wrap(flatten(cube_vertices));
        GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, cube_vertices_buffer.limit() * 4, cube_vertices_buffer, GLES20.GL_DYNAMIC_DRAW);

        GLES20.glEnable(GLES20.GL_BLEND);
        GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
        GLES20.glEnable(GLES20.GL_DEPTH_TEST);
        GLES20.glUseProgram(program_box);
        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_coord_box);
        GLES20.glEnableVertexAttribArray(pos_coord_box);
        GLES20.glVertexAttribPointer(pos_coord_box, 3, GLES20.GL_FLOAT, false, 0, 0);
        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_color_box);
        GLES20.glEnableVertexAttribArray(pos_color_box);
        GLES20.glVertexAttribPointer(pos_color_box, 4, GLES20.GL_UNSIGNED_BYTE, true, 0, 0);
        GLES20.glUniformMatrix4fv(pos_trans_box, 1, false, cameraview.data, 0);
        GLES20.glUniformMatrix4fv(pos_proj_box, 1, false, projectionMatrix.data, 0);
        GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, vbo_faces_box);
        for (int i = 0; i < 6; i++) {
            GLES20.glDrawElements(GLES20.GL_TRIANGLE_FAN, 4, GLES20.GL_UNSIGNED_SHORT, i * 4 * 2);
        }

        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_coord_box);
        float cube_vertices_2[][] = {
                /* +z */{size0 / 4, size1 / 4, size0 / 4}, {size0 / 4, -size1 / 4, size0 / 4}, {-size0 / 4, -size1 / 4, size0 / 4}, {-size0 / 4, size1 / 4, size0 / 4},
                /* -z */{size0 / 4, size1 / 4, 0}, {size0 / 4, -size1 / 4, 0}, {-size0 / 4, -size1 / 4, 0}, {-size0 / 4, size1 / 4, 0}};
        FloatBuffer cube_vertices_2_buffer = FloatBuffer.wrap(flatten(cube_vertices_2));
        GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, cube_vertices_2_buffer.limit() * 4, cube_vertices_2_buffer, GLES20.GL_DYNAMIC_DRAW);
        GLES20.glEnableVertexAttribArray(pos_coord_box);
        GLES20.glVertexAttribPointer(pos_coord_box, 3, GLES20.GL_FLOAT, false, 0, 0);
        GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbo_color_box_2);
        GLES20.glEnableVertexAttribArray(pos_color_box);

        GLES20.glVertexAttribPointer(pos_color_box, 4, GLES20.GL_UNSIGNED_BYTE, true, 0, 0);
        for (int i = 0; i < 6; i++) {
            GLES20.glDrawElements(GLES20.GL_TRIANGLE_FAN, 4, GLES20.GL_UNSIGNED_SHORT, i * 4 * 2);
        }
    }
}

Answer 1

This one might be a bit much but it is rather simple and flexible because you can pretty much use any kind of text or font or even background.

Basically we draw text on a bitmap and render this bitmap on a 2D plane. The background of the bitmap won't be rendered (using discard in the fragment shader) as long as it is a predefined color.

So first we need to setup one additional vertex attribute for texture coordinates. Here is the complete setup including vertices and texture coordinates for a simple 2D-plane:

//the geometry with texture coordinates
public int vbs[] = new int[2];

public void initSprite(){
    float vertices[] = {
            1.0f, -1.0f, 0.0f,  //triangle 1
            -1.0f, -1.0f, 0.0f,
            -1.0f, 1.0f, 0.0f,

            -1.0f, 1.0f, 0.0f,  //triangle 2
            1.0f, 1.0f, 0.0f,
            1.0f, -1.0f, 0.0f
    };
    float texcoords[] = {
            1.0f, 1.0f, 0.0f,  //triangle 1
            0.0f, 1.0f, 0.0f,
            0.0f, 0.0f, 0.0f,

            0.0f, 0.0f, 0.0f,  //triangle 2
            1.0f, 0.0f, 0.0f,
            1.0f, 1.0f, 0.0f
    };

    int triangle_count = 2;

    FloatBuffer vertex_pos_buffer;
    FloatBuffer tex_coord_buffer;

    int bytes_per_float = 4;

    //generate buffers on gpu
    GLES20.glGenBuffers(2, vbs,0);

    // Allocate a direct block of memory on the native heap,
    // size in bytes is equal to vertices.length * BYTES_PER_FLOAT.
    // BYTES_PER_FLOAT is equal to 4, since a float is 32-bits, or 4 bytes.
    vertex_pos_buffer = ByteBuffer.allocateDirect(vertices.length * bytes_per_float)
            // Floats can be in big-endian or little-endian order.
            // We want the same as the native platform.
            .order(ByteOrder.nativeOrder())
            // Give us a floating-point view on this byte buffer.
            .asFloatBuffer();

    //Transferring data from the Java heap to the native heap is then a matter of a couple calls:
    // Copy data from the Java heap to the native heap.
    vertex_pos_buffer.put(vertices)
            // Reset the buffer position to the beginning of the buffer.
            .position(0);

    //Bind the vertices buffer and give OpenGL the data
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbs[0]);
    GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, triangle_count * 3* 3 * bytes_per_float, vertex_pos_buffer, GLES20.GL_STATIC_DRAW);
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0);

    tex_coord_buffer = ByteBuffer.allocateDirect(texcoords.length * bytes_per_float)
            .order(ByteOrder.nativeOrder())
            .asFloatBuffer();
    tex_coord_buffer.put(texcoords).position(0);

    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbs[1]);
    GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, triangle_count * 3* 3 * bytes_per_float, tex_coord_buffer, GLES20.GL_STATIC_DRAW);
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0);
}

Next we need our Texture. We load a background image and draw our desired text ontop of it. size is the font size we want to use and should be a bit smaller than the background bitmap height. r g b are the color values of the font:

//the texture we gonna use during rendering
int tex = 0;

public void initTextTexture(String backgroundBitmapPath, String text, float size, int r, int g, int b){
    //load the bitmap
    Bitmap background = loadBitmapRGBA(backgroundBitmapPath);
    //check if image could load
    if(background == null){
        return;
    }

    android.graphics.Bitmap.Config bitmapConfig = background.getConfig();
    // set default bitmap config if none
    if(bitmapConfig == null) {
        bitmapConfig = android.graphics.Bitmap.Config.ARGB_8888;
    }
    // resource bitmaps are imutable,
    // so we need to convert it to mutable one
    background = background.copy(bitmapConfig, true);


    Canvas canvas = new Canvas(background);
    // new antialised Paint
    Paint paint = new Paint();

    paint.setColor(Color.rgb(r, g, b));
    // text size in pixels
    paint.setTextSize(size);


    // draw text to the Canvas center
    Rect bounds = new Rect();
    paint.getTextBounds(text, 0, text.length(), bounds);
    //left
    int x = 1;
    //center
    int y = (background.getHeight() + bounds.height())/2;
    canvas.drawText(text, x, y, paint);


    //create a texture with the bitmap we just created
    //try to allocate texture on GPU
    int gl_map[] = new int[1];
    GLES20.glGenTextures(1, gl_map, 0);
    tex = gl_map[0];

    //bind texture
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, tex);
    //move the bitmap to the openGL texture
    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, background, 0);

    //set nearest filter
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
}

private Bitmap loadBitmapRGBA(String path){
    if(path == null){
        return null;
    }
    //replace this with your application/activity context
    AssetManager assetManager = GlobalContext.getAppContext().getAssets();
    InputStream istr = null;
    try {
        istr = assetManager.open(path);
    } catch (IOException e) {
        e.printStackTrace();
    }

    Rect outPadding = new Rect();
    final BitmapFactory.Options options = new BitmapFactory.Options();
    options.inScaled = false;
    Bitmap image = BitmapFactory.decodeStream(istr, outPadding, options);

    return image;
}

Next we need draw our geometry with the texture we created, notice the glBindTexture :

public void drawTextSprite(){
    //program is the shader programm we gonna use to draw the 2d plane
    GLES20.glUseProgram(program);

    int locPosition = GLES20.glGetAttribLocation(program, "a_Position");
    int locTexcoord = GLES20.glGetAttribLocation(program, "a_TexCoord");

    int locTexture = GLES20.glGetUniformLocation(program, "tex_sampler");
    int locMVPMatrix = GLES20.glGetUniformLocation(program, "u_MVPMatrix");


    //bind the vertex data
    GLES20.glEnableVertexAttribArray(locPosition);
    GLES20.glEnableVertexAttribArray(locTexcoord);
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbs[0]);
    GLES20.glVertexAttribPointer(locPosition, 3, GLES20.GL_FLOAT, false, 0, 0);
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, vbs[1]);
    GLES20.glVertexAttribPointer(locTexcoord, 3, GLES20.GL_FLOAT, false, 0, 0);

    //bind texture
    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, tex);
    // Tell the texture uniform sampler to use this texture in the shader by binding to texture unit 0.
    GLES20.glUniform1i(locTexture, 0);

    //set up the mvp matrix
    float mvp[] = {
            1.0f, 0.0f, 0.0f, 0.0f, 
            0.0f, 1.0f, 0.0f, 0.0f,
            0.0f, 0.0f, 1.0f, 0.0f,
            0.0f, 0.0f, 0.0f, 1.0f
    };
    GLES20.glUniformMatrix4fv(locMVPMatrix, 1, false, mvp, 0);

    //draw 2 triangles
    GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, 2*3);
}

Now we just need our shader:

//vertex shader
uniform lowp mat4 u_MVPMatrix;
attribute lowp vec4 a_Position;
attribute lowp vec3 a_TexCoord;
varying lowp vec3 texc;
void main()
{
    texc = a_TexCoord;
    gl_Position = u_MVPMatrix * a_Position;
}

//fragment shader
uniform lowp sampler2D tex_sampler;
varying lowp vec3 texc;
void main()
{
    lowp vec3 color = texture2D(tex_sampler, texc.st).rgb;
    //test for the background color
    if(color.r == 1.0 && color.g == 0.0 && color.b == 1.0){
        discard; //get rid of the background
    }
    gl_FragColor = vec4(color.r, color.g, color.b, 1.0);
}

And to set everything up we call the following two lines:

initSprite();
initTextTexture("img/FF00FF_TEXT_BG.png","Loading...", 20.0f, 255, 255, 255);

FF00FF_TEXT_BG is stored under assets/img/ and looks like this.

If we call drawTextSprite(); during the renderloop we should get something like this:

Of course the output is a bit stretched, this is because i used the identity matrix to draw it. You just need to make sure you draw this over your box by providing the proper matrix. Also make sure not to draw the plane directly at the same position as the box's side but slightly further way, otherwise you wont see the text or artefacts if you use depthtest.

If you don't need to generate strings during runtime you can ofcourse just load bitmaps with prerendered text.

Hope that helps.