Corrupted decoding of a video using MediaCodec in Android 5+

Question

Simply trying to decode frames from videos. While working with Android 4+ (<5), it worked just fine. I'm using parts of the example that can be found here: http://bigflake.com/mediacodec/ "ExtractMpegFramesTest.java (requires 4.1, API 16)"

The problem is - it extracts a frame, but the result Bitmap is as can be seen here (Saved an image right after decoding it):

The real video of course has "real" frames, and not "stretched" 1 column.

I've saved this image right after the code line:

bmp.copyPixelsFromBuffer(mPixelBuf);
// <-- here I saved the above image

Is there some major update (I can't find) to the decoder that solves this ?

Answer 1

yst's answer creates a mirrored image

st.getTransformMatrix(mIntermediateMatrix);
if(invert){
   Matrix.setIdentityM(identityMatrix, 0);
   Matrix.translateM(identityMatrix, 0, 1, 1, 0);
   Matrix.rotateM(identityMatrix, 0, 180, 0, 0, 1);
   //fixes mirror image
   Matrix.translateM(identityMatrix, 0, 0, 1, 0);
   Matrix.rotateM(identityMatrix, 0, 180, 1, 0, 0);
   Matrix.multiplyMM(mSTMatrix, 0, identityMatrix, 0,mIntermediateMatrix,0);
} else {
   identityMatrix = mIntermediateMatrix;
}

This is my slightly altered STextureRender class from bigflake's ExtractMpegFramesTest.java (requires 4.2, API 17)

/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
private static class STextureRender {


private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
        // X, Y, Z, U, V
        -1.0f, -1.0f, 0, 0.f, 0.f,
        1.0f, -1.0f, 0, 1.f, 0.f,
        -1.0f,  1.0f, 0, 0.f, 1.f,
        1.0f,  1.0f, 0, 1.f, 1.f,
};

private FloatBuffer mTriangleVertices;

private static final String VERTEX_SHADER =
        "uniform mat4 uMVPMatrix;\n" +
                "uniform mat4 uSTMatrix;\n" +
                "attribute vec4 aPosition;\n" +
                "attribute vec4 aTextureCoord;\n" +
                "varying vec2 vTextureCoord;\n" +
                "void main() {\n" +
                "    gl_Position = uMVPMatrix * aPosition;\n" +
                "    vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
                "}\n";

private static final String FRAGMENT_SHADER =
        "#extension GL_OES_EGL_image_external : require\n" +
                "precision mediump float;\n" +      // highp here doesn't seem to matter
                "varying vec2 vTextureCoord;\n" +
                "uniform samplerExternalOES sTexture;\n" +
                "void main() {\n" +
                "    gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
                "}\n";

private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private float[] identityMatrix = new float[16];
private float[] mIntermediateMatrix = new float[16];

private int mProgram;
private int mTextureID = -12345;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;

public STextureRender() {
    mTriangleVertices = ByteBuffer.allocateDirect(
            mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
            .order(ByteOrder.nativeOrder()).asFloatBuffer();
    mTriangleVertices.put(mTriangleVerticesData).position(0);

    Matrix.setIdentityM(mSTMatrix, 0);
}

public int getTextureId() {
    return mTextureID;
}

/**
 * Draws the external texture in SurfaceTexture onto the current EGL surface.
 */
public void drawFrame(SurfaceTexture st, boolean invert) {
    checkGlError("onDrawFrame start");
    st.getTransformMatrix(mIntermediateMatrix);


    if(invert){
        Matrix.setIdentityM(identityMatrix, 0);
        Matrix.translateM(identityMatrix, 0, 1, 1, 0);
        Matrix.rotateM(identityMatrix, 0, 180, 0, 0, 1);
        //fixes mirror image
        Matrix.translateM(identityMatrix, 0, 0, 1, 0);
        Matrix.rotateM(identityMatrix, 0, 180, 1, 0, 0);
        Matrix.multiplyMM(mSTMatrix, 0, identityMatrix, 0, mIntermediateMatrix,0);
    } else {
        mSTMatrix = mIntermediateMatrix;
    }

/*
    if (invert) {
        mSTMatrix[5] = -mSTMatrix[5];
        mSTMatrix[13] = 1.0f - mSTMatrix[13];
    }
*/
    // (optional) clear to green so we can see if we're failing to set pixels
    GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);

    GLES20.glUseProgram(mProgram);
    checkGlError("glUseProgram");

    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);

    mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
    GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
            TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
    checkGlError("glVertexAttribPointer maPosition");
    GLES20.glEnableVertexAttribArray(maPositionHandle);
    checkGlError("glEnableVertexAttribArray maPositionHandle");

    mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
    GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false,
            TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
    checkGlError("glVertexAttribPointer maTextureHandle");
    GLES20.glEnableVertexAttribArray(maTextureHandle);
    checkGlError("glEnableVertexAttribArray maTextureHandle");

    Matrix.setIdentityM(mMVPMatrix, 0);
    GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
    GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
    checkGlError("glDrawArrays");

    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
}

/**
 * Initializes GL state.  Call this after the EGL surface has been created and made current.
 */
public void surfaceCreated() {
    mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
    if (mProgram == 0) {
        throw new RuntimeException("failed creating program");
    }

    maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
    checkLocation(maPositionHandle, "aPosition");
    maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
    checkLocation(maTextureHandle, "aTextureCoord");

    muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
    checkLocation(muMVPMatrixHandle, "uMVPMatrix");
    muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
    checkLocation(muSTMatrixHandle, "uSTMatrix");

    int[] textures = new int[1];
    GLES20.glGenTextures(1, textures, 0);

    mTextureID = textures[0];
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
    checkGlError("glBindTexture mTextureID");

    GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
            GLES20.GL_NEAREST);
    GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
            GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
            GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
            GLES20.GL_CLAMP_TO_EDGE);
    checkGlError("glTexParameter");
}

/**
 * Replaces the fragment shader.  Pass in null to reset to default.
 */
public void changeFragmentShader(String fragmentShader) {
    if (fragmentShader == null) {
        fragmentShader = FRAGMENT_SHADER;
    }
    GLES20.glDeleteProgram(mProgram);
    mProgram = createProgram(VERTEX_SHADER, fragmentShader);
    if (mProgram == 0) {
        throw new RuntimeException("failed creating program");
    }
}

private int loadShader(int shaderType, String source) {
    int shader = GLES20.glCreateShader(shaderType);
    checkGlError("glCreateShader type=" + shaderType);
    GLES20.glShaderSource(shader, source);
    GLES20.glCompileShader(shader);
    int[] compiled = new int[1];
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
    if (compiled[0] == 0) {
        Log.e(TAG, "Could not compile shader " + shaderType + ":");
        Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
        GLES20.glDeleteShader(shader);
        shader = 0;
    }
    return shader;
}

private int createProgram(String vertexSource, String fragmentSource) {
    int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
    if (vertexShader == 0) {
        return 0;
    }
    int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
    if (pixelShader == 0) {
        return 0;
    }

    int program = GLES20.glCreateProgram();
    if (program == 0) {
        Log.e(TAG, "Could not create program");
    }
    GLES20.glAttachShader(program, vertexShader);
    checkGlError("glAttachShader");
    GLES20.glAttachShader(program, pixelShader);
    checkGlError("glAttachShader");
    GLES20.glLinkProgram(program);
    int[] linkStatus = new int[1];
    GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
    if (linkStatus[0] != GLES20.GL_TRUE) {
        Log.e(TAG, "Could not link program: ");
        Log.e(TAG, GLES20.glGetProgramInfoLog(program));
        GLES20.glDeleteProgram(program);
        program = 0;
    }
    return program;
}

public void checkGlError(String op) {
    int error;
    while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
        Log.e(TAG, op + ": glError " + error);
        throw new RuntimeException(op + ": glError " + error);
    }
}

public static void checkLocation(int location, String label) {
    if (location < 0) {
        throw new RuntimeException("Unable to locate '" + label + "' in program");
    }
}

}

Answer 2

On API level 21 and above the decoder applies the rotation when rendering to the surface. Therefore, the transformMatrix you got from SurfaceTexture contains the rotation info, which means the way use to invert the SurfaceTexture in the example doesn't work. To correctly invert the texture, I rotation it by z axis and do x, y axis transform. Following are what I do :

chnage

st.getTransformMatrix(mSTMatrix);
if (invert) {
     mSTMatrix[5] = -mSTMatrix[5];
     mSTMatrix[13] = 1.0f - mSTMatrix[13];
}

to

st.getTransformMatrix(mMatrix);
if(invert){
    Matrix.setIdentityM(identifyMatrix, 0); 
    Matrix.translateM(identifyMatrix, 0, 1, 1, 0);      
    Matrix.rotateM(identifyMatrix, 0, 180, 0, 0, 1); 
    Matrix.multiplyMM(mSTMatrix, 0, identifyMatrix, 0, mMatrix,0);
}

where mMatrix and identifyMatrix are both

new float[16];