CODEWITHSUNDEEP
c#ffmpegopentk
7h3_C0d3r
7h3_C0d3r

Reputation: 11

How can I get decoded frames, which I sent into GPU, can i get them from GPU in rgb

I use ffmpeg.av_hwframe_transfer_data to sent decoded frames into GPU, but i can not get them again in another good format. I try to change my shaders, use av_hwframe_transfer_get_formats but it is not working!dfghkdsfiuhsgiherghoeirughoighweroigjoiwejgoiwrjgjeoijgoiewgoheroighoieqfgoihqeoigheiogieiqrhgihergh2eouirghou2rerhg My code:

{
    private static bool _readingComplete = false;
    private static bool _decodingComplete = false;
    private static readonly object _lock = new object();
    private static Queue<AVPacket> packets = new Queue<AVPacket>();
    private static readonly object _fileLock = new object();
    public static MyWindow myWindow;
    public static SKBitmap myBitmap;

    public static async unsafe Task Main(string[] args)
    {
        FFmpegBinariesHelper.RegisterFFmpegBinaries();
        DynamicallyLoadedBindings.Initialize();
        Console.WriteLine($"FFmpeg version info: {ffmpeg.av_version_info()}");

        Directory.Delete("frames", true);
        Directory.CreateDirectory("frames");

        var url = "rtsp://admin:[email protected]:554/stream0?username=admin&password=E10ADC3949BA59ABBE56E057F20";

        AVDictionary* opts = null;
        ffmpeg.av_dict_set(&opts, "-rtsp_transport", "tcp", 0);

        var vsr = new VideoStreamReader(url, opts);
        var vsd = new VideoStreamDecoder(*vsr.GetCodecParameters(), AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA);

        Task readerTask = Task.Factory.StartNew(() => ReadPackets(vsr), TaskCreationOptions.LongRunning);
        Task decoderTask = Task.Factory.StartNew(() => DecodeFrames(vsd), TaskCreationOptions.LongRunning);

        var nativeWindowSettings = new NativeWindowSettings()
        {
            ClientSize = new Vector2i(800, 600),
            Title = "My first OpenTK program!"
        };

        using (var myWindow = new MyWindow(GameWindowSettings.Default, nativeWindowSettings))
        {
            myWindow.Run();
        }
    }

    private static unsafe void ReadPackets(VideoStreamReader vsr)
    {
        while (!_readingComplete)
        {
            vsr.TryReadNextPacket(out var packet);
            lock (_lock)
            {
                packets.Enqueue(packet);
            }
        }

        _readingComplete = true;
    }

    private static unsafe void DecodeFrames(VideoStreamDecoder vsd)
    {

        Console.WriteLine($"codec name: {vsd.CodecName}");

        //var sourceSize = vsd.FrameSize;
        //var sourcePixelFormat = vsd.PixelFormat;
        //var destinationSize = sourceSize;
        //var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_RGBA;
        //using var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat);

        var frameNumber = 0;

        while (true)
        {
            AVPacket packet;
            lock (_lock)
            {
                if (packets.Count == 0)
                {
                    if (_readingComplete)
                    {
                        break;
                    }
                    else
                    {
                        continue;
                    }
                }
                packet = packets.Dequeue();
            }

            vsd.TryDecodeNextFrame(out var frame, packet);
            //var convertedFrame = vfc.Convert(frame);

            //var bitmap = new SKBitmap(convertedFrame.width, convertedFrame.height, SKColorType.Bgra8888, SKAlphaType.Opaque);
            //bitmap.InstallPixels(new SKImageInfo(convertedFrame.width, convertedFrame.height, SKColorType.Bgra8888, SKAlphaType.Opaque), (IntPtr)convertedFrame.data[0]);
            //myBitmap = bitmap;
            var bitmap = new SKBitmap(frame.width, frame.height, SKColorType.Bgra8888, SKAlphaType.Opaque);
            bitmap.InstallPixels(new SKImageInfo(frame.width, frame.height, SKColorType.Bgra8888, SKAlphaType.Opaque), (IntPtr)frame.data[0]);
            myBitmap = bitmap;
            
            Console.WriteLine($"frame: {frameNumber}");
            frameNumber++;
        }

        _decodingComplete = true;
    }

    //private static unsafe void WriteFrame(AVFrame convertedFrame, int frameNumber)
    //{
    //    var imageInfo = new SKImageInfo(convertedFrame.width, convertedFrame.height, SKColorType.Bgra8888, SKAlphaType.Opaque);
    //    using var bitmap = new SKBitmap();
    //    bitmap.InstallPixels(imageInfo, (IntPtr)convertedFrame.data[0]);

    //    string filePath;
    //    lock (_fileLock)
    //    {
    //        filePath = $"frames/frame.{frameNumber:D8}.jpg";
    //    }

    //    using var stream = File.Create(filePath);
    //    bitmap.Encode(stream, SKEncodedImageFormat.Jpeg, 90);
    //}
}
using OpenTK.Graphics.OpenGL4;
using OpenTK.Mathematics;
using OpenTK.Windowing.Common;
using OpenTK.Windowing.Desktop;
using OpenTK.Windowing.GraphicsLibraryFramework;
using SkiaSharp;

namespace OpenTKTask;

public class MyWindow : GameWindow
{
    private Shader shader;
    private int vertexBufferHandle;
    private int elementBufferHandle;
    private int vertexArrayHandle;
    private int texture;

    //float[] vertices =
    //{
    //    1.0f, 1.0f, 0.0f, 1.0f, 0.0f,
    //    1.0f, -1.0f, 0.0f, 1.0f, 1.0f,
    //    -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
    //    -1.0f, 1.0f, 0.0f, 0.0f, 1.0f
    //};

    float[] vertices =
{
    //Position         | Texture coordinates
     1.0f,  1.0f, 0.0f, 1.0f, 0.0f, // top right
     1.0f, -1.0f, 0.0f, 1.0f, 1.0f, // bottom right
    -1.0f, -1.0f, 0.0f, 0.0f, 1.0f, // bottom left
    -1.0f,  1.0f, 0.0f, 0.0f, 0.0f  // top left
};

    uint[] indices =
{
        0, 1, 3,
        1, 2, 3
    };

    float[] texCoords =
    {
        0.0f, 0.0f,
        1.0f, 0.0f,
        0.5f, 1.0f,
    };

    public MyWindow(GameWindowSettings gameWindowSettings, NativeWindowSettings nativeWindowSettings) : base(gameWindowSettings, nativeWindowSettings)
    {
        this.CenterWindow(new Vector2i(1280, 760));
    }

    protected override void OnResize(ResizeEventArgs e)
    {
        GL.Viewport(0, 0, e.Width, e.Height);
        base.OnResize(e);
    }

    protected override void OnLoad()
    {
        base.OnLoad();

        shader = new Shader("C:\\Users\\1\\Desktop\\7h3_C0d3r\\OpenTKTask\\vertexShader.vert", "C:\\Users\\1\\Desktop\\7h3_C0d3r\\OpenTKTask\\fragShader.frag");
        shader.Use();

        vertexArrayHandle = GL.GenVertexArray();
        GL.BindVertexArray(vertexArrayHandle);

        vertexBufferHandle = GL.GenBuffer();
        GL.BindBuffer(BufferTarget.ArrayBuffer, vertexBufferHandle);
        GL.BufferData(BufferTarget.ArrayBuffer, vertices.Length * sizeof(float), vertices, BufferUsageHint.StaticDraw);

        elementBufferHandle = GL.GenBuffer();
        GL.BindBuffer(BufferTarget.ElementArrayBuffer, elementBufferHandle);
        GL.BufferData(BufferTarget.ElementArrayBuffer, indices.Length * sizeof(uint), indices, BufferUsageHint.StaticDraw);

        var positionLocation = shader.GetAttribLocation("aPosition");
        GL.VertexAttribPointer(positionLocation, 3, VertexAttribPointerType.Float, false, 5 * sizeof(float), 0);
        GL.EnableVertexAttribArray(positionLocation);

        var texCoordLocation = shader.GetAttribLocation("aTexCoord");
        GL.VertexAttribPointer(texCoordLocation, 2, VertexAttribPointerType.Float, false, 5 * sizeof(float), 3 * sizeof(float));
        GL.EnableVertexAttribArray(texCoordLocation);

        texture = GL.GenTexture();
        GL.BindTexture(TextureTarget.Texture2D, texture);

        GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.Repeat);
        GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.Repeat);
        GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
        GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

        GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgba, 800, 600, 0, PixelFormat.Bgra, PixelType.UnsignedByte, IntPtr.Zero);
        GL.BindTexture(TextureTarget.Texture2D, 0);
    }

    protected override void OnUnload()
    {
        base.OnUnload();
        GL.DeleteBuffer(vertexBufferHandle);
        GL.DeleteVertexArray(vertexArrayHandle);
        GL.DeleteProgram(shader.Handle);
        GL.DeleteTexture(texture);
    }

    protected override void OnUpdateFrame(FrameEventArgs args)
    {
        base.OnUpdateFrame(args);
    }

    protected override void OnRenderFrame(FrameEventArgs args)
    {
        base.OnRenderFrame(args);

        UpdateTexture(Program.myBitmap);

        GL.Clear(ClearBufferMask.ColorBufferBit);

        GL.BindTexture(TextureTarget.Texture2D, texture);

        shader.Use();

        GL.BindVertexArray(vertexArrayHandle);

        GL.DrawElements(PrimitiveType.Triangles, indices.Length, DrawElementsType.UnsignedInt, 0);

        SwapBuffers();
    }

    public void UpdateTexture(SKBitmap bitmap)
    {
        GL.BindTexture(TextureTarget.Texture2D, texture);

        if (bitmap != null)
        {
            //byte[] pixels = bitmap.Bytes;
            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgba, bitmap.Width, bitmap.Height, 0, PixelFormat.Bgra, PixelType.UnsignedByte, 0);
        }
    }

    public SKBitmap LoadBitmap(string path)
    {
        using (var stream = File.OpenRead(path))
        {
            return SKBitmap.Decode(stream);
        }
    }
}
using FFmpeg.AutoGen;
using System.Drawing;

namespace OpenTKTask;

public sealed unsafe class VideoStreamDecoder : IDisposable
{
    private readonly AVCodecContext* _pCodecContext;
    private readonly AVPacket* _pPacket;
    private readonly AVFrame* _pFrame;
    private readonly AVFrame* _receivedFrame;
    private AVFrame* _pSwFrame;
    private AVBufferRef* _pHWDeviceCtx;

    public VideoStreamDecoder(AVCodecParameters parameter, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA)
    {
        _receivedFrame = ffmpeg.av_frame_alloc();

        AVCodec* codec = ffmpeg.avcodec_find_decoder(parameter.codec_id);
        if (codec == null)
            throw new InvalidOperationException("Codec not found.");
        _pCodecContext = ffmpeg.avcodec_alloc_context3(codec);

        ffmpeg.av_hwdevice_ctx_create(&_pCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError();

        ffmpeg.avcodec_parameters_to_context(_pCodecContext, &parameter)
            .ThrowExceptionIfError();
        ffmpeg.avcodec_open2(_pCodecContext, codec, null).ThrowExceptionIfError();

        CodecName = ffmpeg.avcodec_get_name(codec->id);
        FrameSize = new Size(_pCodecContext->width, _pCodecContext->height);
        PixelFormat = _pCodecContext->pix_fmt;

        _pFrame = ffmpeg.av_frame_alloc();
        _pPacket = ffmpeg.av_packet_alloc();
    }

    public string CodecName { get; }
    public Size FrameSize { get; }
    public AVPixelFormat PixelFormat { get; }

    public bool TryDecodeNextFrame(out AVFrame frame, AVPacket packet)
    {
        ffmpeg.av_frame_unref(_pFrame);
        ffmpeg.av_frame_unref(_receivedFrame);
        int error;

        do
        {
            ffmpeg.avcodec_send_packet(_pCodecContext, &packet).ThrowExceptionIfError();
            error = ffmpeg.avcodec_receive_frame(_pCodecContext, _pFrame);
        } while (error == ffmpeg.AVERROR(ffmpeg.EAGAIN));

        error.ThrowExceptionIfError();

        if (_pCodecContext->hw_device_ctx != null)
        {
            ffmpeg.av_hwframe_transfer_data(_receivedFrame, _pFrame, 0).ThrowExceptionIfError();
            Console.WriteLine((AVPixelFormat)171);
            frame = *_receivedFrame; // AV_PIX_FMT_NV11
            //Console.WriteLine((AVPixelFormat)frame.format);
        }
        else
            frame = *_pFrame; // AV_PIX_FMT_NV11
        //Console.WriteLine((AVPixelFormat)frame.format);
        return true;
    }

    public void Dispose()
    {
        var pFrame = _pFrame;
        ffmpeg.av_frame_free(&pFrame);

        var pCodecContext = _pCodecContext;
        ffmpeg.avcodec_free_context(&pCodecContext);

        if (_pHWDeviceCtx != null)
        {
            var pHWDeviceCtx = _pHWDeviceCtx;
            ffmpeg.av_buffer_unref(&pHWDeviceCtx);
        }

        if (_pSwFrame != null)
        {
            var pSwFrame = _pSwFrame;
            ffmpeg.av_frame_free(&pSwFrame);
        }
    }
}````

Upvotes: 0

Views: 70

Answers (0)

Related Questions