Can MFVideoFormat_H264 directly support the input media type of MFVideoFormat_RGB32?

Question

• I am writing a recording program mp4(H264 + AAC) based on D3D11 + MediaFoundation. The basic process is that thread A uses D3D11 API to generate various graphics, and then copy swap chain's back buffer to the shared ID3D11Texture2D, while recording thread B acts as a consumer to copy the shared texture to the local _capture_texture2d and push it into video frame of MediaFoundation.

• I have a question here: https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-encoder requirements "The input media type must have one of the following subtypes:

  ##MFVideoFormat_I420

  ##MFVideoFormat_IYUV

  ##MFVideoFormat_NV12

  ##MFVideoFormat_YUY2

  ##MFVideoFormat_YV12"

  But in fact, in order to interoperate with GDI/D2D, I use the DXGI_FORMAT_B8G8R8A8_UNORM format for CreateSwapChain, which corresponds to MFVideoFormat_RGB32. It works nice on my Win10 and company's several other win7/10 machines, but I don’t know whether it will work normal on User real machine environment?

• So i am not sure whether it must convert format from MFVideoFormat_RGB32 to MFVideoFormat_NV12? and how to do this conversion? the user's win7/8/10 system can Built-in support the conversion? (in addition the requirement is to use hardware acceleration as much as possible).

  HRESULT InitVideoStreamHelper(IMFSinkWriter *ppWriter, DWORD *pStreamIndex, const UINT32 uiWidth, const UINT32 uiHeight, const UINT32& fps, IMFDXGIDeviceManager* dxgi_device_manager)
  {
       HRESULT hr = S_OK;
  
       // Set the output media type.
       CComPtr<IMFMediaType>   pMediaTypeOut = NULL;
       hr = MFCreateMediaTypeWrapper(&pMediaTypeOut);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeOut->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeOut->SetUINT32(MF_MT_AVG_BITRATE, 1000000);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
       RETURN_ON_FAIL(hr);
  
       hr = MFSetAttributeSize(pMediaTypeOut, MF_MT_FRAME_SIZE, uiWidth, uiHeight);
       RETURN_ON_FAIL(hr);
  
       hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_FRAME_RATE, fps, 1);
       RETURN_ON_FAIL(hr);
  
       hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
       RETURN_ON_FAIL(hr);
  
       hr = ppWriter->AddStream(pMediaTypeOut, pStreamIndex);
       RETURN_ON_FAIL(hr);
  
       // Set the input media type.
       CComPtr<IMFMediaType>   pMediaTypeIn = NULL;
       hr = MFCreateMediaTypeWrapper(&pMediaTypeIn);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32);
       RETURN_ON_FAIL(hr);
  
       hr = pMediaTypeIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
       RETURN_ON_FAIL(hr);
  
       hr = MFSetAttributeSize(pMediaTypeIn, MF_MT_FRAME_SIZE, uiWidth, uiHeight);
       RETURN_ON_FAIL(hr);
  
       hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_FRAME_RATE, fps, 1);
       RETURN_ON_FAIL(hr);
  
       hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
       RETURN_ON_FAIL(hr);
  
       hr = ppWriter->SetInputMediaType(*pStreamIndex, pMediaTypeIn, NULL);
       RETURN_ON_FAIL(hr);
  
   // Tell the sink _writer to start accepting data.
   #if 1
       hr = MFTRegisterLocalByCLSID(
           __uuidof(CColorConvertDMO),
           MFT_CATEGORY_VIDEO_PROCESSOR,
           L"",
           MFT_ENUM_FLAG_SYNCMFT,
           0,
           NULL,
           0,
           NULL
       );
   #endif
       RETURN_ON_FAIL(hr);
  
       return hr;
   }
  
   HRESULT CMF_MP4FileWriter::MakeVideoFrame(IMFSample** sample, ID3D11Texture2D* texture)
   {   
       if (!_d3d_device || !_capture_texture2d)
       {
           return E_NOTIMPL;
       }
  
       HRESULT hr = E_NOTIMPL;
  
       CComPtr<IMFMediaBuffer> media_buffer;
       if (IsWindows8OrGreater())
       {
           hr = MFCreateDXGISurfaceBufferWrapper(__uuidof(ID3D11Texture2D), _capture_texture2d, 0, FALSE, &media_buffer);
  
           CComPtr<IMF2DBuffer> twod_buffer;
           hr = media_buffer->QueryInterface(&twod_buffer);
           RETURN_ON_FAIL(hr);
  
           DWORD length = 0;
           hr = twod_buffer->GetContiguousLength(&length);
           RETURN_ON_FAIL(hr);
  
           hr = media_buffer->SetCurrentLength(length);
           RETURN_ON_FAIL(hr);
       }
       else
       {   
           CComPtr<ID3D11DeviceContext> immediate_context;
           _d3d_device->GetImmediateContext(&immediate_context);
  
           D3D11_TEXTURE2D_DESC desc;
           _capture_texture2d->GetDesc(&desc);
  
           UINT32 img_size = 0;
           GUID dst_format = MFVideoFormat_RGB32;
           hr = MFCalculateImageSize(dst_format, desc.Width, desc.Height, &img_size);
           RETURN_ON_FAIL(hr);
  
           LONG dst_stride = 0;
           hr = MFGetStrideForBitmapInfoHeader(dst_format.Data1, desc.Width, &dst_stride);
           RETURN_ON_FAIL(hr);
  
           D3D11_MAPPED_SUBRESOURCE resource;
           UINT subresource = D3D11CalcSubresource(0, 0, 0);
           hr = immediate_context->Map(_capture_texture2d, subresource, D3D11_MAP_READ, 0, &resource);
           RETURN_ON_FAIL(hr);
  
           hr = MFCreateMemoryBuffer(img_size, &media_buffer);
           if (SUCCEEDED(hr))
           {
               BYTE *dst_data = NULL;
               hr = media_buffer->Lock(&dst_data, NULL, NULL);
               if (SUCCEEDED(hr))
               {
                   if (dst_stride >= 0)
                   {
                       hr = MFCopyImage(dst_data, dst_stride, (BYTE*)resource.pData, resource.RowPitch, desc.Width * 4, desc.Height);
                   }
                   else
                   {
                       BYTE *src_ptr = (BYTE*)resource.pData;
                       BYTE *dst_ptr = dst_data - dst_stride * (desc.Height - 1);
                       size_t copy_bytes = min(resource.RowPitch, (UINT)-dst_stride);
                       for (size_t i = 0; i < desc.Height; i++)
                       {
                           memcpy(dst_ptr, src_ptr, copy_bytes);
  
                           src_ptr += resource.RowPitch;
                           dst_ptr += dst_stride; 
                       }
                   }
  
                   media_buffer->SetCurrentLength(img_size);
  
                   media_buffer->Unlock();
               }
           }
  
           immediate_context->Unmap(_capture_texture2d, subresource);
       }   
       RETURN_ON_FAIL(hr);
  
       hr = MFCreateSampleWrapper(sample);
       RETURN_ON_FAIL(hr);
  
       hr = (*sample)->AddBuffer(media_buffer);
       RETURN_ON_FAIL(hr);
  
       return hr;
   }
  
   HRESULT CMF_MP4FileWriter::WriteVideoFrame(IMFSample* sample)
   {
       if (sample && _writer)
       {
           _video_frame_count++;
  
           HRESULT hr = sample->SetSampleTime(_video_frame_count * 10000000 / _video_fps);
           RETURN_ON_FAIL(hr);
  
           hr = sample->SetSampleDuration(10000000 / _video_fps);  
           RETURN_ON_FAIL(hr);
  
           hr = _writer->WriteSample(_video_sink_stream, sample);
           RETURN_ON_FAIL(hr);
       }
  
       return S_OK;
   }
  

Answer 1

All H.264 encoders are capable of taking MFVideoFormat_NV12 input. Some of them take frames in other format. If you need to convert, Video Processor MFT can convert between formats for you.

See use example here: Setup of IMTransform Video Processor for color space conversion