(Nvidia Codec SDK) nvEncOpenEncodeSessionEx() errors with unknown nvencstatus code (15)

Question

I'm trying to set up an nvenc encode session that accepts d3d11 textures using nvidia codec SDK in visual c++, Below is the beginning of my code up to that point. At line 93, nvEncOpenEncodeSessionEx() fails with nvencstatus code 15, I set my code to print out the status codes I'm aware of, and it doesn't match any of them, so I'm confused:

6
1
4
2
5
12
20
Error opening encode session! 15

#include <iostream>

#include <stdio.h>

#include <windows.h>

#pragma comment(lib, "d3d11")
#include <d3d11.h>

#pragma comment(lib, "dxgi")
#include <dxgi1_2.h>

#pragma comment(lib, "nvencodeapi.lib")
#include "nvEncodeAPI.h"

using namespace std;

int main()
{
    
    HRESULT HR;
    NVENCSTATUS NVS;

    
    // intermediate variables for casting
    IDXGIOutput* pDisplay_old;

    IDXGIFactory1* pFactory;
    IDXGIAdapter* pGPU;
    ID3D11Device* pD3D;
    IDXGIOutput1* pDisplay;

    // create DXGI factory
    HR = CreateDXGIFactory1(IID_PPV_ARGS(&pFactory));
    if (HR != S_OK)
    {
        cout << "Error creating DXGI factory!";
        return 1;
    }

    // create D3D11 device
    D3D11CreateDevice(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL, 0, NULL, 0, D3D11_SDK_VERSION, &pD3D, NULL, NULL);
    if (HR != S_OK)
    {
        cout << "Error creating D3D11 device!";
        return 1;
    }

    // get GPU adapter
    HR = pFactory->EnumAdapters(0, &pGPU);
    if (HR != S_OK)
    {
        cout << "Error querying GPU!";
        return 1;
    }

    // get display
    HR = pGPU->EnumOutputs(0, &pDisplay_old) != S_OK;
    if (HR != S_OK)
    {
        cout << "Error querying display!";
        return 1;
    }
    pDisplay_old->QueryInterface(&pDisplay);

    // retrieve list of nvenc functions
    NV_ENCODE_API_FUNCTION_LIST NvEnc;
    NV_ENCODE_API_FUNCTION_LIST* pNvEnc = &NvEnc;
    NVS = NvEncodeAPICreateInstance(pNvEnc);
    if (NVS != NV_ENC_SUCCESS) {
        cout << "Error initializing NVENC API!";
        return 1;
    }

    void* encoder = NULL;

    cout << NV_ENC_ERR_INVALID_PTR << "\n";
    cout << NV_ENC_ERR_NO_ENCODE_DEVICE << "\n";
    cout << NV_ENC_ERR_INVALID_DEVICE << "\n";
    cout << NV_ENC_ERR_UNSUPPORTED_DEVICE << "\n";
    cout << NV_ENC_ERR_DEVICE_NOT_EXIST << "\n";
    cout << NV_ENC_ERR_UNSUPPORTED_PARAM << "\n";
    cout << NV_ENC_ERR_GENERIC << "\n";

    // start nvenc encode session
    NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS encode_session_params = { 0 };
    encode_session_params.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
    encode_session_params.apiVersion = NVENCAPI_VERSION;
    encode_session_params.reserved = NULL;
    encode_session_params.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX;
    encode_session_params.device = (void*)pD3D;
    NVS = pNvEnc->nvEncOpenEncodeSessionEx(&encode_session_params, &encoder);
    if (NVS != NV_ENC_SUCCESS) {
        cout << "Error opening encode session! " << NVS;
        return 1;
    }

    NV_ENC_INITIALIZE_PARAMS initialize_params = { 0 };
    initialize_params.version = NV_ENC_INITIALIZE_PARAMS_VER;
    initialize_params.encodeGUID = NV_ENC_CODEC_HEVC_GUID;
    initialize_params.encodeWidth = 1920;
    initialize_params.encodeHeight = 1080;
    initialize_params.frameRateNum = 60;
    initialize_params.enableEncodeAsync = 0;
    initialize_params.enablePTD = 0;
    initialize_params.enableOutputInVidmem = 0;
    NVS = pNvEnc->nvEncInitializeEncoder((void*)pNvEnc, &initialize_params);
    if (NVS != NV_ENC_SUCCESS) {
        cout << "Error creating encoder!";
        return 1;
    }

I wrote this code with assistance from chatgpt, as I couldn't find any good API documentation online, but it gets really dumb and inconsistent when I ask it any details. printing out my known error codes was the main thing I tried to troubleshoot the error.

Update: I reached the official signature of nvEncOpenEncodeSessionEx() and NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS by pressing f12 on the methods, and to be safe, I defined evey possible parameter of NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS like this:

void* pEncoder = NULL;

    // start nvenc encode session
    NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS encode_session_params = {};
    encode_session_params.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
    encode_session_params.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX;
    encode_session_params.device = pD3D;
    encode_session_params.reserved = 0;
    encode_session_params.apiVersion = NVENCAPI_VERSION;
    encode_session_params.reserved1[253] = 0;
    encode_session_params.reserved2[64] = NULL;
    NVS = pNvEnc->nvEncOpenEncodeSessionEx(&encode_session_params, &pEncoder);
    if (NVS != NV_ENC_SUCCESS) {
        cout << "Error opening encode session! " << NVS;
        return 1;
    }

However, this caused this:

I believe that I havent solved the problem and have created a new one

Answer 1

Actually you should recompile nv-codec-headers.

git clone https://github.com/FFmpeg/nv-codec-headers
cd ./nv-codec-headers
git checkout n11.1.5.3
make -j4 && make install

Then you can compile your code and find the bugs disappear.

Answer 2

I realized that NvEncodeAPICreateInstance requires NV_ENCODE_API_FUNCTION_LIST.version = NV_ENCODE_API_FUNCTION_LIST_VER, which caused problems down the line.

// retrieve list of nvenc functions NV_ENCODE_API_FUNCTION_LIST NvEncFunctions = { NV_ENCODE_API_FUNCTION_LIST_VER }; NvEncodeAPICreateInstance(&NvEncFunctions);