NV_ENC_ERR_UNSUPPORTED_PARAM and NV_ENC_ERR_INVALID_PARAM while calling nvEncInitializeEncoder() in Nvidia Video Codec SDK 12.0.16

Question

Im working on a program that uses the nvidia video codec SDK, below is my entire code:

#include <iostream>

#include <stdio.h>

#include <windows.h>

#pragma comment(lib, "d3d11")
#include <d3d11.h>

#pragma comment(lib, "dxgi")
#include <dxgi1_2.h>

#pragma comment(lib, "nvencodeapi.lib")
#include "nvEncodeAPI.h"

using namespace std;

int main()
{
    
    HRESULT HR;
    NVENCSTATUS NVS;
    
    // intermediate variables for casting
    IDXGIOutput* pDisplay_old;

    IDXGIFactory1* pFactory;
    IDXGIAdapter* pGPU;
    ID3D11Device* pD3D;
    ID3D11DeviceContext* pD3DContext;
    IDXGIOutput1* pDisplay;

    // create DXGI factory
    CreateDXGIFactory1(IID_PPV_ARGS(&pFactory));

    // create D3D11 device
    D3D11CreateDevice(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL, 0, NULL, 0, D3D11_SDK_VERSION, &pD3D, NULL, &pD3DContext);

    // get GPU adapter
    pFactory->EnumAdapters(0, &pGPU);

    // get display
    pGPU->EnumOutputs(0, &pDisplay_old) != S_OK;
    pDisplay_old->QueryInterface(&pDisplay);

    // free resources
    pDisplay_old->Release();
    pGPU->Release();
    pFactory->Release();

    // retrieve list of nvenc functions
    NV_ENCODE_API_FUNCTION_LIST NvEncFunctions = { NV_ENCODE_API_FUNCTION_LIST_VER };
    NvEncodeAPICreateInstance(&NvEncFunctions);

    // start nvenc encode session
    void* pEncoder = NULL;
    NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS encode_session_params = { NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER };
    encode_session_params.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX;
    encode_session_params.device = pD3D;
    encode_session_params.apiVersion = NVENCAPI_VERSION;
    NVS = NvEncFunctions.nvEncOpenEncodeSessionEx(&encode_session_params, &pEncoder);
    if (NVS != NV_ENC_SUCCESS) {
        cerr << "nvEncOpenEncodeSessionEx " << NVS;
        return 1;
    }

    NV_ENC_RC_PARAMS codec_params3 = { NV_ENC_RC_PARAMS_VER };
    codec_params3.rateControlMode = NV_ENC_PARAMS_RC_CBR;
    codec_params3.averageBitRate = 800000;
    codec_params3.enableAQ = 1;
    codec_params3.zeroReorderDelay = 1;

    //NV_ENC_CONFIG_HEVC codec_params4 = {};

    //NV_ENC_CODEC_CONFIG codec_params2 = {};
    //codec_params2.hevcConfig = codec_params4;

    NV_ENC_CONFIG codec_params = { NV_ENC_CONFIG_VER };
    codec_params.gopLength = NVENC_INFINITE_GOPLENGTH;
    codec_params.frameIntervalP = 1;
    codec_params.rcParams = codec_params3;
    //codec_params.encodeCodecConfig = codec_params2;

    NV_ENC_INITIALIZE_PARAMS encoder_params = { NV_ENC_INITIALIZE_PARAMS_VER };
    encoder_params.encodeGUID = NV_ENC_CODEC_HEVC_GUID;
    //encoder_params.presetGUID = NV_ENC_PRESET_P7_GUID;
    //encoder_params.encodeConfig = &codec_params;
    encoder_params.encodeWidth = 1920;
    encoder_params.encodeHeight = 1080;
    encoder_params.frameRateNum = 60;
    encoder_params.frameRateDen = 1;
    encoder_params.enablePTD = 1;
    encoder_params.enableEncodeAsync = false;
    NVS = NvEncFunctions.nvEncInitializeEncoder(pEncoder, &encoder_params);
    if (NVS != NV_ENC_SUCCESS) {
        cerr << "nvEncInitializeEncoder " << NVS;
        return 1;
    }

    IDXGIOutputDuplication* pCapture;
    DXGI_OUTDUPL_DESC captureDesc;

    DXGI_OUTDUPL_FRAME_INFO frameInfo;
    IDXGIResource* pFrame;
    D3D11_TEXTURE2D_DESC textureInfo;
    ID3D11Texture2D* pTexture;

    NV_ENC_REGISTER_RESOURCE RR = { NV_ENC_REGISTER_RESOURCE_VER };
    RR.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX;
    RR.width = 1920;
    RR.height = 1080;
    RR.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB;

    NV_ENC_MAP_INPUT_RESOURCE MIR = { NV_ENC_MAP_INPUT_RESOURCE_VER };

    NV_ENC_CREATE_BITSTREAM_BUFFER CBB = {NV_ENC_CREATE_BITSTREAM_BUFFER_VER};
    NVS = NvEncFunctions.nvEncCreateBitstreamBuffer(pEncoder, &CBB);
    if (NVS != NV_ENC_SUCCESS) {
        cerr << "nvEncCreateBitstreamBuffer " << NVS;
        return 1;
    }

    NV_ENC_LOCK_BITSTREAM LB = {NV_ENC_LOCK_BITSTREAM_VER};
    LB.outputBitstream = CBB.bitstreamBuffer;
    
    NV_ENC_PIC_PARAMS PP = { NV_ENC_PIC_PARAMS_VER };
    PP.pictureStruct = NV_ENC_PIC_STRUCT_FRAME;
    PP.bufferFmt = NV_ENC_BUFFER_FORMAT_ARGB;
    PP.inputWidth = 1920;
    PP.inputHeight = 1080;
    PP.outputBitstream = CBB.bitstreamBuffer;

    do
    {
        // create capture
        do
        {
            HR = pDisplay->DuplicateOutput(pD3D, &pCapture);
            Sleep(0.1);
        } 
        while (HR != S_OK);

        pCapture->GetDesc(&captureDesc);


        if (captureDesc.ModeDesc.Width == 1920 && captureDesc.ModeDesc.Height == 1080)
        {

            uint32_t idx = 0;

            do
            {
                HR = pCapture->AcquireNextFrame(INFINITE, &frameInfo, &pFrame);
                if (HR == S_OK)
                {
                    if (frameInfo.LastPresentTime.QuadPart != 0)
                    {

                        // Query DXGI resource for D3D11 texture
                        HR = pFrame->QueryInterface(IID_PPV_ARGS(&pTexture));
                        if (HR != S_OK)
                        {
                            cerr << "Error acquiring D3D11 texture from frame!";
                            return 1;
                        }

                        // register the texture for use by nvenc
                        RR.resourceToRegister = pTexture;
                        NVS = NvEncFunctions.nvEncRegisterResource(pEncoder, &RR);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncRegisterResource " << NVS;
                            return 1;
                        }

                        // Map the texture to an buffer
                        MIR.registeredResource = RR.registeredResource;
                        NVS = NvEncFunctions.nvEncMapInputResource(pEncoder, &MIR);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncMapInputResource " << NVS;
                            return 1;
                        }

                        // Encode the buffer
                        idx++;
                        //if (idx < 60) PP.pictureType = NV_ENC_PIC_TYPE_I;
                        //else PP.pictureType = NV_ENC_PIC_TYPE_I;
                        PP.frameIdx = idx;
                        PP.inputBuffer = MIR.mappedResource;
                        NVS = NvEncFunctions.nvEncEncodePicture(pEncoder, &PP);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncEncodePicture " << NVS;
                            return 1;
                        }

                        // Write the encoded bitstream to stdout
                        NVS = NvEncFunctions.nvEncLockBitstream(pEncoder, &LB);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncLockBitstream " << NVS;
                            return 1;
                        }
                        cout << LB.bitstreamSizeInBytes << " ";
                        //fwrite(lock_bitstream.bitstreamBufferPtr, lock_bitstream.bitstreamSizeInBytes, 1, stdout);
                        NVS = NvEncFunctions.nvEncUnlockBitstream(pEncoder, LB.outputBitstream);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncUnlockBitstream " << NVS;
                            return 1;
                        }

                        // Release resources
                        NVS = NvEncFunctions.nvEncUnmapInputResource(pEncoder, RR.registeredResource);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncUnmapInputResource" << NVS;
                            return 1;
                        }
                        NVS = NvEncFunctions.nvEncUnregisterResource(pEncoder, RR.registeredResource);
                        if (NVS != NV_ENC_SUCCESS) {
                            cerr << "nvEncUnregisterResource " << NVS;
                            return 1;
                        }
                        pTexture->Release();

                    }

                    pFrame->Release();
                    HR = pCapture->ReleaseFrame();
                    if (HR != S_OK)
                    {
                        cerr << "Error releasing frame!";
                        return 1;
                    }
                }
                else if (HR == DXGI_ERROR_ACCESS_LOST) break;
                else
                {
                    cerr << "Unexpected error acquiring nest frame!!";
                    return 1;
                }
            }
            while (true);

        }
        else
        {
            do
            {
                HR = pCapture->AcquireNextFrame(INFINITE, &frameInfo, &pFrame);
                if (HR == S_OK)
                {
                    HR = pCapture->ReleaseFrame();
                    if (HR != S_OK)
                    {
                        cerr << "Error releasing frame!";
                        return 1;
                    }
                }
                else if (HR == DXGI_ERROR_ACCESS_LOST) break;
                else
                {
                    cerr << "Unexpected error acquiring nest frame!!";
                    return 1;
                }
            }
            while (true);
        }
    }
    while (true);

}

The problem area is line 87 and 88, where I have two parameters of NV_ENC_INITIALIZE_PARAMS commented. when I uncomment presetGUID, it compains NV_ENC_ERR_UNSUPPORTED_PARAM (12), and this happens for any preset guide I give it. when I uncomment encodeConfig I get NV_ENC_ERR_INVALID_PARAM (8), and when I uncomment both, it is also NV_ENC_ERR_INVALID_PARAM .

the code works perfectly otherwise. I'm using Nvidia Video Codec SDK 12.0.16 and I have no idea what's causing the errors as they're not really specific.

Update: I figured out line 87, both tuningInfo and presetGUID need to be set, but I still don't know about 88

Answer 1

I solved both problems by getting a pre-prepared parameter structure with nvEncGetEncodePresetConfigEx, then editing the parameters I see fit

NV_ENC_INITIALIZE_PARAMS IP = {};
    NV_ENC_CONFIG C = {};
    IP.encodeConfig = &C;
    IP.encodeConfig->version = NV_ENC_CONFIG_VER;
    IP.version = NV_ENC_INITIALIZE_PARAMS_VER;

    IP.encodeGUID = NV_ENC_CODEC_HEVC_GUID;
    IP.presetGUID = NV_ENC_PRESET_P7_GUID;
    IP.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY;
    IP.encodeWidth = 1920;
    IP.encodeHeight = 1080;
    IP.frameRateNum = 60;
    IP.frameRateDen = 1;
    IP.enablePTD = 1;
    IP.enableEncodeAsync = false;

    NV_ENC_PRESET_CONFIG presetConfig = { NV_ENC_PRESET_CONFIG_VER, { NV_ENC_CONFIG_VER } };
    NvEncFunctions.nvEncGetEncodePresetConfigEx(pEncoder, NV_ENC_CODEC_HEVC_GUID, NV_ENC_PRESET_P7_GUID, NV_ENC_TUNING_INFO_LOW_LATENCY, &presetConfig);
    memcpy(IP.encodeConfig, &presetConfig.presetCfg, sizeof(NV_ENC_CONFIG));

    IP.encodeConfig->frameIntervalP = 1;
    IP.encodeConfig->gopLength = NVENC_INFINITE_GOPLENGTH;
    IP.encodeConfig->rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
    IP.encodeConfig->rcParams.averageBitRate = 800000;
    IP.encodeConfig->rcParams.enableAQ = 1;
    IP.encodeConfig->rcParams.zeroReorderDelay = 1;

    NVS = NvEncFunctions.nvEncInitializeEncoder(pEncoder, &IP);
    if (NVS != NV_ENC_SUCCESS) {
        cerr << "nvEncInitializeEncoder " << NVS;
        return 1;
    }