How to use openCV's SphericalWarper?

Question

I have an equirectangular stitched panoramic image that I want to display with spherical projection. I was previously displaying the image as-is (with equirectangular projection) using openCV's imshow, but the top and bottom parts of the image are distorted (as with any equirectangular projection) and I want to get rid of this distortion.

I found SphericalWarper in openCV that can possibly help me do this. But I'm having some problems understanding exactly how to use warp.

For now, the section of the my code that does warping looks like this:

    Mat panorama;
    Mat K = Mat::zeros(3, 3, CV_32F);
    Mat R = Mat::eye(3, 3, CV_32F);
    detail::SphericalWarper warper = detail::SphericalWarper(1.0f);
    warper.warp(imgBgr, K, R, INTER_LINEAR, BORDER_DEFAULT,panorama);

    imshow("Display frame", panorama);
    waitKey(0);

My source image, imgBgr looks like this (not exactly mine, just an example):

Currently, the output image I get, panorama, looks like an image from an image sensor without any lens:

I guess this makes sense because currently my camera intrinsic matrix is a 3x3 matrix of all zeros. So my question is: What should the camera intrinsic matrix contain? I have camera intrinsic parameters for the cameras that took images that were stitched to produce my source equirectangular image (imgBgr), but I'm not sure if the warper needs these same parameters. I just want to view my source image with a spherical projection so that the distortion caused by equirectangular projection is not there anymore. I want the output image to be similar to what Google street-view looks like.

Answer 1

I stick in the same problem, but after some experiments got out of the hole. Intrinsics matrix looks like:

fx,  0, cx,
 0, fy, cy,
 0,  0,  1

where f(x,y) is scale factor, c(x,y) is center offset. You can play out with all of the parameters in code:

#include <iostream>
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/stitching/warpers.hpp>

using namespace std;
using namespace cv;

// Calculates rotation matrix given euler angles.
Mat eulerAnglesToRotationMatrix(Vec3f &theta)
{
    // Calculate rotation about x axis
    Mat R_x = (Mat_<float>(3,3) <<
               1,       0,              0,
               0,       cosf(theta[0]),   -sinf(theta[0]),
               0,       sinf(theta[0]),   cosf(theta[0])
               );

    // Calculate rotation about y axis
    Mat R_y = (Mat_<float>(3,3) <<
               cosf(theta[1]),    0,      sinf(theta[1]),
               0,               1,      0,
               -sinf(theta[1]),   0,      cosf(theta[1])
               );

    // Calculate rotation about z axis
    Mat R_z = (Mat_<float>(3,3) <<
               cosf(theta[2]),    -sinf(theta[2]),      0,
               sinf(theta[2]),    cosf(theta[2]),       0,
               0,               0,                  1);


    // Combined rotation matrix
    Mat R = R_z * R_y * R_x;

    return R;

}

int main(int argc, const char * argv[]) {
    // insert code here...
    std::cout << "Hello, World!\n";

    Mat origImg = imread("..path to file..");
    imshow("src", origImg);

    float scale = 100.0;
    float fx = 100, fy = 100, cx = 500, cy = 300;
    Vec3f rot = {};

    while (true) {
        cout << "•" << endl;
        cout << "Fx: " << fx << "; Fy: " << fy << endl;
        cout << "Cx: " << fx << "; Cy: " << fy << endl;
        cout << "Scale: " << scale << endl;
        cout << "Ang: " << rot << endl;

        detail::SphericalWarper wrap(scale);
        Mat K = (Mat_<float>(3,3) <<
                 fx, 0, cx,
                 0, fy, cy,
                 0, 0, 1);
        Mat R = eulerAnglesToRotationMatrix(rot);

        Mat dst;
        wrap.warp(origImg, K, R, INTER_LINEAR, BORDER_CONSTANT, dst);
        imshow("dst", dst);
        cout << dst.size() << endl;
        char c = waitKey();

        if (c == 'q') break;
        else if (c == 'a') fx += 10;
        else if (c == 'z') fx -= 10;
        else if (c == 's') fy += 10;
        else if (c == 'x') fy -= 10;
        else if (c == 'd') scale += 10;
        else if (c == 'c') scale -= 10;

        else if (c == 'f') rot[0] += 0.1;
        else if (c == 'v') rot[0] -= 0.1;
        else if (c == 'g') rot[1] += 0.1;
        else if (c == 'b') rot[1] -= 0.1;
        else if (c == 'h') rot[2] += 0.1;
        else if (c == 'n') rot[2] -= 0.1;
    }

    return 0;
}