Image rectification for depth perception is not correct

Question

I am struggling a lot trying to write code to rectify two images. I'm hoping to calculate depth and then pull out a point cloud. I can't calculate depth because I can't rectify my images and calculate a disparity map.

These are my functions to calibrate my cameras and rectify the stereo images

def CamIntr(image_paths, chessboard_params, square_size):
    # Calibration chessboard parameters
    #chessboard_params = [6, 9]  # Number of corners in y, x
    chessboard_params.append(square_size)  # Square size in m
    num_corners = chessboard_params[0] * chessboard_params[1]

    # Termination criteria for corner refinement
    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)

    # Prepare object points (world coordinates)
    objp = np.zeros((chessboard_params[0] * chessboard_params[1], 3), np.float32)
    objp[:, :2] = np.mgrid[0:chessboard_params[0], 0:chessboard_params[1]].T.reshape(-1, 2)*chessboard_params[2]

    # Arrays to store object points and image points from all images
    objpoints = []  # 3D points in real-world space
    imgpoints = []  # 2D points in image plane

    for path in image_paths:
        # Read the image
        image = cv.imread(path)
        gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)  # Convert to grayscale

        # Find chessboard corners
        ret, corners = cv.findChessboardCorners(gray, (chessboard_params[0], chessboard_params[1]), None)

        
        #print(f"Chessboard found in {path}")
        objpoints.append(objp)
        corners2 = cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners2)
        # Draw and display the corners
        #cv.drawChessboardCorners(image, (chessboard_params[1], chessboard_params[0]), corners2, ret)
        #cv.imshow('img', image)
        #cv.imwrite("Chessboard_draw.jpg", image) 
        #cv.waitKey(500)
      
    
    ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
    
    
    return ret, mtx, dist, rvecs, tvecs, objpoints, imgpoints 



def RectifyStereoImages(img_left, img_right, chessboardL, chessboardR):
    # Load stereo calibration data (intrinsics, extrinsics, etc.)
    
    gray_left = cv.imread(img_left,0)
    gray_right = cv.imread(img_right,0)
    
    ##CALIBRATION
    
    _, mtx_left, dist_left, rvecs_left, tvecs_left, objpoints_left, imgpoints_left = CamIntr(chessboardL, [6,9], 0.025)


    _, mtx_right, dist_right, rvecs_right, tvecs_right, objpoints_right, imgpoints_right = CamIntr(chessboardR, [6,9], 0.025)

    

   ##STEREO VISION CALIBRATION
   
    _, _, _, _, _, R, T, E, F = cv.stereoCalibrate(objpoints_left, imgpoints_left, imgpoints_right,
                                                    mtx_left, dist_left, mtx_right, dist_right,
                                                    gray_left.shape[::-1], criteria=(cv.CALIB_FIX_INTRINSIC, 30, 0.001))
    
    flags = 0
    flags |= cv.CALIB_FIX_INTRINSIC
    # Here we fix the intrinsic camara matrixes so that only Rot, Trns, Emat and Fmat are calculated.
    # Hence intrinsic parameters are the same 
    
    criteria_stereo= (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
    
    # This step is performed to transformation between the two cameras and calculate Essential and Fundamenatl matrix
    retStereo, newCameraMatrixL, distL, newCameraMatrixR, distR, rot, trans, essentialMatrix, fundamentalMatrix = cv.stereoCalibrate(objpoints_left, imgpoints_left, imgpoints_right, mtx_left, dist_left, mtx_right, dist_right, gray_left.shape[::-1], criteria_stereo, flags)
    
    ##STEREO RECTIFICATION

    rectifyScale= 1
    rectL, rectR, projMatrixL, projMatrixR, Q, roi_L, roi_R= cv.stereoRectify(newCameraMatrixL, distL, newCameraMatrixR, distR, gray_left.shape[::-1], rot, trans, rectifyScale,(0,0), alpha = 0.5)
    
    stereoMapL = cv.initUndistortRectifyMap(mtx_left, distL, rectL, projMatrixL, gray_left.shape[::-1], cv.CV_16SC2)
    stereoMapR = cv.initUndistortRectifyMap(mtx_right, distR, rectR, projMatrixR, gray_right.shape[::-1], cv.CV_16SC2)
    
    map_left_x = stereoMapL[0]
    map_left_y = stereoMapL[1]
    map_right_x = stereoMapR[0]
    map_right_y = stereoMapR[1]
    
    # Rectify the images
    img_left_rectified = cv.remap(gray_left, map_left_x, map_left_y, cv.INTER_LANCZOS4, cv.BORDER_CONSTANT, 0)
    img_right_rectified = cv.remap(gray_right, map_right_x, map_right_y, cv.INTER_LANCZOS4, cv.BORDER_CONSTANT, 0)
    
    
    return img_left_rectified, img_right_rectified

And this is how I call it all.

# Get all files in the directory
all_files = os.listdir(r"C:\Users\emmay\Desktop\Folder")
# Filter files starting with the prefix "Chessboard"
chessboard1 = [file for file in all_files if file.startswith("Chessboard1")]
chessboard2 = [file for file in all_files if file.startswith("Chessboard2")]
squaresize = 0.025

img1 = "Chessboard1_1.png"
img2 = "Chessboard2_1.png"


imgL = cv.imread(img1,0)
imgR = cv.imread(img2,0)

fig, ax = plt.subplots()
im = ax.imshow(imgL)
plt.show()
ax.set_title("Left original image")

fig, ax = plt.subplots()
im = ax.imshow(imgR)
plt.show()
ax.set_title("Right original image")

L_rect, R_rect = RectifyStereoImages(img1, img2, chessboard1, chessboard2)
#images after rectification
fig, ax = plt.subplots()
im = ax.imshow(L_rect)
plt.show()
ax.set_title("Left rectified image")

fig, ax = plt.subplots()
im = ax.imshow(R_rect)
plt.show()
ax.set_title("Right rectified image")

I have 10 chessboard images from each camera, taken simultaneously. Do I calibrate each camera individually, even though they're the same camera? I have the images separated, just incase, but I have better results with image rectification if I pool them all together to get the camera intrinsics. When I separate the images I get different camera intrinsics and distortion variables, even though they're the same camera. For ease I am trying to rectify one of my chessboard images.

These are the images I'm trying to rectify:

This is my camera matrix and distortion coefficients for the first camera

array([[553.76951265,   0.        , 359.52945636],
       [  0.        , 558.7809926 , 322.92191203],
       [  0.        ,   0.        ,   1.        ]])

array([[-5.74144340e-01,  2.21330577e+00,  2.87945868e-03,
         9.47036694e-04, -3.28003833e+00]])

for the second camera

([[643.44291723,   0.        , 297.31281198],
       [  0.        , 639.47052736, 216.32413232],
       [  0.        ,   0.        ,   1.        ]])

array([[ 0.16309873, -1.99119008, -0.02374205, -0.01189547, 10.39937883]])

And then when I try to see the rectified image this is what I see:

If I try to find the camera intrinsics with all the pictures and don't separate them by left or right camera like this (forcing the intrinsics to be identical):

_, mtx_left, dist_left, rvecs_left, tvecs_left, objpoints_left, imgpoints_left = CamIntr(chessboardL + chessboardR, [6,9], 0.025)


    _, mtx_right, dist_right, rvecs_right, tvecs_right, objpoints_right, imgpoints_right = CamIntr(chessboardR, chessboardL, [6,9], 0.025)

I get this slightly better picture, but they're too different from each other now to calculate a disparity map

Does anyone know what's going on or could help me?

Image rectification for depth perception is not correct

Answers (1)

Related Questions