Display an image over another image at a particular co-ordinates in openCV

Question

I am trying to display an image over another image at a particular co-ordinates. I have detected the aruco markers using the webcam and I want to display another image over the aruco marker. The aruco marker can be moved and the overlaying image should move along with the marker.

There is various draw functions and to input text into the image. I have tried image overlay and image homography.

I can obtain the co-ordinates for the corners. Is there any function to insert the image at those co-ordinates?

import cv2
import cv2.aruco as aruco
import glob

markerLength = 0.25

cap = cv2.VideoCapture(0)

criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

objp = np.zeros((6*7,3), np.float32)
objp[:,:2] = np.mgrid[0:7,0:6].T.reshape(-1,2)

objpoints = [] 
imgpoints = []

images = glob.glob('calib_images/*.jpg')

for fname in images:
    img = cv2.imread(fname)
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    ret, corners = cv2.findChessboardCorners(gray, (7,6),None)

    if ret == True:
        objpoints.append(objp)

        corners2 = cv2.cornerSubPix(gray,corners,(11,11),(-1,-1),criteria)
        imgpoints.append(corners2)
        img = cv2.drawChessboardCorners(img, (7,6), corners2,ret)


ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1],None,None)

calibrationFile = "calibrationFileName.xml"
calibrationParams = cv2.FileStorage(calibrationFile, cv2.FILE_STORAGE_READ) 
camera_matrix = calibrationParams.getNode("cameraMatrix").mat() 
dist_coeffs = calibrationParams.getNode("distCoeffs").mat() 

while(True):
    ret, frame = cap.read()

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    aruco_dict = aruco.Dictionary_get(aruco.DICT_6X6_250)
    arucoParameters =  aruco.DetectorParameters_create()

    corners, ids, rejectedImgPoints = aruco.detectMarkers(gray, aruco_dict, parameters=arucoParameters)
    if np.all(ids != None):
        rvec, tvec, _ = aruco.estimatePoseSingleMarkers(corners, markerLength, mtx, dist) 
        axis = aruco.drawAxis(frame, mtx, dist, rvec, tvec, 0.3) 
        print(ids)
        display = aruco.drawDetectedMarkers(axis, corners)
        display = np.array(display)
    else:
        display = frame

    cv2.imshow('Display',display)
    if cv2.waitKey(1) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()```

Answer 1

Picture-In-Picture

import cv2
import numpy as np

def picture_in_picture(main_image_path, overlay_image_path, img_ratio=4, border_size=5, x_margin=25, y_offset_adjust=-150):
    """
    Overlay an image onto a main image with a white border.
    
    Args:
        main_image_path (str): Path to the main image.
        overlay_image_path (str): Path to the overlay image.
        img_ratio (int): The ratio to resize the overlay image height relative to the main image.
        border_size (int): Thickness of the white border around the overlay image.
        x_margin (int): Margin from the right edge of the main image.
        y_offset_adjust (int): Adjustment for vertical offset.

    Returns:
        np.ndarray: The resulting image with the overlay applied.
    """
    # Load images
    main_image = cv2.imread(main_image_path)
    overlay_image = cv2.imread(overlay_image_path)

    if main_image is None or overlay_image is None:
        raise FileNotFoundError("One or both images not found.")

    # Resize the overlay image to 1/img_ratio of the main image height
    new_height = main_image.shape[0] // img_ratio
    new_width = int(new_height * (overlay_image.shape[1] / overlay_image.shape[0]))
    overlay_resized = cv2.resize(overlay_image, (new_width, new_height))

    # Add a white border to the overlay image
    overlay_with_border = cv2.copyMakeBorder(
        overlay_resized,
        border_size, border_size, border_size, border_size,
        cv2.BORDER_CONSTANT, value=[255, 255, 255]
    )

    # Determine overlay position
    x_offset = main_image.shape[1] - overlay_with_border.shape[1] - x_margin
    y_offset = (main_image.shape[0] // 2) - overlay_with_border.shape[0] + y_offset_adjust

    # Overlay the image
    main_image[y_offset:y_offset + overlay_with_border.shape[0], x_offset:x_offset + overlay_with_border.shape[1]] = overlay_with_border

    return main_image

# Usage example
result_image = picture_in_picture("points_img.jpg", "points_img.jpg")
cv2.imshow("Image with Picture-in-Picture", result_image)
cv2.imwrite("output_image_with_border.jpg", result_image)
cv2.waitKey(0)
cv2.destroyAllWindows()

Answer 2

@user8190410's answer works fine. Just to give a complete answer, in order to alpha blend two images with different size at a particular position, you can do the following:

alpha= 0.7
img1_mod = img1.copy()
img1_mod[:pos_x,:pos_y,:] = img1[:pos_x,:pos_y,:]*alpha + img2*(1-alpha)
cv2.imshow('Image1Mod', img1_mod)

Answer 3

Actually, I found that image homography can be used to do it. Here is the updated code.

import numpy as np
import cv2
import cv2.aruco as aruco

cap = cv2.VideoCapture(0)

while(True):
    ret, frame = cap.read()

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    aruco_dict = aruco.Dictionary_get(aruco.DICT_6X6_250)
    arucoParameters =  aruco.DetectorParameters_create()

    corners, ids, rejectedImgPoints = aruco.detectMarkers(gray, aruco_dict, parameters=arucoParameters)
    if np.all(ids != None):
        display = aruco.drawDetectedMarkers(frame, corners)
        x1 = (corners[0][0][0][0], corners[0][0][0][1]) 
        x2 = (corners[0][0][1][0], corners[0][0][1][1]) 
        x3 = (corners[0][0][2][0], corners[0][0][2][1]) 
        x4 = (corners[0][0][3][0], corners[0][0][3][1])  

        im_dst = frame 
        im_src = cv2.imread("mask.jpg")
        size = im_src.shape
        pts_dst = np.array([x1,x2,x3,x4])
        pts_src = np.array(
                       [
                        [0,0],
                        [size[1] - 1, 0],
                        [size[1] - 1, size[0] -1],
                        [0, size[0] - 1 ]
                        ],dtype=float
                       );


        h, status = cv2.findHomography(pts_src, pts_dst)
        temp = cv2.warpPerspective(im_src, h, (im_dst.shape[1],im_dst.shape[0])) 
        cv2.fillConvexPoly(im_dst, pts_dst.astype(int), 0, 16);
        im_dst = im_dst + temp  
        cv2.imshow('Display',im_dst) 
    else:
        display = frame
        cv2.imshow('Display',display)
    if cv2.waitKey(1) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()

Answer 4

To replace a part of image

import cv2
import numpy as np

img1 = cv2.imread('Desert.jpg')
img2 = cv2.imread('Penguins.jpg')

img3 = img1.copy()
# replace values at coordinates (100, 100) to (399, 399) of img3 with region of img2
img3[100:400,100:400,:] = img2[100:400,100:400,:]
cv2.imshow('Result1', img3)

To alpha blend two images

alpha = 0.5
img3 = np.uint8(img1*alpha + img2*(1-alpha))
cv2.imshow('Result2', img3)