Reputation: 1
I'm creating an Augmented Reality application in Python where I use:
solvePnP
.pyrender
or some form of OpenGL rendering to overlay the full 3D model.What's working:
r_vec, t_vec
from solvePnP
successfully (no errors).cv2.projectPoints(...)
on the video frames. I see the green lines, so I know the projected 2D coordinates are correct and inside the image boundaries.What's not working:
Relevant code snippet (simplified):
def process_video_with_moving_car(
template_img_path, video_input_path, video_output_path,
camera_matrix, dist_coeffs, square_size, obj_file_path
):
sift = cv2.SIFT_create()
template_img = cv2.imread(template_img_path, cv2.IMREAD_GRAYSCALE)
template_kp, template_des = sift.detectAndCompute(template_img, None)
cap = cv2.VideoCapture(video_input_path)
if not cap.isOpened():
print("Error: Could not open video file.")
return
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(cap.get(cv2.CAP_PROP_FPS))
fourcc = cv2.VideoWriter_fourcc(*"XVID")
out = cv2.VideoWriter(video_output_path, fourcc, fps, (frame_width, frame_height))
bf = cv2.BFMatcher()
# Load 3D model
vertices_3d, faces = load_obj_as_mesh(obj_file_path, desired_size=square_size)
while True:
ret, frame = cap.read()
if not ret:
break
gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
frame_kp, frame_des = sift.detectAndCompute(gray_frame, None)
if frame_des is None or len(frame_des) < 2:
out.write(frame)
continue
matches = bf.knnMatch(template_des, frame_des, k=2)
good_matches = [m for m, n in matches if m.distance < 0.5 * n.distance]
if len(good_matches) > 10:
src_pts = np.float32([template_kp[m.queryIdx].pt for m in good_matches]).reshape(-1, 1, 2)
dst_pts = np.float32([frame_kp[m.trainIdx].pt for m in good_matches]).reshape(-1, 1, 2)
H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
if H is not None:
retval, r_vec, t_vec = cv2.solvePnP(
np.array([[x, y, 0] for x, y in src_pts[:, 0]], dtype=np.float32),
dst_pts.reshape(-1, 2),
camera_matrix,
dist_coeffs,
flags=cv2.SOLVEPNP_ITERATIVE
)
if retval:
projected_pts, _ = cv2.projectPoints(vertices_3d, r_vec, t_vec, camera_matrix, dist_coeffs)
# The wireframe appears correctly
frame = draw_wireframe(frame, projected_pts.reshape(-1, 2), faces, color=(0, 255, 0), thickness=2)
out.write(frame)
cap.release()
out.release()
cv2.destroyAllWindows()
print("Processing complete. Output saved to:", video_output_path)
# End result: no real "solid" object, only the 2D wireframe from projectPoints
Upvotes: -1
Views: 36