Mediapipe Interaction with GUI program through gesture recognition not detected

Question

I currently had a working example of a custom haar cascade object detection where the users fist would be used to navigate through the programmed GUI components with arrow shapes through the menu pages, however for an assignment I have to convert this to a mediapipe example that would detect the fist drawing circles on each index point and navigating through the menu in the same way. I'm currently stuck as to how I'm calling the gesture recognition to interact with the menu and not too sure what I need to add or remove from the program after a few iterations. Sorry if this is a really newb example I am completely new to mediapipe, python and opencv simply using them for one assignment.

# Fist hand function
def detect_fist(image):
    # Image convert to RGB
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    results = hands.process(image)

    # Are hands in the image
    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            # Check if the index and middle fingers are extended
            if (hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y <
                    hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP].y):
                return True
    return False


while camera.isOpened():
    # Main Camera
    ret, frame = camera.read()

    if ret:
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)


        # Draws circle landmarks onto the videocapture
        def draw_landmarks(image, landmarks):
            for landmark in landmarks.landmark:
                x, y = int(landmark.x * image.shape[1]), int(landmark.y * image.shape[0])
                cv2.circle(image, (x, y), 5, (0, 255, 0), -1)


        # Draw landmarks on the image
        results = hands.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                draw_landmarks(frame, hand_landmarks)

                fist_pos = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
                thumb_pos = hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]

if detect_fist(frame):
            for (fist_pos, thumb_pos) in landmarks:
                cv2.putText(frame, "Fist detected!", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
                dx = (fist_pos.x - thumb_pos.x)
                dy = (fist_pos.y - thumb_pos.y)
                if change_menu == "" or change_menu == "variable":

                    for i in range(len(menu_components)):
                        x_shape_start = menu_components[i]['startEnd'][0]
                        y_shape_start = menu_components[i]['startEnd'][1]
                        x_shape_end = menu_components[i]['startEnd'][2]
                        y_shape_end = menu_components[i]['startEnd'][3]
                        if (x_shape_start < dx < x_shape_end) and \
                                (y_shape_start < dy < y_shape_end):
                            if i < 6:
                                act = i
                            else:
                                if i == 6:
                                    act = "back"
                                elif i == 7:
                                    act = "next"
                        elif change_menu == "number":
                            for i in range(10):
                                x_shape_start = numbers_menu[i][0]
                                y_shape_start = numbers_menu[i][1]
                                x_shape_end = numbers_menu[i][2]
                                y_shape_end = numbers_menu[i][3]
                                if (x_shape_start < dx < x_shape_end) and (
                                        y_shape_start < dy < y_shape_end):
                                    # Number Acknowledgement
                                    # menusound.play(1)
                                    # menusound.wait()
                                    temp_code += str(i)
                                    change_menu = ""
                                    selected_bool = True

I will also include a pastebin link to the full code here: https://pastebin.com/PjEGhmQ5

Any guidance is greatly appreciated and please feel free to call me a newb if its something really obvious!

Thanks in advance and take care :)

I've tried to use mediapipe fist recognition to be detected by the camera and navigate through the custom made GUI, it draws the circle landmarks around the fist however does not interact with the menu items at all.