Mouse recording and replay script does not work in video games

Question

Im a huge fan of speed runs, so I wanted to do my own one. Therefore i am trying to write myself a script which tracks the ingame mouse movement and keyboard inputs and replays them with the correct timing and order.

I use pydirectinput and pynput libraries on my python script. I use pynput mouse and keyboard listener with its on_move and on_press functions to detect inputs.

I have come so far, that i can record the mouse and keyboard inputs and it replays them correctly.

I can start a recording with pressing F1. Every input is now being saved in a list. The recording can be stopped again. Pressing F2 starts the replay. Now every element of the input-list is being executed with pydirectinput.moveTo(x,y) or keyboard.Controller().press(event_data) moving the mouse for example the same way i recorded it.

Trying my script in the fields (3d video game) i noticed a problem. recording and replaying keyboard inputs works fine with no problems, but the replay of the camera movement doesn't. In the Replay the Camera jumps randomly around and doesn't follow my previous recorded inputs at all. Anyone an idea why this is?

I tried to tackle this problems with some workarounds. I guessed the problem lies in the mouse to camera translation.

1. First I wanted to fixate the cursor in the center of the screen and use relative mouse movment to control the camera. That didnt work, because with the pydirectinput.moveRel(x,y) function the camera didnt move at all.
2. Secondly i tried to create a little "window" onscreen where the mouse is bound to. The mouse cannot move outside the window and when it crosses the window boundaries it will teleport the mouse to the other side. The idea was to create an "infinite" window. Sadly this didnt work either.

Here is the Code i have right now, it may be a bit messy, but with my explanation it should be fairly understandable:

import time
import threading
import keyboard as keyb
from pynput import mouse, keyboard
import pydirectinput 


pydirectinput.PAUSE = 0.01

trigger_move = True

class InputRecorder:
    def __init__(self):
        self.current_recording = None
        self.saved_recordings = {}
        self.recorded_events = []
        self.recording = False
        self.replaying = False
        self.running = True
        self.last_key_pressed = None
        self.screen_width, self.screen_height = pydirectinput.size()
        self.center_x = self.screen_width // 2
        self.center_y = self.screen_height // 2
        self.prev_mouse_position = None

    def record_event(self, event_type, event_data):
        timestamp = time.time()
        # print(timestamp)
        self.recorded_events.append((timestamp, event_type, event_data))
        if event_type == "move":
            global trigger_move
            trigger_move = False
            
            # pydirectinput.moveTo(self.center_x, self.center_y)
            trigger_move = True
            # del self.recorded_events[-1]
        
    def start_recording(self):
        self.recording = True
        self.recorded_events = []

    def stop_recording(self):
        self.recording = False

    def save_recording(self, key_binding):
        self.current_recording = self.recorded_events
        if len(self.current_recording):
            if key_binding not in self.saved_recordings:
                self.saved_recordings[key_binding] = self.current_recording
                print(f'Recording saved with key binding: {key_binding}')
                print(self.saved_recordings)
            else:
                print('Key already gebindet')
        else:
            print('No recording to save.')

    def replay_inputs(self):
        if not self.replaying:
            self.replaying = True
            start_time = time.time()
            for timestamp, event_type, event_data in self.recorded_events:
                # Check if the 'q' key is pressed to interrupt the loop
                if keyb.is_pressed('esc'):
                    print("Loop interrupted by pressing 'esc'")
                    break
                if not self.running:
                    break  # Exit the loop if interrupted
                elapsed_time = timestamp - start_time
                if elapsed_time > 0:
                    time.sleep(elapsed_time)
                start_time = timestamp  # Update start_time for the next iteration
                
                # -------- MOUSE MOVE ---------
                if event_type == 'move':
                    # Calculate relative movement
                    stick_x = event_data[0]
                    stick_y = event_data[1]
                    # Replay relative mouse movement using pydirectinput
                    pydirectinput.moveTo(stick_x, stick_y, _pause=False)
                    print("stick x, y: ", (stick_x, stick_y))
                
                # --------- MOUSE DELTA ---------
                elif event_type == 'delta':
                    dx, dy = event_data
                    current_x, current_y = pydirectinput.position()
                    new_x, new_y = current_x + dx, current_y + dy
                    pydirectinput.moveTo(new_x, new_y, _pause=False)
                    print("stick x, y: ", (new_x, new_y))
                    
                # -------- MOUSE CLICK ---------
                elif event_type == 'click':
                    action = event_data[3]
                    button = event_data[2]
                    if action == 'press':
                        print("press mouse {}".format(button))
                        mouse.Controller().press(button)
                    elif action == 'release':
                        print("release mouse {}".format(button))
                        mouse.Controller().release(button)
                
                # -------- MOUSE SCROLL --------- 
                elif event_type == 'scroll':
                    print("scrolled")
                    mouse.Controller().scroll(event_data[2], event_data[3])
                    
                # -------- KEY PRESS --------- 
                elif event_type == 'press':
                    if event_data.startswith('Key.f1') or event_data.startswith('Key.f2'):
                        continue
                    if event_data.startswith('Key.'):
                        key = getattr(keyboard.Key, event_data.split('.')[1])
                        print("press Key {}".format(key))
                        keyboard.Controller().press(key)
                    else:
                        print("press Key {}".format(event_data))
                        keyboard.Controller().press(event_data)
                        
                # -------- KEY RELEASE ---------  
                elif event_type == 'release':
                    if event_data.startswith('Key.f1') or event_data.startswith('Key.f2'):
                        continue
                    if event_data.startswith('Key.'):
                        key = getattr(keyboard.Key, event_data.split('.')[1])
                        print("release Key {}".format(key))
                        keyboard.Controller().release(key)
                    else:
                        print("press Key {}".format(event_data))
                        keyboard.Controller().release(event_data)
            self.replaying = False

def on_press(key):
    print(key)
    # print(recorder.saved_recordings)
    if key == keyboard.Key.f1 and not recorder.recording:
        recorder.start_recording()
        print('\nRecording started. Press F1 again to stop recording.')
    elif key == keyboard.Key.f1 and recorder.recording:
        recorder.stop_recording()
        print('Recording stopped.')
        print('\nPress "F2" to replay last Recording\nPress "F1" to record new Recording\nPress "L" to save last Recording or ')
    elif key == keyboard.Key.f2:
        print('\nReplaying Recordings: ...')
        recorder.replay_inputs()
        print('\nReplay ended. ')
        print('\nPress "F2" to replay last Recording\nPress "F1" to record new Recording\nPress "L" to save last Recording or ')
    elif key == keyboard.Key.esc:
        recorder.running = False
        print('Script terminated.')
        return False  # Stop the listener
    elif keyb.is_pressed('L') and not recorder.recording:
        # Save the current recording and bind the next key pressed
        bind_key_listener.start()
        print(f'\nPress a Key you want to save last Recording to: (NOT: F1, F2, ESC, L)')
    elif key in recorder.saved_recordings and not recorder.recording:
        print(f'\nReplaying Recording saved on "{key}":')
        recorder.recorded_events = recorder.saved_recordings[key]
        recorder.replay_inputs()

def bind_key_pressed(key):
    # Stop listening for the bind keylmm
    bind_key_listener.stop()
    # Save the current recording and bind the next key pressed
    recorder.save_recording(key)

def limit_cursor_position(x, y):
    new_x = max(screen_width_min, min(x, screen_width_max))
    new_y = max(screen_height_min, min(y, screen_height_max))
    return new_x, new_y

def on_move(x, y):
    new_x, new_y = limit_cursor_position(x, y)
    pydirectinput.moveTo(new_x, new_y, _pause=False)
    if recorder.recording and trigger_move:
        recorder.record_event('move', (x, y))
        print('Pointer moved to {0}'.format((x, y)))


def on_click(x, y, button, pressed):
    if recorder.recording:
        action = 'press' if pressed else 'release'
        recorder.record_event('click', (x, y, button, action))
        print('{0} at {1}'.format('Pressed' if pressed else 'Released', (x, y)))

def on_scroll(x, y, dx, dy):
    if recorder.recording:
        recorder.record_event('scroll', (x, y, dx, dy))
        print('Scrolled {0} at {1}'.format('down' if dy < 0 else 'up', (x, y)))

def on_key_press(key):
    if key != keyboard.Key.f1 or key != keyboard.Key.f2: 
        if recorder.recording:
            try:
                recorder.record_event('press', key.char)
                print(f'Key {key.char} pressed')
            except AttributeError:
                recorder.record_event('press', str(key))
                print(f'Special key {key} pressed')

def on_key_release(key):
    if recorder.recording:
        if key != keyboard.Key.f1 or key != keyboard.Key.f2: 
            try:
                if key == keyboard.Key.esc:
                    # Stop listener
                    return False
                recorder.record_event('release', key.char)
                # print(f'Key {key} released')
            except AttributeError:
                recorder.record_event('release', str(key))
                # print(f'Special Key {key} released')



# Set your screen width and height (adjust these values based on your screen resolution)
screen_width, screen_height = pydirectinput.size()
screen_width_min = int(screen_width / 2 - 200)
screen_width_max = int(screen_width / 2 + 200)
screen_height_min = int(screen_height / 2 - 200)
screen_height_max = int(screen_height / 2 + 200)
print(screen_width, screen_height)
print(screen_width_min)
print(screen_width_max)
print(screen_height_min)
print(screen_height_max)


recorder = InputRecorder()

# Set up listeners to record input events
mouse_listener = mouse.Listener(on_move=on_move, on_click=on_click, on_scroll=on_scroll)
keyboard_listener = keyboard.Listener(on_press=on_key_press, on_release=on_key_release)

# Set up a listener to detect key presses for binding
bind_key_listener = keyboard.Listener(on_press=bind_key_pressed)

mouse_listener.start()

# Set up a separate thread for the keyboard listener
keyboard_thread = threading.Thread(target=keyboard_listener.start)
keyboard_thread.start()

# Set up a listener to detect F1 and F2 key presses
with keyboard.Listener(on_press=on_press) as key_listener:
    key_listener.join()
    keyboard_thread.join()

I hope you guys can help me out here or give an explanation on why this maybe doesn't even work the way i imagine it.