How to listen for mouse click events on task bar icon for a currently running application in windows using python?

Question

I'm trying to listen for mouse click events on task bar app icons for currently running apps in windows using python. In the example below, I use python to initiate the internet explorer application. After the app initiates, I then use the FindWindow windows API function to get the window ID/handle.

Is there a way to listen for mouse clicks on the internet explorer task bar icon by the user using windows API or some other method? I need to print a message when the IE icon in the task bar is clicked.

Here is the code I am working with currently:

from win32gui import GetWindowText, GetForegroundWindow, FindWindow
import win32api, win32gui
import win32con
import sys, os

os.system('"C:\Program Files (x86)\Internet Explorer\iexplore.exe"') #open internet explorer application.. this will make the internet explorer icon appear in the taskbar..

window_id = FindWindow(0, 'Internet Explorer') #find the newly opened IE window's id/handle
print (window_id) 

class test:

    def __init__(self, **kwargs):
        super(test, self).__init__(**kwargs)
        self.oldWndProc = win32gui.SetWindowLong(window_id, win32con.GWL_WNDPROC, self.MyWndProc)
        self.msgdict = {} 
        for name in dir(win32con):
            if name.startswith("WM_"):
                value = getattr(win32con, name)
                self.msgdict[value] = name
        print (self.msgdict) #I can access all the win32 window events using this dictionary..

    #listener to track win32 window events related to the newly opened internet explorer window..
    #how do I track when the internet explorer icon is CLICKED in the task bar??
    def MyWndProc(self, hWnd, msg, wParam, lParam):
        print (self.msgdict.get(msg)) #none of the window events print here..
        return win32gui.CallWindowProc(self.oldWndProc, hWnd, msg, wParam, lParam)

t = test()

Justin CR · Accepted Answer

Thanks to Drake Wu pointing me in the right direction, I was able to use python to get UI Automation info about the element that is clicked and its parent. Using that information, I can tell that an app icon in the taskbar is clicked when the ControlType of the element clicked is 'ButtonControl' and the class name of its parent is 'MSTaskListWClass'. You can figure out which application was clicked by looking at the Automation ID of the clicked element.

You need to pip install uiautomation for this example. I had to add a few additional functions/methods to the uiautomation.py file because it was missing functions that were needed.

Here are the methods that I added to the uiautomation.py file to make it more useful for this example (see where it says NEW CODE HERE):

def ControlFromPoint(x: int, y: int) -> Control:
    """
    Call IUIAutomation ElementFromPoint x,y. May return None if mouse is over cmd's title bar icon.
    Return `Control` subclass or None.
    """
    element = _AutomationClient.instance().IUIAutomation.ElementFromPoint(ctypes.wintypes.POINT(x, y))
    return Control.CreateControlFromElement(element)

#NEW CODE HERE
def GetElementFromPoint(x: int, y: int) -> Control: #gets the element from a point (x, y)
    element = _AutomationClient.instance().IUIAutomation.ElementFromPoint(ctypes.wintypes.POINT(x, y))
    return element

#NEW CODE HERE
def GetParentElement(element): #gets the parent of an element input
    parent_element = _AutomationClient.instance().ViewWalker.GetParentElement(element)
    return parent_element

#NEW CODE HERE
def GetElementInfo(element): #gets the property information about an element
    element_info = Control.CreateControlFromElement(element)
    return element_info

I then utilized these newly added uiautomation methods in the main example below:

import sys, os
from pynput import mouse
from pynput.mouse import Controller
from pynput.mouse import Listener
import uiautomation
from uiautomation import GetElementFromPoint, GetParentElement, GetElementInfo #these are newly added methods to make this example work

mouse2 = mouse.Controller()

class test:

    last_mouse_pos_x = ''
    last_mouse_pos_y = ''

    def __init__(self, **kwargs):
        super(test, self).__init__(**kwargs)
        self.mouse_listener() #start mouse listener

    def mouse_listener(self):
        global element_clicked
        while True:
            if mouse2.position[0] != test.last_mouse_pos_x or mouse2.position[1] != test.last_mouse_pos_y:
                element_clicked = uiautomation.GetElementFromPoint(mouse2.position[0], mouse2.position[1]) #returns the element that you clicked (this has to run here or it errors)
                test.last_mouse_pos_x = mouse2.position[0]
                test.last_mouse_pos_y = mouse2.position[1]

    def on_click(x, y, button, pressed):
        if pressed:
            try:
                element_clicked_info = GetElementInfo(element_clicked) #returns info about the element you clicked
                parent_of_element_clicked = uiautomation.GetParentElement(element_clicked) #returns the parent of the element that you clicked
                parent_of_element_clicked_info = GetElementInfo(parent_of_element_clicked) #returns info about the parent of the element you clicked
                #print info about the element you clicked and its parent
                print ("ELEMENT CLICKED INFO: " + str(element_clicked_info))
                print ("PARENT ELEMENT INFO: " + str(parent_of_element_clicked_info)) 
            except:
                pass #skip over any errors

            #ANALYSIS: If ControlType of element clicked = 'ButtonControl' AND class name of its parent = 'MSTaskListWClass', then an app icon in the taskbar was clicked!

    listener = mouse.Listener(on_click=on_click)
    listener.start()

t = test()

How to listen for mouse click events on task bar icon for a currently running application in windows using python?

Answers (2)

Related Questions