How to long press (Press and Hold) mouse left key using only Selenium in Python

I am trying to scrape some review data from the Walmart site using Selenium in Python, but it connects this site for human verification. After inspecting this 'Press & Hold' button, somehow when I find the element, it comes out as an [object HTMLIFrameElement], not as a web element. And the element appears randomly inside any of the iframes, among 10 iframes. It can be checked using a loop, but, ultimately we can't take any action in selenium without a web element.

Though this verification also occurs as a popup, I was trying to solve it for this page first. Somehow I located the position of this button using the div as a webelement.

actions = ActionChains(driver)
iframe = driver.find_element_by_xpath("//div[@id='px-captcha']")
frame_x = iframe.location['x']
frame_y = iframe.location['y']
actions.move_to_element(iframe).move_by_offset(frame_x-550, frame_y+70).build().perform()

if I perform a context.click() or right click, it is visible that mouse position is in the middle of the button.

Now, if I can perform long press or Press & Hold the left mouse button for a while, I guess this verification can be cleared. For this I tried to take action using click() and click_and_hold and also with the key_down methods (as pressing ctrl and enter does the same as long press) in action, but no response as these methods release the buttons, can't be long pressed. I tried

actions.move_to_element(iframe).move_by_offset(frame_x-550,frame_y+70).click_and_hold().pause(20).perform()
actions.move_to_element(iframe).move_by_offset(frame_x-550, frame_y+70).actions.key_down(Keys.CONTROL).actions.key_down(Keys.ENTER).pause(20).perform()

.....and so many ways! How can I solve it using Selenium?

Upvotes: 7

Answers (5)

lucas aisin

Reputation: 19

# -*- coding: utf-8 -*-
import os
import time
import cv2
import numpy as np
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from io import BytesIO
from PIL import Image

pixelRatio = 2
def solve_blocked(browser, retry=3):
    '''
    Solve blocked
    (Cross-domain iframe cannot get elements temporarily)
    Simulate the mouse press and hold to complete the verification
    '''
    if not retry:
        return False
    element = None
    try:
        element = WebDriverWait(browser,15).until(EC.presence_of_element_located((By.ID,'px-captcha')))
        # Wait for the px-captcha element styles to fully load
        time.sleep(0.5)
    except BaseException as e:
        print(f'px-captcha element not found')
        return
    print(f'solve blocked:{browser.current_url}, Retry {retry} remaining times')
    template = cv2.imread(os.path.join('./captcha.png'), 0)
    # Set the minimum number of feature points to match value 10
    MIN_MATCH_COUNT = 8 
    if  element:
        print(f'start press and hold')
        ActionChains(browser).click_and_hold(element).perform()
        start_time = time.time()
        while 1:
            if time.time() - start_time > 20:
                break
            x, y = element.location['x'], element.location['y']
            width, height = element.size.get('width'), element.size.get('height')                
            left = x * pixelRatio
            top = y * pixelRatio
            right = (x+width) * pixelRatio
            bottom = (y+height) * pixelRatio
            png = browser.get_screenshot_as_png() 
            im = Image.open(BytesIO(png))
            im = im.crop((left, top, right, bottom)) 
            target = cv2.cvtColor(np.asarray(im),cv2.COLOR_RGB2BGR)  
            # Initiate SIFT detector
            sift = cv2.SIFT_create()
            # find the keypoints and descriptors with SIFT
            kp1, des1 = sift.detectAndCompute(template,None)
            kp2, des2 = sift.detectAndCompute(target,None)
            # create set FLANN match
            FLANN_INDEX_KDTREE = 0
            index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
            search_params = dict(checks = 50)
            flann = cv2.FlannBasedMatcher(index_params, search_params)
            matches = flann.knnMatch(des1,des2,k=2)
            # store all the good matches as per Lowe's ratio test.
            good = []
            # Discard matches greater than 0.7
            for m,n in matches:
                if m.distance < 0.7*n.distance:
                    good.append(m)
            print( "Not enough matches are found - %d/%d" % (len(good),MIN_MATCH_COUNT))
            if len(good)>=MIN_MATCH_COUNT:
                print(f'release button')
                ActionChains(browser).release(element).perform()
                return
            time.sleep(0.5)
    time.sleep(1)
    retry -= 1
    solve_blocked(retry)

options = webdriver.ChromeOptions()
options.add_experimental_option("debuggerAddress","127.0.0.1:9222")
browser = webdriver.Chrome(options=options, executable_path='./chromedriver')
while browser.window_handles:
    browser.switch_to.window(browser.window_handles[0])
    if browser.window_handles.__len__() == 1:
        break
    browser.close()
browser.get('about:blank')
browser.get('https://www.walmart.com/blocked')
# # You need to add the following method to the middleware
solve_blocked(browser)

Upvotes: -1

Malhar Lakdawala

Reputation: 11

One can also use Chromedriver Undetected.. You can follow the below git link for setup. Quite simple to integrate https://github.com/ultrafunkamsterdam/undetected-chromedriver

import undetected_chromedriver as uc
driver = uc.Chrome()
driver.get('https://www.example.com')

Upvotes: 0

lucas aisin

Reputation: 19

@Prata Palit
'Press & Hold' button uses 10 iframes, Random one iframe is visible, other 9 ifame is hidden, The iframe has cross-domain and cannot get element with javascript. 'Press and hold' button complete the verification speed is also random. I used feature matching with FLANN.

Get a verified captcha image https://i.sstatic.net/ADzMT.png
Use selenium screen shot to get captcha element image.
Don't use OS screen shot. Because you need to compare the same screenshot.
Check whether the page contains captcha
press and hold
Loop the following operations at intervals of 0.5 seconds
Use the pre-prepared captcha to compare with the captcha on the page
until match or timeout

match captcha https://i.sstatic.net/xCqhy.jpg

def solve_blocked(self, retry=3):
    '''
    Solve blocked
    (Cross-domain iframe cannot get elements temporarily)
    Simulate the mouse press and hold to complete the verification
    '''
    if not retry:
        return False
    element = None
    try:
        element = WebDriverWait(self.browser,15).until(EC.presence_of_element_located((By.ID,'px-captcha')))
        # Wait for the px-captcha element styles to fully load
        time.sleep(0.5)
    except BaseException as e:
        self.logger.info(f'px-captcha element not found')
        return
    self.logger.info(f'solve blocked:{self.browser.current_url}, Retry {retry} remaining times')
    template = cv2.imread(os.path.join(settings.TPL_DIR, 'captcha.png'), 0)
    # Set the minimum number of feature points to match value 10
    MIN_MATCH_COUNT = 8 
    if  element:
        self.logger.info(f'start press and hold')
        ActionChains(self.browser).click_and_hold(element).perform()
        start_time = time.time()
        while 1:
            # timeout
            if time.time() - start_time > 20:
                break
            x, y = element.location['x'], element.location['y']
            width, height = element.size.get('width'), element.size.get('height')                
            left = x*self.pixelRatio
            top = y*self.pixelRatio
            right = (x+width)*self.pixelRatio
            bottom = (y+height)*self.pixelRatio
            # full screenshot
            png = self.browser.get_screenshot_as_png() 
            im = Image.open(BytesIO(png))
            # px-captcha screenshot
            im = im.crop((left, top, right, bottom)) 
            target = cv2.cvtColor(np.asarray(im),cv2.COLOR_RGB2BGR)  
            # Initiate SIFT detector
            sift = cv2.SIFT_create()
            # find the keypoints and descriptors with SIFT
            kp1, des1 = sift.detectAndCompute(template,None)
            kp2, des2 = sift.detectAndCompute(target,None)
            # create set FLANN match
            FLANN_INDEX_KDTREE = 0
            index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
            search_params = dict(checks = 50)
            flann = cv2.FlannBasedMatcher(index_params, search_params)
            matches = flann.knnMatch(des1,des2,k=2)
            # store all the good matches as per Lowe's ratio test.
            good = []
            # Discard matches greater than 0.7
            for m,n in matches:
                if m.distance < 0.7*n.distance:
                    good.append(m)
            self.logger.info( "matches are found - %d/%d" % (len(good),MIN_MATCH_COUNT))
            if len(good)>=MIN_MATCH_COUNT:
                self.logger.info(f'release button')
                ActionChains(self.browser).release(element).perform()
                return
            time.sleep(0.5)
    time.sleep(1)
    retry -= 1
    self.solve_blocked(retry)

Upvotes: 2

Zirene Nguyễn

Reputation: 56

Here is the full code to handle the Press & Hold captcha case. I have add code to automatically resize the captcha box size to click & hold in the middle of the captcha that is required to be verified.

import os
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.action_chains import ActionChains

import time
from time import sleep
from random import randint

chromedriver = "C:\Program Files\Python39\Scripts\chromedriver"
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
chromedriver = "C:\Program Files\Python39\Scripts\chromedriver"
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)

url = "{Your URL}"
driver.get(url)
sleep(randint(2,3))

element = driver.find_element_by_xpath("//div[@id='px-captcha']")
# print(len(element.text), '- Value found by method text')
action = ActionChains(driver)
click = ActionChains(driver)
frame_x = element.location['x']
frame_y = element.location['y']
# print('x: ', frame_x)
# print('y: ', frame_y)
# print('size box: ', element.size)
# print('x max click: ', frame_x + element.size['width'])
# print('y max click: ', frame_y + element.size['height'])
x_move = frame_x + element.size['width']*0.5
y_move = frame_y + element.size['height']*0.5
action.move_to_element_with_offset(element, x_move, y_move).click_and_hold().perform()
time.sleep(10)
action.release(element)
action.perform()
time.sleep(0.2)
action.release(element)

Upvotes: 2

Jacob G

Reputation: 365

Here's my make-shift solution. The key is the release after 10 seconds and click again. This is how I was able to trick the captcha into thinking I held it for just the right amount of time (in my experiments, the captcha hold-down time is randomized and 10 seconds ensures enough time to fully-complete the captcha).

element = driver.find_element_by_css_selector('#px-captcha')
action = ActionChains(driver)
click = ActionChains(driver)
action.click_and_hold(element)
action.perform()
time.sleep(10)
action.release(element)
action.perform()
time.sleep(0.2)
action.release(element)

Upvotes: 21

How to long press (Press and Hold) mouse left key using only Selenium in Python

Answers (5)

Related Questions