How to fix the proxy server error ERR_TUNNEL_CONNECTION_FAILED with free-proxy?

Question

I have followed a tutorial to put all required modules for a modified selenium webdriver into one single class. However, as I implemented the codes, the following errors keeps happening:

Traceback (most recent call last):
  File "c:\Users\s1982\Documents\GitHub\Preventing-Selenium-from-being-detected\master.py", line 149, in <module>
    main()
  File "c:\Users\s1982\Documents\GitHub\Preventing-Selenium-from-being-detected\master.py", line 140, in main
    driverinstance.get("https://bot.sannysoft.com")
  File "C:\Python311\Lib\site-packages\selenium\webdriver\remote\webdriver.py", line 455, in get
    self.execute(Command.GET, {"url": url})
  File "C:\Python311\Lib\site-packages\selenium\webdriver\remote\webdriver.py", line 444, in execute
    self.error_handler.check_response(response)
  File "C:\Python311\Lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 249, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: net::ERR_TUNNEL_CONNECTION_FAILED
  (Session info: chrome=109.0.5414.75)
Stacktrace:
Backtrace:
        (No symbol) [0x007F6643]
        (No symbol) [0x0078BE21]
        (No symbol) [0x0068DA9D]
        (No symbol) [0x00689E22]
        (No symbol) [0x0067FCFD]
        (No symbol) [0x00681101]
        (No symbol) [0x0067FFDD]
        (No symbol) [0x0067F3BC]
        (No symbol) [0x0067F2D8]
        (No symbol) [0x0067DC68]
        (No symbol) [0x0067E512]
        (No symbol) [0x0068F75B]
        (No symbol) [0x006F7727]
        (No symbol) [0x006DFD7C]
        (No symbol) [0x006F6B09]
        (No symbol) [0x006DFB76]
        (No symbol) [0x006B49C1]
        (No symbol) [0x006B5E5D]
        GetHandleVerifier [0x00A6A142+2497106]
        GetHandleVerifier [0x00A985D3+2686691]
        GetHandleVerifier [0x00A9BB9C+2700460]
        GetHandleVerifier [0x008A3B10+635936]
        (No symbol) [0x00794A1F]
        (No symbol) [0x0079A418]
        (No symbol) [0x0079A505]
        (No symbol) [0x007A508B]
        BaseThreadInitThunk [0x751E7D69+25]
        RtlInitializeExceptionChain [0x76F1BB9B+107]
        RtlClearBits [0x76F1BB1F+191]

The code is provided whole as below, as though I suspect the problem comes from the free-proxy class, I am unsure of the actual problem. However, I believe it is at least reproducible.

try:

    import sys
    import os
    from fp.fp import FreeProxy
    from fake_useragent import UserAgent
    from bs4 import BeautifulSoup
    from selenium import webdriver
    from selenium.webdriver import Chrome
    from selenium.webdriver.common.keys import Keys
    from selenium.webdriver.common.by import By
    from selenium.webdriver.chrome.options import Options
    from selenium.webdriver.chrome.service import Service
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.common.action_chains import ActionChains
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.common.exceptions import TimeoutException
    import time
    print('all module are loaded ')

except Exception as e:

    print("Error ->>>: {} ".format(e))


class Spoofer(object):

    def __init__(self, country_id=['US'], rand=True, anonym=True):
        self.country_id = country_id
        self.rand = rand
        self.anonym = anonym
        self.userAgent, self.ip = self.get()

    def get(self):
        ua = UserAgent()
        proxy = FreeProxy(country_id=self.country_id,
                          rand=self.rand, anonym=self.anonym).get()
        ip = proxy.split("://")[1]
        return ua.random, ip


class DriverOptions(object):

    def __init__(self):
        self.options = Options()
        self.options.add_argument('--no-sandbox')
        self.options.add_argument('--start-maximized')
        self.options.add_argument('--disable-dev-shm-usage')
        self.options.add_argument("--incognito")
        self.options.add_argument('--disable-blink-features')
        self.options.add_argument(
            '--disable-blink-features=AutomationControlled')
        self.options.add_experimental_option(
            "excludeSwitches", ["enable-automation"])
        self.options.add_experimental_option('useAutomationExtension', False)
        self.options.add_argument('disable-infobars')
        self.options.add_experimental_option(
            "excludeSwitches", ["enable-logging"])

        self.helperSpoofer = Spoofer()

        self.options.add_argument(
            'user-agent={}'.format(self.helperSpoofer.userAgent))
        self.options.add_argument('--proxy-server=%s' % self.helperSpoofer.ip)


class WebDriver(DriverOptions):

    def __init__(self, path=''):
        DriverOptions.__init__(self)
        self.driver_instance = self.get_driver()

    def get_driver(self):

        print("""
        IP:{}
        UserAgent: {}
        """.format(self.helperSpoofer.ip, self.helperSpoofer.userAgent))

        PROXY = self.helperSpoofer.ip
        webdriver.DesiredCapabilities.CHROME['proxy'] = {
            "httpProxy": PROXY,
            "ftpProxy": PROXY,
            "sslProxy": PROXY,
            "noProxy": None,
            "proxyType": "MANUAL",
            "autodetect": False,
        }
        webdriver.DesiredCapabilities.CHROME['acceptSslCerts'] = True

        os.environ['PATH'] += r"C:\Program Files (x86)"
        driver = webdriver.Chrome(service=Service(), options=self.options)
        driver.execute_script(
            "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source":
                "const newProto = navigator.__proto__;"
                "delete newProto.webdriver;"
                "navigator.__proto__ = newProto;"
        })

        return driver


def main():

    driver = WebDriver()
    driverinstance = driver.driver_instance
    driverinstance.get("https://www.expressvpn.com/what-is-my-ip")
    time.sleep(5)
    user_agent_check = driverinstance.execute_script(
        "return navigator.userAgent;")
    print(user_agent_check)
    print("done")


if __name__ == "__main__":
    main()

I have also checked this two questions on StackOverFlow, but both answers fail to solve the problem in my scenario: Error with Proxy Selenium Webdriver Python : ERR_TUNNEL_CONNECTION_FAILED and Selenium proxy server argument - unknown error: net::ERR_TUNNEL_CONNECTION_FAILED.

Thank you for all the help, and please let me know what I could have corrected in the code.

Answer 1

I have tried different options to validate the problem. After proofing by exhaustion, the issue is clarified as the problem between proxy and self.options.add_argument('--proxy-server=%s' % self.helperSpoofer.ip).

Even though removing the latter argument can solve the net::ERR_TUNNEL_CONNECTION_FAILED problem, the code itself is essential to block the tracking of actual IP Address examined in www.expressvpn.com/what-is-my-ip.

Therefore, I have drawn out a minimal reproducible code example for the next step:

from selenium import webdriver
from fp.fp import FreeProxy
import os

proxy = FreeProxy(country_id='US',
                  rand=True, anonym=True).get() # IP:PORT or HOST:PORT
ip = proxy.split("://")[1]
print(ip)

webdriver.DesiredCapabilities.CHROME['proxy'] = {
            "httpProxy": ip,
            "ftpProxy": ip,
            "sslProxy": ip,
            "noProxy": None,
            "proxyType": "MANUAL",
            "autodetect": False,
}

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=https://%s' % ip)    
chrome_options.add_experimental_option('detach', True)

chrome = webdriver.Chrome(options=chrome_options)
chrome.get("https://www.expressvpn.com/what-is-my-ip")

And even though I can still ping every single proxy server in cmd, the above code still results in either Cloudfront 403 or ERR_TUNNEL_CONNECTION_FAILED. Therefore, the problem could be further concluded as issues with proxy servers.

---

After that, I tried different methods to eliminate the problem while encountering errors not limited to net::ERR_TIMED_OUT, net::ERR_EMPTY_RESPONSE, net::ERR_CONNECTION_RESET.

Multiple StackOverflow answers have provided essential guidelines for such a problem, including Getting 403 when using Selenium to automate checkout process and Error ERR_TUNNEL_CONNECTION_FAILED Selenium. Yet, the problem hasn't been fixed, but I think it does lower the opportunities of the issue.

After reviewing all methods, it appears that manually inserting some specific proxy can overcome the issue; in my case, for example, both 54.208.22.128:8080 and 129.226.100.188:8080 did the work perfectly well.

As a result, inspired by an answer of @PDHide (unfortunately, I cannot find the exact solution), a valid way would be to search for a random proxy once, then switch to ensured well-performed proxy:

from selenium import webdriver
from selenium.common.exceptions import WebDriverException
from fp.fp import FreeProxy
import os

is_valid = False
is_failed = False

while not is_valid:
    try:
        if not is_failed:
            proxy = FreeProxy(country_id='US',
                            rand=True, anonym=True).get()
            ip = proxy.split("://")[1]
            print(ip)
        else:
            # Get a random proxy server from a stored list (for example, a txt document).
            continue

        webdriver.DesiredCapabilities.CHROME['proxy'] = {
                    "httpProxy": ip,
                    "ftpProxy": ip,
                    "sslProxy": ip,
                    "noProxy": None,
                    "proxyType": "MANUAL",
                    "autodetect": False,
        }

        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--proxy-server=https://%s' % ip) 

        chrome_options.add_argument('--disable-blink-features')
        chrome_options.add_argument('--disable-blink-features=AutomationControlled')
        chrome_options.add_argument("--disable-extensions")
        chrome_options.add_experimental_option('useAutomationExtension', False)
        chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])

        chrome_options.add_experimental_option('detach', True)

        os.environ['PATH'] += r"C:\Program Files (x86)\Chromedriver.exe"

        chrome = webdriver.Chrome(options=chrome_options)
        chrome.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
        chrome.get("https://www.expressvpn.com/what-is-my-ip")

        # Check if the pre-stored list has the valid proxy server stored, if not, store the new valid proxy server.
    except WebDriverException:
        continue

Or to just keep rotating until a valid proxy is found.

Answer 2

Here is a bare minimal version of your code. I have used webdriver_manager to initialize chrome driver.

from fp.fp import FreeProxy
from fake_useragent import UserAgent
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
import time
from webdriver_manager.chrome import ChromeDriverManager


class Spoofer(object):

    def __init__(self, country_id=['US'], rand=True, anonym=True):
        self.country_id = country_id
        self.rand = rand
        self.anonym = anonym
        self.userAgent, self.ip = self.get()

    def get(self):
        ua = UserAgent()
        proxy = FreeProxy(country_id=self.country_id, rand=self.rand, anonym=self.anonym).get()

        print(proxy)
        ip = proxy.split("://")[1]
        print(ip)
        return ua.random, ip


def main():
    ser = Service(ChromeDriverManager().install())
    options = Options()
    options.add_argument('--no-sandbox')
    options.add_argument('--start-maximized')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument("--incognito")
    options.add_argument('--disable-blink-features')
    options.add_argument('--disable-blink-features=AutomationControlled')
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)
    options.add_argument('disable-infobars')
    options.add_experimental_option("excludeSwitches", ["enable-logging"])

    helperSpoofer = Spoofer()

    webdriver.DesiredCapabilities.CHROME['proxy'] = {
        "httpProxy": helperSpoofer.ip,
        "ftpProxy": helperSpoofer.ip,
        "sslProxy": helperSpoofer.ip,
        "proxyType": "MANUAL",
    }

    webdriver.DesiredCapabilities.CHROME['acceptSslCerts'] = True

    options.add_argument("user-agent=" + helperSpoofer.userAgent)

    print(webdriver.DesiredCapabilities.CHROME)

    driver = webdriver.Chrome(options=options, service=ser)

    driver.get("https://www.expressvpn.com/what-is-my-ip")
    time.sleep(25)
    user_agent_check = driver.execute_script(
        "return navigator.userAgent;")
    print(user_agent_check)
    print("done")


if __name__ == "__main__":
    main()