Selenium Webdriver halting with [Errno 10054]

Question

I am trying to run a Python 2.7.0 routine that uses Selenium 2.37.2 to launch Firefox 26.0 browsers and submit queries to the Google n-grams site (all on my Windows 8 machine). The program works perfectly for the first ten entries in an input file, then halts with the following traceback:

Traceback (most recent call last):
  File "C:\Python27\lib\lib-tk\Tkinter.py", line 1410, in __call__
    return self.func(*args)
  File "C:\Users\Douglas\Desktop\n-grams\n_gram_api.py", line 43, in query_n_gra
ms
    driver.get("https://books.google.com/ngrams")
  File "C:\Python27\lib\site-packages\selenium-2.37.2-py2.7.egg\selenium\webdriv
er\remote\webdriver.py", line 176, in get
    self.execute(Command.GET, {'url': url})
  File "C:\Python27\lib\site-packages\selenium-2.37.2-py2.7.egg\selenium\webdriv
er\remote\webdriver.py", line 162, in execute
    response = self.command_executor.execute(driver_command, params)
  File "C:\Python27\lib\site-packages\selenium-2.37.2-py2.7.egg\selenium\webdriv
er\remote\remote_connection.py", line 355, in execute
    return self._request(url, method=command_info[0], data=data)
  File "C:\Python27\lib\site-packages\selenium-2.37.2-py2.7.egg\selenium\webdriv
er\remote\remote_connection.py", line 402, in _request
    response = opener.open(request)
  File "C:\Python27\lib\urllib2.py", line 391, in open
    response = self._open(req, data)
  File "C:\Python27\lib\urllib2.py", line 409, in _open
    '_open', req)
  File "C:\Python27\lib\urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "C:\Python27\lib\urllib2.py", line 1173, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "C:\Python27\lib\urllib2.py", line 1148, in do_open
    raise URLError(err)
URLError: <urlopen error [Errno 10054] An existing connection was forcibly close
d by the remote host>

I found a number of informative sites that discuss the error message, but I haven't been able to figure out why my own process is halting after ten interations through the for loop. Here is the code that I'm running (sorry it's a bit long, I didn't want to trim it, in case the culprit is hidden in the GUI):

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from Tkinter import *
import Tkinter as tk
from tkFileDialog import askopenfilename
import time

#out
out = open("n_grams_outfile.txt", "w")
out.write("search string" + "\t" + "pub year" + "\t" + "frequency" + "\n")

#create a function that will return the filepath for a file provided by the user
user_defined_filepath = {}
def selectfile():
    user_defined_filepath['filename'] = askopenfilename(filetypes=[("Text","*.txt")]) # user_defined_filepath['filename'] may now be accessed in the global scope.

#create function we'll call when start button is pressed
def query_n_grams(event = "<Button>"):

    #create binary switch we'll use to only start new browser in first pass. Set default to true
    first_pass = 1

    #identify the input file
    inputfile = user_defined_filepath['filename']
    readinputfile = open(inputfile).read()
    stringinputfile = str(readinputfile)

    #assume input file = tsv. Left hand column = string of len <= 6; right hand column = pub year of text
    split_by_row = stringinputfile.split("\n")
    for row in split_by_row: 
        search_terms = row.split("\t")[0]
        actual_pub_year = row.split("\t")[1]
        pub_year_minus_five = int(actual_pub_year) - 5
        pub_year_plus_five = int(actual_pub_year) + 5        

        #you now have terms and pub yaer. Fire up webdriver and ride, cowboy
        if first_pass == 1:
            driver = webdriver.Firefox()
            first_pass = 0

        #otherwise, use extant driver
        driver.implicitly_wait(10)
        driver.get("https://books.google.com/ngrams")
        driver.implicitly_wait(10)

        #send keys
        driver.implicitly_wait(10)
        keyword = driver.find_element_by_class_name("query")
        driver.implicitly_wait(10)
        keyword.clear()
        driver.implicitly_wait(10)
        keyword.send_keys(str(search_terms))
        driver.implicitly_wait(10)

        #find start year
        driver.implicitly_wait(10)
        start_year = driver.find_element_by_name("year_start")
        driver.implicitly_wait(10)
        start_year.clear()
        driver.implicitly_wait(10)
        start_year.send_keys(str(pub_year_minus_five))
        driver.implicitly_wait(10)

        #find end year
        driver.implicitly_wait(10)
        end_year = driver.find_element_by_name("year_end")
        driver.implicitly_wait(10)
        end_year.clear()
        driver.implicitly_wait(10)
        end_year.send_keys(pub_year_plus_five)
        driver.implicitly_wait(10)

        #click enter
        driver.implicitly_wait(10)
        submit_button = driver.find_element_by_class_name("kd_submit")
        driver.implicitly_wait(10)
        submit_button.click()
        driver.implicitly_wait(10)

        #grab html
        driver.implicitly_wait(10)
        html = driver.page_source
        driver.implicitly_wait(10)

        #if you run a search that yields no hits, can't split the html, so use try/except
        try:

            #we want the list object that comes right after timeseries and before the comma
            desired_percent_figures = html.split('"timeseries": [')[1].split("]")[0]

            #now desired_percent_figures contains comma-separated list of percents (which we still need to convert out of mathematical notation). Convert out of mathematical notation (with e)
            percents_as_list = desired_percent_figures.split(",")

            #convert to ints
            percent_list_as_ints = [float(i) for i in percents_as_list]

            #take your list and find mean
            mean_percent = sum(percent_list_as_ints) / float(len(percent_list_as_ints))

            out.write(str(search_terms) + "\t" + str(actual_pub_year) + "\t" + str(mean_percent) + "\n")

        #you'll get IndexError if you run a query like "Hello Garrett" for which there are no entries in the database at all. (Other queries, like 'animal oeconomy' for year 1700, yields result 0, but because search string is in database elsewhere, won't throw IndexError)
        except IndexError:

            mean_percent = "0.0"

            #because we got an index error, we know that the search yielded no results. so let's type 0.0 as percent
            out.write(str(search_terms) + "\t" + str(actual_pub_year) + "\t" + str(mean_percent) + "\n")

        time.sleep(6)

#create TK frame
root = tk.Tk()
canvas = tk.Canvas(root, width=157, height=100)
canvas.pack()

#create label for tk
ngram_label = tk.Button(root, text = "Google N-Gram API", command = "", anchor = 'w', width = 14, activebackground = "#33B5E5")
ngram_label_canvas = canvas.create_window(20, 20, anchor='nw', width = 119, window=ngram_label)

#create a button that allows users to find a file for analysis
file_label = tk.Button(root, text = "Input file", command = selectfile, anchor = 'w', width = 7, activebackground = "#33B5E5")
file_label_canvas = canvas.create_window(20, 60, anchor='nw', window=file_label)

#create a start button that allows users to submit selected parameters and run the "startviewing" processes
start_label = tk.Button(root, text = "Go!", command = query_n_grams, anchor = 'w', width = 3, activebackground = "#33B5E5")
start_label_canvas = canvas.create_window(107, 60, anchor='nw', window=start_label)

root.mainloop()

Does anyone know why this script would generate the error message I posted above? I would be very grateful for any advice others can offer on this question.

Answer 1

I was facing the same issue... the problem was in new Firefox Update(from 46 to 47) this was a big mistake :)

anyway i did fix the issue this way ..

Download&Install Firefox 46: Downgrade from 47.0 to 46.0

you can do it from this link : If you have 32Bit : Click here | or if you have 64bit: Click here

Note Before installing the old Version you need to delete your current firefox .. :) Watch out its really important

That's it pretty mush you're now ready :)

if you have any issue with the urls you can find your way using this link

Have fun.

Answer 2

I invoked Firefox 23 instead of 26, and that resolved the issue.

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from Tkinter import *
import Tkinter as tk
from tkFileDialog import askopenfilename
import time
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary

#out
out = open("n_grams_outfile.txt", "w")
out.write("search string" + "\t" + "pub year" + "\t" + "frequency" + "\n")

#create a function that will return the filepath for a file provided by the user
user_defined_filepath = {}
def selectfile():
    user_defined_filepath['filename'] = askopenfilename(filetypes=[("Text","*.txt")]) # user_defined_filepath['filename'] may now be accessed in the global scope.

#create function we'll call when start button is pressed
def query_n_grams(event = "<Button>"):

    #create binary switch we'll use to only start new browser in first pass. Set default to true
    first_pass = 1

    #identify the input file
    inputfile = user_defined_filepath['filename']
    readinputfile = open(inputfile).read()
    stringinputfile = str(readinputfile)

    #assume input file = tsv. Left hand column = string of len <= 6; right hand column = pub year of text
    split_by_row = stringinputfile.split("\n")
    for row in split_by_row:

        #because the program will yelp if it reaches the end of the input file and then tries to split an empty line on "\t", wrap call in try/except
        try:
            search_terms = row.split("\t")[0]
            actual_pub_year = row.split("\t")[1]
        except IndexError:
            pass

        pub_year_minus_five = int(actual_pub_year) - 5
        pub_year_plus_five = int(actual_pub_year) + 5        

        #you now have terms and pub yaer. Fire up webdriver and ride, cowboy
        if first_pass == 1:

            binary = FirefoxBinary('C:\Text\Professional\Digital Humanities\Programming Languages\Python\Query Literature Online\LION 3.0\Firefox Versions\Firefox23\FirefoxPortable.exe')
            driver = webdriver.Firefox(firefox_binary=binary)

            first_pass = 0

        #otherwise, use extant driver
        driver.implicitly_wait(10)
        driver.get("https://books.google.com/ngrams")
        driver.refresh()
        driver.implicitly_wait(10)

        #send keys
        driver.implicitly_wait(10)
        keyword = driver.find_element_by_class_name("query")
        driver.implicitly_wait(10)
        keyword.clear()
        driver.implicitly_wait(10)
        keyword.send_keys(str(search_terms))
        driver.implicitly_wait(10)

        #find start year
        driver.implicitly_wait(10)
        start_year = driver.find_element_by_name("year_start")
        driver.implicitly_wait(10)
        start_year.clear()
        driver.implicitly_wait(10)
        start_year.send_keys(str(pub_year_minus_five))
        driver.implicitly_wait(10)

        #find end year
        driver.implicitly_wait(10)
        end_year = driver.find_element_by_name("year_end")
        driver.implicitly_wait(10)
        end_year.clear()
        driver.implicitly_wait(10)
        end_year.send_keys(pub_year_plus_five)
        driver.implicitly_wait(10)

        #click enter
        driver.implicitly_wait(10)
        submit_button = driver.find_element_by_class_name("kd_submit")
        driver.implicitly_wait(10)
        submit_button.click()
        driver.implicitly_wait(10)

        #grab html
        driver.implicitly_wait(10)
        html = driver.page_source
        driver.implicitly_wait(10)

        #if you run a search that yields no hits, can't split the html, so use try/except
        try:

            #we want the list object that comes right after timeseries and before the comma
            desired_percent_figures = html.split('"timeseries": [')[1].split("]")[0]

            #now desired_percent_figures contains comma-separated list of percents (which we still need to convert out of mathematical notation). Convert out of mathematical notation (with e)
            percents_as_list = desired_percent_figures.split(",")

            #convert to ints
            percent_list_as_ints = [float(i) for i in percents_as_list]

            #take your list and find mean
            mean_percent = sum(percent_list_as_ints) / float(len(percent_list_as_ints))

            out.write(str(search_terms) + "\t" + str(actual_pub_year) + "\t" + str(mean_percent) + "\n")

        #you'll get IndexError if you run a query like "Hello Garrett" for which there are no entries in the database at all. (Other queries, like 'animal oeconomy' for year 1700, yields result 0, but because search string is in database elsewhere, won't throw IndexError)
        except IndexError:

            mean_percent = "0.0"

            #because we got an index error, we know that the search yielded no results. so let's type 0.0 as percent
            out.write(str(search_terms) + "\t" + str(actual_pub_year) + "\t" + str(mean_percent) + "\n")

#create TK frame
root = tk.Tk()
canvas = tk.Canvas(root, width=157, height=100)
canvas.pack()

#create label for tk
ngram_label = tk.Button(root, text = "Google N-Gram API", command = "", anchor = 'w', width = 14, activebackground = "#33B5E5")
ngram_label_canvas = canvas.create_window(20, 20, anchor='nw', width = 119, window=ngram_label)

#create a button that allows users to find a file for analysis
file_label = tk.Button(root, text = "Input file", command = selectfile, anchor = 'w', width = 7, activebackground = "#33B5E5")
file_label_canvas = canvas.create_window(20, 60, anchor='nw', window=file_label)

#create a start button that allows users to submit selected parameters and run the "startviewing" processes
start_label = tk.Button(root, text = "Go!", command = query_n_grams, anchor = 'w', width = 3, activebackground = "#33B5E5")
start_label_canvas = canvas.create_window(107, 60, anchor='nw', window=start_label)

root.mainloop()