code to click on button works once, but not twice

Question

I'm trying to use selenium web driver to scrape search results from a website. After I search for something, there is a button at the bottom of the page to show more search results.

This is the code on the website:

<a id="more_results_link" class="but-g but-g-normal" onclick="return changeSuche('more_results',30,null,true);" style="width:600px; float:none; margin:20px auto 0px auto;" href="#">Weitere anzeigen (+30)</a>

And this is how I am trying to (successfully) press the button in my code:

moreElement  = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(".//*[@id='more_results_link']"))
moreElement.click()

Now, I want to press the button a second time, but now it doesn't work (I checked with Fire Path, and the XPath .//*[@id='more_results_link'] still leads to that button). Instead I get the following error message:

Traceback (most recent call last):
  File "test.py", line 56, in test_Login
    moreElement.click()
  File "/usr/local/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 75, in click
    self._execute(Command.CLICK_ELEMENT)
  File "/usr/local/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 454, in _execute
    return self._parent.execute(command, params)
  File "/usr/local/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 201, in execute
    self.error_handler.check_response(response)
  File "/usr/local/lib/python2.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 107, in check_response
    message = value["value"]["message"]
TypeError: string indices must be integers

Any idea how to solve the problem? Ideally (and more elegantly) I would like to implement a loop which clicks that button as long as it exists (if there are no more search results the button disappears).

EDIT:

Here is my full code:

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.select import Select
import unittest
import csv

class LoginTest(unittest.TestCase):

    def setUp(self):
        self.driver = webdriver.Firefox()

    def test_Login(self):
        driver = self.driver

        # enter output file
        with open('test.csv','w') as f:
            file = csv.writer(f)
            file.writerow(("bla", "blub"))

        # enter input file
        with open('NRWPLZ.csv') as csvfile:
            readCSV = csv.reader(csvfile,delimiter=',')

            for row in readCSV:
                self.driver.get("URL")
                type = "Allgemeinarzt"
                city = row[0]
                print(city)

                typeFieldID = "what"
                cityFieldID = "where"
                submitButtonXPath = "//*[@id='suche']/form/div[5]/div/input"

                typeFieldElement   = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_id(typeFieldID))
                cityFieldElement    = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_id(cityFieldID))
                submitButtonElement  = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(submitButtonXPath))

                typeFieldElement.clear()
                typeFieldElement.send_keys(type)
                cityFieldElement.clear()
                cityFieldElement.send_keys(city)
                submitButtonElement.click()

                while driver.find_element_by_xpath(".//*[@id='more_results_link']"):
                    moreElement  = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(".//*[@id='more_results_link']"))
                    moreElement.click()
                    print("ok")

                with open('test.csv','a') as f:
                    file = csv.writer(f)

                    select = Select(driver.find_element_by_id("select_sort"))
                    select.select_by_visible_text("Entfernung")
                    print("dist")

                    count = len(driver.find_elements_by_xpath(".//*[@id='ansicht-ergebnisliste']/div"))
                    print(count)

                    for i in range(1,count):

                        try:
                            print(i)
                            nameXPath = str(".//*[@id='ansicht-ergebnisliste']/div[") + str(i) + str("]/div[4]/h2/a")
                            addressXPath = str(".//*[@id='ansicht-ergebnisliste']/div[") +str(i)+ str("]/div[4]/p[1]")
                            gradeXPath = str(".//*[@id='ansicht-ergebnisliste']/div[") + str(i) + str("]/div[3]/div[2]/div[1]")
                            reviewsXPath = str(".//*[@id='ansicht-ergebnisliste']/div[") + str(i) + str("]/div[3]/div[3]/a")
                            recommendationsXPath = str(".//*[@id='ansicht-ergebnisliste']/div[")  + str(i) + str("]/div[3]/div[3]/div[1]/strong")

                            gp = str(WebDriverWait(driver, 1).until(lambda driver: driver.find_element_by_xpath(nameXPath)).text.encode("utf-8"))
                            address = str(WebDriverWait(driver, 1).until(lambda driver: driver.find_element_by_xpath(addressXPath)).text.encode("utf-8"))
                            grade = str(WebDriverWait(driver, 1).until(lambda driver: driver.find_element_by_xpath(gradeXPath)).text.encode("utf-8"))
                            reviews = str(WebDriverWait(driver, 1).until(lambda driver: driver.find_element_by_xpath(reviewsXPath)).text.encode("utf-8"))
                            recommendations = str(WebDriverWait(driver, 1).until(lambda driver: driver.find_element_by_xpath(recommendationsXPath)).text.encode("utf-8"))
                            file.writerow([''.join(gp.split()),''.join(address.split()),''.join(grade.split()),''.join(reviews.split()),''.join(recommendations.split())])

                        except:
                            pass


    def tearDown(self):
        self.driver.quit()

if __name__ == '__main__':
    unittest.main()

EDIT2:

I figured out how to press the button the second time. the trick was to wait until the first element of the second results "batch" has loaded. However, for some reason the website doesn't allow me to sort the results by distance (select the option from a dropdown box) after I loaded the next result pages. So I first have to sort the results and then load the next result pages.

In other words I wanna run the following code:

select = Select(WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_id("select_sort")))
select.select_by_visible_text("Entfernung")

moreElement  = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(".//*[@id='more_results_link']"))
moreElement.click()
WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(".//*[@id='ansicht-ergebnisliste']/div[31]/div[4]/h2/a"))
moreElement  = WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(".//*[@id='more_results_link']"))
moreElement.click()
WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_xpath(".//*[@id='ansicht-ergebnisliste']/div[61]/div[4]/h2/a"))

However this is giving me the following error message:

Error
Traceback (most recent call last):
  File "/Users/Jonas/PycharmProjects/jameda/test.py", line 54, in test_Login
    moreElement.click()
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webelement.py", line 75, in click
    self._execute(Command.CLICK_ELEMENT)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webelement.py", line 454, in _execute
    return self._parent.execute(command, params)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py", line 201, in execute
    self.error_handler.check_response(response)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/errorhandler.py", line 107, in check_response
    message = value["value"]["message"]
TypeError: string indices must be integers

any idea how to resolve this last problem?

Answer 1

Thanks for your comments everyone.

I found the solution myself in the end: I just inserted time.sleep(2) after each of the actions (e.g. selecting an option from the drop down, and clicking on the buttons) and now it's working fine.