How to solve Dynamic DOM problem when iterating through URLs?

Question

I want to do the following:

1. Open a URL
2. Click on download
3. Open a new tab with new URL
4. Close previous tab
5. Click on download on the new URL (new tab)
6. Repeat

Here is my code (error at the bottom):

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from datetime import datetime, timedelta


# Load Chrome driver and movement.uber.com/cities website
PATH = 'C:\Program Files (x86)\chromedriver.exe'
driver = webdriver.Chrome(PATH)
driver.get('https://movement.uber.com/explore/atlanta/travel-times/query?si1074&ti=&ag=taz&dt[tpb]=ALL_DAY&dt[wd;]=1,2,3,4,5,6,7&dt[dr][sd]=2016-01-02&dt[dr][ed]=2016-01-02&cd=&sa;=&sdn=&lat.=33.7489&lng.=-84.4234622&z.=12&lang=en-US')


# Clicking on download (note that this download the dataset I need, but it shows the point I want to make)
download_button = driver.find_element_by_xpath('/html/body/div[1]/div[1]/div[1]/div[3]/div/div[3]/button')
download_button.click()


# Open new tab
driver.execute_script("window.open('https://movement.uber.com/explore/atlanta/travel-times/query?si1074&ti=&ag=taz&dt[tpb]=ALL_DAY&dt[wd;]=1,2,3,4,5,6,7&dt[dr][sd]=2016-01-02&dt[dr][ed]=2016-01-02&cd=&sa;=&sdn=&lat.=33.7489&lng.=-84.4234622&z.=12&lang=en-US')")


# Switch to the previous tab and close it (leaving us with the new above-opened tab)
tabs = driver.window_handles

if len(tabs) > 1:
    driver.switch_to.window(tabs[0])
    driver.close()
    driver.switch_to.window(tabs[1])


# Click on download AGAIN, but in the newest window (this is where I have the problem)
download_button.click()

The error I get is:

StaleElementReferenceException            Traceback (most recent call last)
<ipython-input-88-1b0df7cfcd96> in <module>
----> 1 download_button.click()

c:\users\i539797\appdata\local\programs\python\python38-32\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
     78     def click(self):
     79         """Clicks the element."""
---> 80         self._execute(Command.CLICK_ELEMENT)
     81 
     82     def submit(self):

c:\users\i539797\appdata\local\programs\python\python38-32\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
    631             params = {}
    632         params['id'] = self._id
--> 633         return self._parent.execute(command, params)
    634 
    635     def find_element(self, by=By.ID, value=None):

c:\users\i539797\appdata\local\programs\python\python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
    319         response = self.command_executor.execute(driver_command, params)
    320         if response:
--> 321             self.error_handler.check_response(response)
    322             response['value'] = self._unwrap_value(
    323                 response.get('value', None))

c:\users\i539797\appdata\local\programs\python\python38-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
    240                 alert_text = value['alert'].get('text')
    241             raise exception_class(message, screen, stacktrace, alert_text)
--> 242         raise exception_class(message, screen, stacktrace)
    243 
    244     def _value_or_default(self, obj, key, default):

StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=84.0.4147.105)

How can I go around this dynamic DOM problem and be able to click on the download button when I'm at the new tab without problems?

My ultimate objective is to iterate through URLs with different dates and click on download on each one so that I can download the dataset from each day separately. That's why I need to access different URLs.

Answer 1

Always initialize your download_button again before call .click

# Click on download AGAIN, but in the newest window (this is where I have the problem)
download_button = driver.find_element_by_xpath('/html/body/div[1]/div[1]/div[1]/div[3]/div/div[3]/button')
download_button.click()

Although your locator working, try to use this locator for download_button:

download_button = driver.find_element_by_css_selector('div.f5 button')