Python: случайный StaleElementReferenceException

В настоящее время я использую Selenium для простой работы с царапинами, но сталкиваюсь с проблемами. Код бросает StaleElementReferenceException в случайных точках исполнения, возвращая хорошие данные в другие (но никогда не заканчивая полностью)

Я экспериментировал с Seleniums различными ожиданиями, но безрезультатно.

Как я могу улучшить свою ключевую петлю в urlList?

Код:

import time

from selenium import webdriver

# Specifying incognito mode as you launch your browser[OPTIONAL]


option = webdriver.ChromeOptions()
option.add_argument('window-size=1600,1200')
option.add_argument("--headless")  


# Create new Instance of Chrome in incognito mode

browser = webdriver.Chrome(executable_path='/Users/mikael/Documents/StockScraper/scraper/chromedriver', chrome_options=option)
browser.implicitly_wait(2)

# Inital login

browser.get("https://www.tradingview.com/chart/LW1VvtHB/")

browser.find_element_by_xpath(".//div[contains(@class,'menu')]/div[contains(@class,'button')]").click()

browser.find_element_by_xpath(".//div[contains(@class,'menuBox')]/div[2]").click()

browser.find_element_by_xpath('.//*[@id="signin-form"]/div[1]/div[1]/input').send_keys('*****')
browser.find_element_by_xpath('//*[@id="signin-form"]/div[2]/div[1]/div[1]/input').send_keys('*****')
browser.find_element_by_xpath('//*[@id="signin-form"]/div[3]/div[2]/button').click()

time.sleep(2)

#  Populate URL-list

urlList = []
RowCount = 0
with open('urls.txt', 'r') as f:
    for i in f:
        urlList.append(i)

# print(', '.join(urlList))

data = []

# Check and append

print('Name' + ' | ' + 'Ticker' + ' | ' + 'Latest Base' + ' | ' + 'Latest Close' + ' | ' + 'Base Broken')

start_time = time.time()

for i in (urlList):
    baseBroken = 'No'
    browser.get(i)
    time.sleep(2)
    if browser.find_elements_by_xpath(".//div[contains(@class,'wizard-tooltip-btn skip')]"):
        browser.find_element_by_xpath(".//div[contains(@class,'wizard-tooltip-btn skip')]").click()
    title_element = browser.find_element_by_class_name('pane-legend-title__description')
    if browser.find_elements_by_xpath(".//div[contains(@class,'widgetbar-pages hidden')]"):
        browser.find_element_by_xpath(".//div[@data-name='data-window']").click()
    ticker = browser.find_element_by_class_name('chart-data-window-header')
    tickerClean = ticker.text.split(',')[0]
    latestClose = browser.find_element_by_xpath('/html/body/div[1]/div[3]/div/div[1]/div[1]/div[3]/div/div[2]/div[1]/div[1]/div[2]/div[4]/div[2]')
    latestBase = browser.find_element_by_xpath('/html/body/div[1]/div[3]/div/div[1]/div[1]/div[3]/div/div[2]/div[1]/div[3]/div[2]/div[9]/div[2]/span')
    latestCloseValue = float(latestClose.text.strip())
    latestBaseValue = float(latestBase.text.strip())
    if latestCloseValue <= latestBaseValue:
        baseBroken = 'Yes'
    data.append(title_element.text + ',' + tickerClean + ',' + format(latestBaseValue) + ',' + format(latestCloseValue) + ',' + baseBroken)
    print(title_element.text + ' | ' + tickerClean + ' | ' + format(latestBaseValue) + ' | ' + format(latestCloseValue) + ' | ' + baseBroken)

elapsed_time = time.time() - start_time

print(elapsed_time)

# Save to file

filename = 'data.txt'
fileout = open(filename, 'w')
for item in data:
    fileout.write("%s\n" % item)
fileout.close()

print('Done!')

# Quit Selenium

browser.close()
browser.quit()

Пример ошибки:

Iridonia:StockScraper mikael$ python3 scraper3.py
Name | Ticker | Latest Base | Latest Close | Base Broken
A1M PHARMA AB | A1M | 4.445 | 4.77 | No
AAK AB | AAK | 145.66 | 147.9 | No
ABB LTD | ABB | 203.4 | 214.8 | No
ACADEMEDIA AB | ACAD | 49.05 | 49.7 | No
ACANDO AB SER. B | ACAN_B | 32.25 | 36.4 | No
ACCONEER AB | ACCON | 23.6 | 24.0 | No
ACTIVE BIOTECH AB | ACTI | 3.89 | 3.78 | Yes
ADDNODE GROUP AB SER. B | ANOD_B | 100.0 | 102.0 | No
ADDTECH AB SER. B | ADDT_B | 201.0 | 202.0 | No
Traceback (most recent call last):
  File "scraper3.py", line 65, in <module>
    data.append(title_element.text + ',' + tickerClean + ',' + format(latestBaseValue) + ',' + format(latestCloseValue) + ',' + baseBroken)
  File "/usr/local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 76, in text
    return self._execute(Command.GET_ELEMENT_TEXT)['value']
  File "/usr/local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 628, in _execute
    return self._parent.execute(command, params)
  File "/usr/local/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 320, in execute
    self.error_handler.check_response(response)
  File "/usr/local/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=68.0.3440.106)
  (Driver info: chromedriver=2.41.578706 (5f725d1b4f0a4acbf5259df887244095596231db),platform=Mac OS X 10.12.6 x86_64)

Пример URL для загрузки:

https://www.tradingview.com/chart/?symbol=OMXSTO:A1M
https://www.tradingview.com/chart/?symbol=OMXSTO:AAK
https://www.tradingview.com/chart/?symbol=OMXSTO:ABB
https://www.tradingview.com/chart/?symbol=OMXSTO:ACAD
https://www.tradingview.com/chart/?symbol=OMXSTO:ACAN_B
https://www.tradingview.com/chart/?symbol=OMXSTO:ACCON