CODEWITHSUNDEEP
pythonloopserror-handlingweb-scraping
William Bernard
William Bernard

Reputation: 357

Bypassing Loop AttributeError: 'NoneType' object has no attribute 'findAll'

import requests
from bs4 import BeautifulSoup
import csv
from urlparse import urljoin
import urllib2


base_url = 'http://www.baseball-reference.com'
data = requests.get("http://www.baseball-reference.com/players/")
soup = BeautifulSoup(data.content)
player_url = 'http://www.baseball-reference.com/players/'
game_logs = 'http://www.baseball-reference.com/players/gl.cgi?id='
years = ['2000','2001','2002','2003','2004','2005','2005','2006','2007','2008','2009','2010','2011','2012','2013','2014','2015']
url = []
for link in soup.find_all('a'):
    if link.has_attr('href'):
        base_url + link['href']
        url.append(base_url + link['href'])
sink = []
for l in url:
    if l[0:42] in player_url:
        sink.append(l)
abc = []
for aa in sink:
    if len(aa) > 48:
        abc.append(aa)
urlz = []
for ab in abc:
    data = requests.get(ab)
    soup = BeautifulSoup(data.content)
    for link in soup.find_all('a'):
        if link.has_attr('href'):
            urlz.append(base_url + link['href'])
abc = []
for aa in urlz:
    if game_logs in aa:
        abc.append(aa)
urlll = []
for ab in years:
    for ac in abc:
        if ab in ac:
            urlll.append(ac)

for j in urlll:
    response = requests.get(j)
    html = response.content
    soup = BeautifulSoup(html)
    table = soup.find('table', attrs={'id': 'batting_gamelogs'})
    list_of_rows = []
    for row in table.findAll('tr'):
        list_of_cells = []
        for cell in row.findAll('td'):
            text = cell.text.replace(' ', '').encode("utf-8")
            list_of_cells.append(text)
        list_of_rows.append(list_of_cells)
    print list_of_rows

When I loop through the urls in order to get the tables there are urls where the table does not exist. I get an error returned to me that looks like:

Traceback (most recent call last):
  File "py5.py", line 55, in <module>
    list_of_cells.append(text)
AttributeError: 'NoneType' object has no attribute 'findAll'

Is there a way to keep going through the loop even if there isn't a table?

Upvotes: 0

Views: 2670

Answers (1)

Srgrn
Srgrn

Reputation: 1825

Use try and except and handle the error

 for row in table.findAll('tr'):
        list_of_cells = []
        for cell in row.findAll('td'):
            text = cell.text.replace('&nbsp;', '').encode("utf-8")
            try:
                list_of_cells.append(text)
            except Exception, e:
                # handle exception
        list_of_rows.append(list_of_cells)

Upvotes: 1

Related Questions