Scraping a complex table using Beautifulsoup and Python

Question


                    
                        Name Password Rights Bureaus FullName Email Status Logon Tries
                    

                        user1 
                                                    
                                                
                                                    

                                                
                                                    

                                                
                                                    
                                                
                                                    
                                                
                                                    

                                                                                                    
                                                    
                                                
                    

                        user2

Intended to be outputted to csv. So far I have:

headers = [c.get_text(strip=True) for c in soup.find('tr', attrs={'class':'listHeader'}).findAll('th')]

#find_all doesn't work here it just grabs one
for table in soup.find('table', attrs={'id':'MainContent_grdUsers2'}):
        try:
            column3=(table.find("option", attrs={"selected": "selected"}).get('value')) 
        except:
            continue

#this only grabs a specific cell
for table in soup.find('table', attrs={'id':'MainContent_grdUsers2'}):
        try:
            column6=(table.find("input", attrs={"id": "MainContent_grdUsers2_txtManageUsersEmail_0"}).get('value')) 
        except:
            continue

I can go in and individually grab the cells I want but there are around 100 rows of records in this table and I am finding it difficult to figure out how to grab it all at once since there isn't just text, but dropdown option values, and values. Is there a way to do this with Beautifulsoup? I tried briefly with pandas and lxml but I have never used those before.

Updated code:

headers = [c.get_text(strip=True) for c in soup.find('tr', attrs={'class':'listHeader'}).findAll('th')]
table = soup.find('table', attrs={'id':'MainContent_grdUsers2'})
data = []

for tr in table.find_all('tr')[1:] : 
    td = tr.find_all('td') 
    try : 
        data += [ 
            [ 
                td[0].getText() , 
                td[2].find('option', {'selected':'selected'}).getText(), 
                td[3].find('option', {'selected':'selected'}).getText(),
                td[4].find('input').get('value'),
                    if value is None:
                        continue
                td[5].find('input').get('value'),
                td[6].find('option', {'selected':'selected'}).getText()
            ] 
        ]
    except Exception as ex : 
        #print(ex)  ## you can uncomment this line for debugging ##
        continue

for row in data : 
    print(' '.join(row))

t.m.adam · Accepted Answer

Given the html you provided, this should work :

if soup.find('tr', attrs={'class':'listHeader'}) : 
    headers = [ 
        'none' if c is None else c.get_text(strip=True) 
        for c in soup.find('tr', attrs={'class':'listHeader'}).findAll('th') 
    ]
else : 
    headers = None

table = soup.find('table', attrs={'id':'MainContent_grdUsers2'})
data = []

for tr in table.find_all('tr')[1:] : 
    td = tr.find_all('td') 
    try : 
        data += [ 
            [ 
                td[0].getText() , 
                td[2].find('option', {'selected':'selected'}).getText(), 
                td[3].find('option', {'selected':'selected'}).getText(), 
                td[4].find('input').get('value'),  
                td[5].find('input').get('value'),
                td[6].find('option', {'selected':'selected'}).getText()
            ] 
        ]
    except Exception as ex : 
        #print(ex)  ## you can uncomment this line for debugging ##
        continue

for row in data : 
    print(' '.join(str(r) for r in row))

Output:

user1 Supervisor Medium First1 Last1 user1@company.com Inactive
user2 Supervisor Medium First2 Last2 user2@company.com Active

Scraping a complex table using Beautifulsoup and Python

Answers (1)

Related Questions