Beautiful Soup KeyError: return self.attrs[key], but keeping the values before (and after) the error occurs

Question

I have a script, which is scraping the web. But it throws some error:

return self.attrs[key] 
KeyError: 'data-index'

Probably, this is because 'data-index' is not existing. However, I want to scrape all the data that there is available, and

• if there is no data, just place a 'NONE' (I could only write a loop, that continues if there is no data, but I couldn't store the available data);
• And after that, save the file into a data frame (which is now impossible, because of the error above).

Desired Output:

A pandas data frame (this is with dummy data):

Title Price Sponsored url asin index_asin A $12 Yes Y ABCD 1 B $14 No Y ABCD 4 B $14 Yes Y ABCD 1

import requests
from bs4 import BeautifulSoup
#from textwrap import shorten
import pandas as pd 

urls = ['https://www.amazon.com/s?k=shaver+for+men&i=beauty&ref=nb_sb_noss_2',
       "https://www.amazon.com/s?k=electric+shaver&ref=nb_sb_noss_2"]

headers={'User-Agent':'Mozilla/5.0'}

#df = pd.DataFrame(columns =['Title', 'Price', 'Sponsored', 'asin', 'index_asin'])
df = []

for url in urls:
    response = requests.get(url)
    soup = BeautifulSoup(requests.get(url, headers=headers).text, 'lxml') #lxml

    for div in soup.select('div[data-asin]'):
        title, price = div.select_one('span.a-text-normal').text, div.select_one('.a-offscreen').text if div.select_one('.a-offscreen') else '-'
        sponsored = 'Yes' if div.select_one('span:contains("Sponsored")') else 'No'
        url = response.url
        asin = div['data-asin']
        index_asin = div['data-index']  

        print('title',title)
        print('price',price)
        print('sponsored',sponsored)
        print('url',url)
        print('asin',asin)
        print('index_asin',index_asin)

        # I want to store everything in a data frame 
        #df.append(title, price, sponsored, url, asin, index_asin)

Answer 1

Use try..except block if index not present it will go to except block.

import requests
from bs4 import BeautifulSoup
#from textwrap import shorten
import pandas as pd

urls = ['https://www.amazon.com/s?k=shaver+for+men&i=beauty&ref=nb_sb_noss_2',
       "https://www.amazon.com/s?k=electric+shaver&ref=nb_sb_noss_2"]

headers={'User-Agent':'Mozilla/5.0'}

#df = pd.DataFrame(columns =['Title', 'Price', 'Sponsored', 'asin', 'index_asin'])
df = []

for url in urls:
    response = requests.get(url)
    soup = BeautifulSoup(requests.get(url, headers=headers).text, 'lxml') #lxml

    for div in soup.select('div[data-asin]'):
        title, price = div.select_one('span.a-text-normal').text, div.select_one('.a-offscreen').text if div.select_one('.a-offscreen') else '-'
        sponsored = 'Yes' if div.select_one('span:contains("Sponsored")') else 'No'
        url = response.url
        asin = div['data-asin']
        try:
          index_asin = div['data-index']
        except:
          index_asin='NAN'

        print('title',title)
        print('price',price)
        print('sponsored',sponsored)
        print('url',url)
        print('asin',asin)
        print('index_asin',index_asin)

        # I want to store everything in a data frame
        df.append({title, price, sponsored, url, asin, index_asin})


print(df)

---

EDITED df.

df=df.append({'Title':title,'Price':price,'Sponsored':sponsored,'url':url,'asin':asin,'index_asin':index_asin},ignore_index=True)

---

import requests
from bs4 import BeautifulSoup
import pandas as pd

urls = ['https://www.amazon.com/s?k=shaver+for+men&i=beauty&ref=nb_sb_noss_2',
       "https://www.amazon.com/s?k=electric+shaver&ref=nb_sb_noss_2"]

headers={'User-Agent':'Mozilla/5.0'}

df = pd.DataFrame(columns =['Title', 'Price', 'Sponsored','url', 'asin', 'index_asin'])

for url in urls:
    response = requests.get(url)
    soup = BeautifulSoup(requests.get(url, headers=headers).text, 'lxml') #lxml

    for div in soup.select('div[data-asin]'):
        title, price = div.select_one('span.a-text-normal').text, div.select_one('.a-offscreen').text if div.select_one('.a-offscreen') else '-'
        sponsored = 'Yes' if div.select_one('span:contains("Sponsored")') else 'No'
        url = response.url
        asin = div['data-asin']
        try:
          index_asin = div['data-index']
        except:
          index_asin='NAN'

        print('title',title)
        print('price',price)
        print('sponsored',sponsored)
        print('url',url)
        print('asin',asin)
        print('index_asin',index_asin)

        # I want to store everything in a data frame
        df=df.append({'Title':title,'Price':price,'Sponsored':sponsored,'url':url,'asin':asin,'index_asin':index_asin},ignore_index=True)



print(df)