CODEWITHSUNDEEP
pythonjsonscrapy
le Minh Nguyen
le Minh Nguyen

Reputation: 281

scrapy suddenly create multiple item

Scrapy randomly return a much higher number of nested json than it is supposed to

Here is the short version of my code:

import scrapy
from scrapy import Selector
from eventSpider.items import EventspiderItem
import urllib.parse

class EventsSpider(scrapy.Spider):
    name = 'eventSpider'
    
    # base url to link to the end url we receive
    baseUrl = "http://www.olympedia.org"
    
    def start_requests(self):
        start_urls = [
            'http://www.olympedia.org/editions'
        ]
        
        for url in start_urls:
            yield scrapy.Request(url=url, callback=self.parse_urls)
            
    def parse_urls(self, response):
        """
        Go through the table of owinter olympics
        Get all the url to those olympics events
        Send the urls down to parse items to get the items of interest
        """        
        # remove the last 2 as the events haven't happened yet
        for tr in response.xpath("//table[2]//tr")[:-2]:
            url = tr.xpath('td[1]//a//@href').extract_first()
            # check for None. In this case, we elimiate the 2 events that was canelled
            if url is None:
                continue
            else:
                url_to_check = urllib.parse.urljoin(self.baseUrl, url)
                yield scrapy.Request(url=url_to_check, callback=self.parse_items)

    def parse_items(self, response):
    """
    Get the items of interest
    Extract the list of disciplines and their url
    pass the url 
    """
    item = EventspiderItem()
    selector = Selector(response)
    table1_rows = selector.xpath("//table[1]//tr")
    
    item['event_title'] = table1_rows[1].xpath('td//text()').extract_first()
    item['event_place'] = table1_rows[2].xpath('td//text()').extract_first()

    table2 = selector.xpath("//table[3]//tr")
    
    discipline_list = []
    url_list = []

    for tr in table2:            
        urls = tr.xpath('td//a//@href').extract()
        disciplines = tr.xpath('td//a//text()').extract()
        
        for url in urls:
            # # check if we get empty list
            # if not url:
            #     continue
            # else:
            url_list.append(url)   
        for discipline in disciplines:
            discipline_list.append(discipline)
    for i, url in enumerate(url_list):
        final_url = urllib.parse.urljoin(self.baseUrl, url)
        event_name = item['event_title'] + " " + discipline_list[i]
        yield scrapy.Request(url=final_url, callback=self.parse_sports, meta={'event_item': item, 'discipline': event_name})

up until here, if I simply use return item instead of using yield as in the final line, all work just fine. if I return item right now, I would get 23 nested json, exactly what I would expect.

The problem comes when I try ti yield the urls that I obtained in final_url (which has 23 as well), the number of nested json jump to 248 for some reason

def parse_sports(self, response):
    selector = Selector(response)
    item = response.meta.get('event_item')
    return item

I have no idea why this happen. Any help would be appreciated

Upvotes: 0

Views: 40

Answers (1)

Md. Fazlul Hoque
Md. Fazlul Hoque

Reputation: 16187

To select xpath after selecting table2 you have to use .// and try this.

   table2 = selector.xpath("//table[3]//tr")

   discipline_list = []
    url_list = []

    for tr in table2:            
        urls = tr.xpath('.//td//a//@href').extract()
        disciplines = tr.xpath('.//td//a//text()').extract()

Upvotes: 1

Related Questions