CODEWITHSUNDEEP
pythonscrapy
elton
elton

Reputation: 65

Scrapy: How to pass an item between methods using meta

I'm new to scrapy and python and I'm trying to pass the item item['author'] in parse_quotes to the next parse method parse_bio

I tried the request.meta and response.meta approach as shown in the scrapy documentation but without succes. see code as per below.

import scrapy
from tutorial.items import QuotesItem

class QuotesSpider(scrapy.Spider): 

name = "quotes"

    start_urls = [
        'http://quotes.toscrape.com/login',
        #'http://quotes.toscrape.com/page/2',
    ]

    # Scraping a site with login
    # Important: Cookie settings must be "True" to keep the login session alive

    custom_settings = {'COOKIES_ENABLED': True}

    def parse(self, response):
        return scrapy.FormRequest.from_response(
            response,
            formdata={'username': 'john', 'password': 'secret'},
            callback=self.parse_quotes
            )
    
    
    def parse_quotes(self, response):
        for sel in response.css('div.quote'):
            item = QuotesItem()
            item['text'] = sel.css('span.text::text').get()
            item['author'] = sel.css('small.author::text').get()
            item['tags'] = sel.css('div.tags a.tag::text').getall()
            item['quotelink'] = sel.css('small.author ~ a[href*="goodreads.com"]::attr(href)').get()
            item['author_bio_link'] = sel.css('.author + a')
            yield item
        
            # follow the detail links @ shortcut
            # vertical crawling
            for a in item['author_bio_link']:
                yield response.follow(a, callback = self.parse_bio)
                   
    def parse_bio(self, response):
        item = QuotesItem()
        item['author_born'] = response.css('p span::text').getall()
        item['author_born'] = item['author_born'][:2]
        item['author_bio'] = response.css('div.author-description ::text').get().strip()
        yield item
             
        # follow pagination links @ shortcut
        # horizontal crawling
    
        for a in response.css('li.next a'):
            yield response.follow(a, callback = self.parse_quotes)

I expect to get item['author'] from parse_quotes passed to parse_bio

Upvotes: 4

Views: 7007

Answers (1)

vezunchik
vezunchik

Reputation: 3717

I suggest you to use meta in this way:

def parse_quotes(self, response):
    for sel in response.css('div.quote'):
        item = QuotesItem()
        item['text'] = sel.css('span.text::text').get()
        item['author'] = sel.css('small.author::text').get()
        item['tags'] = sel.css('div.tags a.tag::text').getall()
        item['quotelink'] = sel.css('small.author ~ a[href*="goodreads.com"]::attr(href)').get()
        item['author_bio_link'] = sel.css('.author + a')
        yield item

        # follow the detail links @ shortcut
        # vertical crawling
        for a in item['author_bio_link']:
            yield response.follow(a, self.parse_bio, 
                                  meta={'author': item['author']})  # <- you set it here

def parse_bio(self, response):
    item = QuotesItem()
    item['author_born'] = response.css('p span::text').getall()
    item['author_born'] = item['author_born'][:2]
    item['author_data'] = response.meta.get('author')  # <- you get it here
    item['author_bio'] = response.css('div.author-description ::text').get().strip()
    yield item

    # follow pagination links @ shortcut
    # horizontal crawling

    for a in response.css('li.next a'):
        yield response.follow(a, callback = self.parse_quotes)

Upvotes: 11

Related Questions