CODEWITHSUNDEEP
pythonscrapyargumentsweb-crawler
Naltroc
Naltroc

Reputation: 1007

Scrapy passing response, missing one positional argument

Newish to python, coming from php. I would like to scrape some sites using Scrapy and have gone through tutorials and simple scripts well. Now writing the real deal comes this error:

Traceback (most recent call last):

File "C:\Users\Naltroc\Miniconda3\lib\site-packages\twisted\internet\defer.py", line 653, in _runCallbacks current.result = callback(current.result, *args, **kw)

File "C:\Users\Naltroc\Documents\Python Scripts\tutorial\tutorial\spiders\quotes_spider.py", line 52, in parse self.dispatchersite

TypeError: thesaurus() missing 1 required positional argument: 'response'

Scrapy automatically instantiates an object when the shell command scrapy crawl words is called.

From what I understand, self is the first parameter of any class method. When calling a class method, you do not pass self as an argument and instead send it your variable.

First this is called: 

# Scrapy automatically provides `response` to `parse()` when coming from `start_requests()`
def parse(self, response):
        site = response.meta['site']
        #same as "site = thesaurus"
        self.dispatcher[site](response)
        #same as "self.dispatcher['thesaurus'](response)

then 

def thesaurus(self, response):
        filename = 'thesaurus.txt'
        words = ''
        ul = response.css('.relevancy-block ul')
        for idx, u in enumerate(ul):
            if idx == 1: 
                break;
            words = u.css('.text::text').extract()

        self.save_words(filename, words)

In php, this should be the same as calling $this->thesaurus($response). parse is obviously sending response as a variable, but python says it is missing. Where did it go?

Full code here: 

import scrapy

class WordSpider(scrapy.Spider):
    def __init__(self, keyword = 'apprehensive'):
        self.k = keyword
    name = "words"
    # Utilities
    def make_csv(self, words):
        csv = ''
        for word in words:
            csv += word + ','
        return csv

    def save_words(self, words, fp):
        with ofpen(fp, 'w') as f:
            f.seek(0)
            f.truncate()
            csv = self.make_csv(words)
            f.write(csv)

    # site specific parsers
    def thesaurus(self, response):
        filename = 'thesaurus.txt'
        words = ''
        print("in func self is defined as ", self)
        ul = response.css('.relevancy-block ul')
        for idx, u in enumerate(ul):
            if idx == 1:
                break;
            words = u.css('.text::text').extract()
            print("words is ", words)

        self.save_words(filename, words)

    def oxford(self):
        filename = 'oxford.txt'
        words = ''

    def collins(self):
        filename = 'collins.txt'
        words = ''

    # site/function mapping
    dispatcher = {
        'thesaurus': thesaurus,
        'oxford': oxford,
        'collins': collins,
    }

    def parse(self, response):
        site = response.meta['site']
        self.dispatcher[site](response)

    def start_requests(self):
        urls = {
            'thesaurus': 'http://www.thesaurus.com/browse/%s?s=t' % self.k,
            #'collins': 'https://www.collinsdictionary.com/dictionary/english-thesaurus/%s' % self.k,
            #'oxford': 'https://en.oxforddictionaries.com/thesaurus/%s' % self.k,
        }

        for site, url in urls.items():
            print(site, url)
            yield scrapy.Request(url, meta={'site': site}, callback=self.parse)

Upvotes: 0

Views: 2613

Answers (1)

Granitosaurus
Granitosaurus

Reputation: 21436

There are a lot of tiny erorrs all around your code. I took the liberty of cleaning it up a bit to follow common python/scrapy idioms :)

import logging
import scrapy


# Utilities
# should probably use csv module here or `scrapy crawl -o` flag instead
def make_csv(words):
    csv = ''
    for word in words:
        csv += word + ','
    return csv


def save_words(words, fp):
    with open(fp, 'w') as f:
        f.seek(0)
        f.truncate()
        csv = make_csv(words)
        f.write(csv)


class WordSpider(scrapy.Spider):
    name = "words"

    def __init__(self, keyword='apprehensive', **kwargs):
        super(WordSpider, self).__init__(**kwargs)
        self.k = keyword

    def start_requests(self):
        urls = {
            'thesaurus': 'http://www.thesaurus.com/browse/%s?s=t' % self.k,
            # 'collins': 'https://www.collinsdictionary.com/dictionary/english-thesaurus/%s' % self.k,
            # 'oxford': 'https://en.oxforddictionaries.com/thesaurus/%s' % self.k,
        }

        for site, url in urls.items():
            yield scrapy.Request(url, meta={'site': site}, callback=self.parse)

    def parse(self, response):
        parser = getattr(self, response.meta['site'])  # retrieve method by name
        logging.info(f'parsing using: {parser}')
        parser(response)

    # site specific parsers
    def thesaurus(self, response):
        filename = 'thesaurus.txt'
        words = []
        print("in func self is defined as ", self)
        ul = response.css('.relevancy-block ul')
        for idx, u in enumerate(ul):
            if idx == 1:
                break
            words = u.css('.text::text').extract()
            print("words is ", words)
        save_words(filename, words)

    def oxford(self):
        filename = 'oxford.txt'
        words = ''

    def collins(self):
        filename = 'collins.txt'
        words = ''

Upvotes: 3

Related Questions