Scrapy Error 302 and Proxy issues

Question

I've been trying to scrape info about articles from https://academic.oup.com/ilarjournal, with the following code:

class BasicSpider(scrapy.Spider):
name = 'ILAR'

def start_requests(self):
    start_urls = ['https://academic.oup.com/ilarjournal/issue-archive']

    for url in start_urls:
        yield scrapy.Request(url=url, callback = self.parse)

def parse_item(self, response):

    item = PropertiesItem()

    item['authors'] = response.xpath("//*[contains(@class,'linked-name')]/text()").extract()
    self.log("authors %s" % item['authors'])

    articleTags = response.xpath("//*[@id='ContentTab']/div[1]/div/div//p/text()").extract()
    article = ''.join(articleTags)
    #self.log('ARTICLE TEXT IS: '+article)

    textFileTitle = response.xpath('//*[@id="ContentColumn"]/div[2]/div[1]/div/div/h1/text()').extract()
    fileTitle = ''.join(textFileTitle)
    pureFileTitle = fileTitle.replace('\n','').replace('  ','').replace('\r','')
    self.log("TEXT TITLE: " + pureFileTitle)
    item['title'] = pureFileTitle
    self.log("title %s" % item['title'])

    articleFile = str('D:/some path/' + pureFileTitle[:-2] + '.txt')

    with open (articleFile, 'wb') as newArticle:
        newArticle.write(article.encode('utf-8'))

    item['url'] = response.url
    item['project'] = self.settings.get('BOT_NAME')
    item['spider'] = self.name
    item['date'] = datetime.datetime.now()

    return item

def parse(self,response):
    #Get the year and issue URLs and yield Requests
    year_selector = response.xpath('//*[contains(@class,"IssueYear")]//@href')

    for url in year_selector.extract():
        if not year_selector.select('//*[contains(@class,"society-logo-block")]'):
            yield Request((urljoin(response.url, url)), dont_filter=True)
        else:
            yield Request(urljoin(response.url, url))

    issue_selector = response.xpath('//*[contains(@id,"item_Resource")]//@href')

    for url in issue_selector.extract():
        if not issue_selector.select('//*[contains(@class,"society-logo-block")]'):
            yield Request((urljoin(response.url, url)), dont_filter=True)
        else:
            yield Request(urljoin(response.url, url))

    #Get the articles URLs and yield Requests
    article_selector = response.xpath('//*[contains(@class,"viewArticleLink")]//@href')

    for url in article_selector.extract():
        if not article_selector.select('//*[contains(@class,"society-logo-block")]'):
            yield Request((urljoin(response.url, url)), dont_filter=True)
        else:
            yield Request(urljoin(response.url, url), callback=self.parse_item)

The settings for proxies are the following:

RETRY_TIMES = 10
RETRY_HTTP_CODES = [500, 503, 504, 400, 403, 404, 408, 302]
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 90,
    'scrapy_proxies.RandomProxy': 100,
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
}
PROXY_LIST = 'C:/some path/proxies.csv'
PROXY_MODE = 0

However, when I try running the code, it gets all the urls, but seems not to produce any items. The shell just keeps printing these errors:

2018-08-29 16:53:38 [scrapy.proxies] DEBUG: Using proxy http://103.203.133.170:8080, 8 proxies left

2018-08-29 16:53:38 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (302) to https://academic.oup.com/ilarjournal/article/53/1/E99/656113> from https://academic.oup.com/ilarjournal/article-abstract/53/1/E99/656113>

2018-08-29 16:53:38 [scrapy.proxies] DEBUG: Proxy user pass not found

Another probably important thing is that I have tried using the spider without proxies, and it still returns 302 errors for all the articles. Would appreciate any ideas what might be wrong or if there is already a good solution on another topic.

Answer 1

30x codes are normal redirects and you should allow them to happen.

Seems like your parse_item method is returning value instead of yielding, try replacing return item with yield item.