scrapy.core.engine DEBUG: Crawled (200) Scrapy Framework

Question

Recently I started to use scrapy framework. I have tried to extract contect form this page: libgen.io, and I had a bug when I executed the command:

scrapy crawl libgen -t csv

And I do not understand that the error is due.

I will very thankful if you can help me :c

The files that I have in my principal folder are:

libGenFolder
|
|
|_ __pycache__
|_ spiders
     |
     |_ __pycache__
     |_ spider.py
        |                

This is my "spider.py"

import scrapy
from scrapy import Selector
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from scrapy.exceptions import CloseSpider
from getMeMore.items import GetmemoreItem

class libgenSpider(CrawlSpider):
    name = 'libgen'
    item_count = 0
    allowed_domain = ['www.libgen.io']
    start_urls = ['http://libgen.io/search.php?req=ciencia&lg_topic=libgen&open=0&view=detailed&res=25&phrase=1&column=def']
    
    # for url in start_urls:
    #     yield scrapy.Request(url=url, callback=self.parse_item)

    def parse_item (self, response):
        ml_item = GetmemoreItem()

        # info de link
        ml_item['titulo'] = response.xpath('//td[@colspan="2"]/b/a/text()').extract()
        ml_item['autor'] = response.xpath('//td[@colspan="3"]/b/a/text()').extract()
        ml_item['img'] = response.xpath('//td[@rowspan="20"]/a/img[@width="240"]/@src').extract()
        ml_item['language'] = response.xpath('//tr[7]/td[2]/text()').extract()
        ml_item['link'] = response.xpath('//tr[11]/td[2]/a/@href').extract()
        self.item_count += 1
        if self.item_count > 5:
            raise CloseSpider('item_exceeded')
        yield ml_item

|_ items.py
|_ middlewares.py
|_ pipelines.py
     |

This is my "pipelines.py"

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import scrapy
from scrapy import signals
from scrapy.exporters import CsvItemExporter
# from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from scrapy import Request
import csv

class GetmemorePipeline(object):
    def __init__(self):
        self.files = {}

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_opened(self, spider):
        file = open('%s_items.csv' % spider.name, 'w+b')
        self.files[spider] = file
        self.exporter = CsvItemExporter(file)
        self.exporter.fields_to_export = ['titulo', 'autor', 'img', 'language', 'link']
        self.exporter.start_exporting()

    def spider_closed(self, spider):
        self.exporter.finish_exporting()
        file = self.files.pop(spider)
        file.close()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

# class GetmemorePipeline(ImagesPipeline):

#     def get_media_requests(self, item, info):
#         return [Request(x, meta={'image_name': item["image_name"]})
#                 for x in item.get('image_urls', [])]

#     def file_path(self, request, response=None, info=None):
#         return '%s.jpg' % request.meta['image_name']

|_ settings.py
     |

This is my "settings.py"

BOT_NAME = 'getMeMore'

SPIDER_MODULES = ['getMeMore.spiders']
NEWSPIDER_MODULE = 'getMeMore.spiders'

# CSV export
ITEM_PIPELINES = {'getMeMore.pipelines.GetmemorePipeline': 300}

# Obey robots.txt rules
ROBOTSTXT_OBEY = True

Answer 1

The error clearly states that the URL you are trying to scrape is forbidden by their robots.txt

To scrape that, please change following variable in settings.py

ROBOTSTXT_OBEY = False