CODEWITHSUNDEEP
pythonscrapy
Stackbeans
Stackbeans

Reputation: 279

Scrapy only returning a single item from list

I believe that I have my xpaths coded in the incorrect way, as I only get a single result for each url. Whereas, there are in total 25 job posts for each url (not included those in the next page.) How can I correct my xpaths to get all the results?

Here's my scraper:

from scrapy.item import Field
import scrapy
from scrapy.loader import ItemLoader
from scrapy.crawler import CrawlerProcess
from itemloaders.processors import TakeFirst
import pandas as pd
from collections import defaultdict

class CvItem(scrapy.Item):
    category = Field(output_processor = TakeFirst())
    salary = Field(output_processor = TakeFirst())
    title =  Field(output_processor = TakeFirst())
    organisation = Field(output_processor = TakeFirst())

class CvSpider(scrapy.Spider):
    name = 'cv'
    start_urls = {'Accountancy_finance': ['https://www.cv-library.co.uk/Degree-Finance-jobs?us=1',
              'https://www.cv-library.co.uk/Degree-Accounting-jobs?us=1'],
             'Aeronautical_Engineering': ['https://www.cv-library.co.uk/Degree-Aeronautical-Engineering-jobs?us=1'],
             'Manufacturing_Engineering': ['https://www.cv-library.co.uk/Degree-Manufacturing-Engineering-jobs?us=1'],
             'Agriculture_and_Forestry': ['https://www.cv-library.co.uk/Degree-Forestry-jobs?us=1']}

    def start_requests(self):
        for items, urls in self.start_urls.items():
            for url in urls:
                yield scrapy.Request(
                url = url, 
                callback = self.parse, 
                cb_kwargs = {
                    'items':items
                }
            )
    def parse(self, response, items):
        container = response.xpath('//ol[@id="searchResults"]')
        for lists in container:
            loader = ItemLoader(CvItem(), selector = lists)
            loader.add_value('category', items)
            loader.add_xpath('title', '//article[@id]//a[@title]/@title')
            loader.add_xpath('salary', '//article[@id]//dl//dd[@class="job__details-value salary"]//text()')
            loader.add_xpath('organisation', '//article[@id]/div//div/p/a//text()')
            yield loader.load_item()

There was a slight mistake with the requests that I updated for those of you that had checked for the first 15minutes since I uploaded it.

Upvotes: 1

Views: 299

Answers (1)

SuperUser
SuperUser

Reputation: 4822

The problem was in the container's xpath. You only get the container without actually the items in it so you only loop once on the container itself and not the actual items you want to scrape.

from scrapy.item import Field
import scrapy
from scrapy.loader import ItemLoader
from scrapy.crawler import CrawlerProcess
from itemloaders.processors import TakeFirst
import pandas as pd
from collections import defaultdict


class CvItem(scrapy.Item):
    category = Field(output_processor=TakeFirst())
    salary = Field(output_processor=TakeFirst())
    title = Field(output_processor=TakeFirst())
    organisation = Field(output_processor=TakeFirst())


class CvSpider(scrapy.Spider):
    name = 'cv'
    start_urls = {'Accountancy_finance': ['https://www.cv-library.co.uk/Degree-Finance-jobs?us=1',
                                          'https://www.cv-library.co.uk/Degree-Accounting-jobs?us=1'],
                  'Aeronautical_Engineering': ['https://www.cv-library.co.uk/Degree-Aeronautical-Engineering-jobs?us=1'],
                  'Manufacturing_Engineering': ['https://www.cv-library.co.uk/Degree-Manufacturing-Engineering-jobs?us=1'],
                  'Agriculture_and_Forestry': ['https://www.cv-library.co.uk/Degree-Forestry-jobs?us=1']}

    def start_requests(self):
        for items, urls in self.start_urls.items():
            for url in urls:
                yield scrapy.Request(
                    url=url,
                    cb_kwargs={
                        'items': items
                    }
                )

    def parse(self, response, items):
        container = response.xpath('//ol[@id="searchResults"]//li[@class="results__item"]')
        for lists in container:
            loader = ItemLoader(CvItem(), selector=lists)
            loader.add_value('category', items)
            loader.add_xpath('title', '//article[@id]//a[@title]/@title')
            loader.add_xpath('salary', '//article[@id]//dl//dd[@class="job__details-value salary"]//text()')
            loader.add_xpath('organisation', '//article[@id]/div//div/p/a//text()')
            yield loader.load_item()

Upvotes: 1

Related Questions