CODEWITHSUNDEEP
scrapy
mindcast
mindcast

Reputation: 757

Scrapy issue with csv output

Here is my spider:

    from scrapy.contrib.spiders import CrawlSpider,Rule
    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
    from scrapy.selector import HtmlXPathSelector
    from vrisko.items import VriskoItem
    from scrapy.http import Request

    class vriskoSpider(CrawlSpider):
        name = 'vrisko'
        allowed_domains = ['vrisko.gr']
        start_urls = ['http://www.vrisko.gr/search/%CE%B3%CE%B9%CE%B1%CF%84%CF%81%CE%BF%CF%82/%CE%BA%CE%BF%CF%81%CE%B4%CE%B5%CE%BB%CE%B9%CE%BF']
        rules = (Rule(SgmlLinkExtractor(allow=('\?page=\d')),'parse_start_url',follow=True),)

        def parse_start_url(self, response):
         hxs = HtmlXPathSelector(response)
         subpages = hxs.select('//a[@class="detailsHyper_class"]/@href').extract()

         ep = hxs.select('//a[@itemprop="name"]/text()').extract()
         ad = hxs.select('//div[@class="results_address_class"]/text()').extract()
         for eponimia,address,subpage in zip(ep,ad,subpages):
          vriskoit = VriskoItem()
          vriskoit['eponimia'] = eponimia
          vriskoit['address'] = address 
          request = Request(subpage,callback = self.subPage)
          request.meta['vriskoit'] = vriskoit
          yield request

        def subPage(self,response):
         vriskoit = response.meta['vriskoit']
         hxs = HtmlXPathSelector(response)
         vriskoit['category'] = hxs.select('//div[@class="category_class"]/span/text()').extract()
         yield vriskoit

and here is my pipiline:

    import csv

    class myExporter(object):

    def __init__(self):
        self.brandCategoryCsv = csv.writer(open('brandCategoryTable.csv', 'wb'))
        self.brandCategoryCsv.writerow(['eponimia', 'address','category'])

    def process_item(self, item, spider):
            for e,a,c in zip(item['eponimia'],item['address'],item['category']):          
                 self.brandCategoryCsv.writerow([e.encode('utf-8'), a.encode('utf-8'), c.encode('utf-8')])
                 return item

my problem is that for both the first 2 fields (eponimia,address), only the first character is written to the output csv file and i cant find why.

Any help would be much appreciated, i am out of ideas.

Upvotes: 1

Views: 1188

Answers (1)

reclosedev
reclosedev

Reputation: 9522

Remove zip function from myExporter.process_item

def process_item(self, item, spider):          
    self.brandCategoryCsv.writerow([item['eponimia'].encode('utf-8'),
                                    item['address'].encode('utf-8'), 
                                    item['category'].encode('utf-8')])
    return item

You already converted items list to individual items in vriskoSpider.parse_start_url.

zip iterates your strings:

In [1]: a = 'test1'

In [2]: b = 'test2'

In [3]: for x, y in zip(a, b):
   ...:     print x, y
   ...:
t t
e e
s s
t t
1 2

Upvotes: 2

Related Questions