CODEWITHSUNDEEP
pythonrequestscrapy
Andrew Gowa
Andrew Gowa

Reputation: 129

Scrapy making Request with POST method

I'm trying to scrape a list of product from "http://eastasiaeg.com/en/laptop-in-egypt" using Scrapy.

Part of products loads dynamically, and a tried to build Scrapy request. But something is wrong with it. Pls help.

# -*- coding: utf-8 -*-
import scrapy

from v4.items import Product


class IntelEGEastasiaegComSpider(scrapy.Spider):
    name = "intel_eg_eastasiaeg_com_py"

    start_urls = [
            'http://eastasiaeg.com/en/laptop-in-egypt'
        ]

    def start_requests(self):
        request_body = {"categoryId":"3","manufacturerId":"0","vendorId":"0","priceRangeFilterModel7Spikes":{"CategoryId":"3","ManufacturerId":"0","VendorId":"0","SelectedPriceRange":{},"MinPrice":"2400","MaxPrice":"44625"},"specificationFiltersModel7Spikes":{"CategoryId":"3","ManufacturerId":"0","VendorId":"0","SpecificationFilterGroups":[{"Id":"27","FilterItems":[{"Id":"103","FilterItemState":"Unchecked"},{"Id":"104","FilterItemState":"Unchecked"},{"Id":"105","FilterItemState":"Unchecked"},{"Id":"110","FilterItemState":"Unchecked"}]},{"Id":"11","FilterItems":[{"Id":"302","FilterItemState":"Unchecked"},{"Id":"75","FilterItemState":"Unchecked"}]},{"Id":"6","FilterItems":[{"Id":"21","FilterItemState":"Unchecked"},{"Id":"24","FilterItemState":"Unchecked"},{"Id":"25","FilterItemState":"Unchecked"},{"Id":"26","FilterItemState":"Unchecked"}]},{"Id":"5","FilterItems":[{"Id":"1069","FilterItemState":"Unchecked"},{"Id":"1078","FilterItemState":"Unchecked"},{"Id":"1118","FilterItemState":"Unchecked"},{"Id":"1862","FilterItemState":"Unchecked"}]},{"Id":"2","FilterItems":[{"Id":"8","FilterItemState":"Unchecked"},{"Id":"10","FilterItemState":"Unchecked"},{"Id":"1451","FilterItemState":"Unchecked"},{"Id":"1119","FilterItemState":"Unchecked"}]},{"Id":"8","FilterItems":[{"Id":"61","FilterItemState":"Unchecked"},{"Id":"62","FilterItemState":"Unchecked"},{"Id":"63","FilterItemState":"Unchecked"}]},{"Id":"333","FilterItems":[{"Id":"2460","FilterItemState":"Unchecked"}]}]},"attributeFiltersModel7Spikes":"null","manufacturerFiltersModel7Spikes":{"CategoryId":"3","ManufacturerFilterItems":[{"Id":"2","FilterItemState":"Unchecked"},{"Id":"1","FilterItemState":"Unchecked"},{"Id":"3","FilterItemState":"Unchecked"},{"Id":"6","FilterItemState":"Unchecked"}]},"vendorFiltersModel7Spikes":"null","pageNumber":"2","orderby":"10","viewmode":"grid","pagesize":"null","queryString":"","shouldNotStartFromFirstPage":"true","onSaleFilterModel":"null","keyword":"","searchCategoryId":"0","searchManufacturerId":"0","priceFrom":"","priceTo":"","includeSubcategories":"False","searchInProductDescriptions":"False","advancedSearch":"False","isOnSearchPage":"False"}
        for body in request_body:
            request_body = body
            yield scrapy.Request('http://eastasiaeg.com/en/getFilteredProducts',
                                 method="POST",
                                 body=request_body,
                                 callback=self.parse,
                                 headers={'Content-type': 'application/json; charset=UTF-8'}, )

    def parse(self, response):
        print response.body

Upvotes: 4

Views: 10410

Answers (1)

Granitosaurus
Granitosaurus

Reputation: 21406

You should use scrapy.FormRequest when you want to do POST requests with form data in them. 

def start_requests(self):
    form_data = {}  # your formdata
    yield scrapy.FormRequest(url, formdata=form_data)

Your approach could work as well, but your for loop doesn't make much sense here.
for body in request_body: iterates through the keys of the dictionary that your request_body is and you basically make 24 requests with only 1 key in the body.
So to do it with scrapy.Request try: 

def start_requests(self): 
    form_data = {}  # your formdata
    # Request only takes string as body so you need to
    # convert python dict to string
    request_body = json.dumps(form_data)
    yield scrapy.Request('http://eastasiaeg.com/en/getFilteredProducts',
                         method="POST",
                         body=request_body,
                         headers={'Content-Type': 'application/json; charset=UTF-8'}, )
    # Usually Content-Type matters here a lot.

P.S. scrapy Requests default to self.parse callback so you don't need to specify it.

Upvotes: 10

Related Questions