Reputation: 1
I'm trying to save an intercepted XHR response from a scrapy_playwright event_handler and save to a ScrapyItem. However when running the Scrapy spider, the code block HOW TO SAVE IN SCRAPY ITEM
does not execute. From here I would expect Scrapy to take over and process all pipelines and middleware, but nothing happens.
async def handle_response(self, response):
# page = response.meta["playwright_page"]
if "data.json" in response.url:
logging.info(f"######## DATA JSON RECEIVED ###### {response.url}")
data = await response.json()
if data is not None:
homes = data['points']
# logging.info(homes[0])
# logging.info(homes[0]['Avail'])
for home in homes:
## CORRECTLY PARSED A SINGLE HOME
logging.info(home)
## HOW TO SAVE IN SCRAPY ITEM?
listing = ListingItem()
listing['avail'] = home['Avail']
listing['avail_date'] = home['AvailDate']
listing['bath_type'] = home['BathType']
listing['bedroom_count'] = home['BedRoomCount']
listing['bedrooms'] = home['Bedrooms']
listing['ev'] = home['EV']
listing['free_parking'] = home['FreeParking']
listing['furnished'] = home['Furnished']
listing['gym'] = home['Gym']
listing['hot_tub'] = home['HotTub']
listing['housekeeping'] = home['HouseKeeping']
listing['is_multiple_unit'] = home['IsMultipleUnit']
listing['king_size_bed'] = home['KingSizeBed']
listing['kitchen'] = home['Kitchen']
listing['laundry'] = home['Laundry']
listing['description'] = home['MDesc']
listing['pilink'] = home['PILink']
listing['pool'] = home['Pool']
listing['price'] = home['Price']
listing['submitted_by'] = home['Submitted_by']
listing['utilities'] = home['Utilities']
listing['verified'] = home['Verified']
listing['wheelchair'] = home['WheelChair']
listing['wifi'] = home['Wifi']
listing['id'] = home['id']
listing['main_image'] = home['image']
listing['lat'] = home['lat']
listing['long'] = home['long']
listing['name'] = home['name']
yield listing
Example log file:
INFO: ######## INSIDE HOMES LOOP ######
INFO: {'id': '694858_1', 'IsMultipleUnit': 'False', 'name': "<div class='title' title='Dallas Oasis, Perfect for Families, Couples...'>Dallas Oasis, Perfect for Families, Couples...</div>", 'lat': 32.7283216, 'long': -96.8075738, 'MDesc': '<ul><li>Fully Furnished</li><li>Utilities Included</li><li>1 bedroom</li></ul>', 'MDesc2': '$1,900/month ', 'PType': 'Apartment', 'PF': 'False', 'image': 'https://www.furnishedfinder.com/_pserp_/694858/1/694858_1_56006852-full.png', 'PILink': 'https://www.furnishedfinder.com/property/694858_1', 'Icon': '', 'Price': '$1,900', 'Avail': 'Contact for Availability', 'AvailDate': '10/8/2024', 'Preferred': 'True', 'Verified': 'True', 'BathType': 'Private Bath', 'LNRLink': '', 'camount': '0', 'BedRoomCount': '1', 'Furnished': 'True', 'IsDPCEnabled': 'False', 'Utilities': 'True', 'HotTub': 'False', 'FreeParking': 'True', 'HouseKeeping': 'False', 'Wifi': 'True', 'Pool': 'False', 'SmartTV': 'True', 'Laundry': 'True', 'KingSizeBed': 'False', 'EV': 'False', 'Kitchen': 'True', 'WheelChair': 'False', 'Gym': 'False', 'Score': '1000', 'PMScore': '0', 'QualityScore': '0', 'brand': '', 'Subbrand': ', ', 'ComplexPartner': 'False', 'Bedrooms': [{'title': 'Bedroom 1', 'sleep': '3', 'beds': 'Queen Bed'}]}
Upvotes: 0
Views: 25