dcc
dcc

Reputation: 1

How to save ScrapyItem in ScrapyPlaywright event_handler?

I'm trying to save an intercepted XHR response from a scrapy_playwright event_handler and save to a ScrapyItem. However when running the Scrapy spider, the code block HOW TO SAVE IN SCRAPY ITEM does not execute. From here I would expect Scrapy to take over and process all pipelines and middleware, but nothing happens.

    async def handle_response(self, response):
        # page = response.meta["playwright_page"]

        if "data.json" in response.url:
            logging.info(f"######## DATA JSON RECEIVED ###### {response.url}")
            data = await response.json()

            if data is not None:
                homes = data['points']
                # logging.info(homes[0])
                # logging.info(homes[0]['Avail'])

                for home in homes:
                    ## CORRECTLY PARSED A SINGLE HOME 
                    logging.info(home)

                    ## HOW TO SAVE IN SCRAPY ITEM?
                    listing = ListingItem()
                    listing['avail'] = home['Avail']
                    listing['avail_date'] = home['AvailDate']
                    listing['bath_type'] = home['BathType']
                    listing['bedroom_count'] = home['BedRoomCount']
                    listing['bedrooms'] = home['Bedrooms']
                    listing['ev'] = home['EV']
                    listing['free_parking'] = home['FreeParking']
                    listing['furnished'] = home['Furnished']
                    listing['gym'] = home['Gym']
                    listing['hot_tub'] = home['HotTub']
                    listing['housekeeping'] = home['HouseKeeping']
                    listing['is_multiple_unit'] = home['IsMultipleUnit']
                    listing['king_size_bed'] = home['KingSizeBed']
                    listing['kitchen'] = home['Kitchen']
                    listing['laundry'] = home['Laundry']
                    listing['description'] = home['MDesc']
                    listing['pilink'] = home['PILink']
                    listing['pool'] = home['Pool']
                    listing['price'] = home['Price']
                    listing['submitted_by'] = home['Submitted_by']
                    listing['utilities'] = home['Utilities']
                    listing['verified'] = home['Verified']
                    listing['wheelchair'] = home['WheelChair']
                    listing['wifi'] = home['Wifi']
                    listing['id'] = home['id']
                    listing['main_image'] = home['image']
                    listing['lat'] = home['lat']
                    listing['long'] = home['long']
                    listing['name'] = home['name']

                    yield listing
                

Example log file:

INFO: ######## INSIDE HOMES LOOP ######
INFO: {'id': '694858_1', 'IsMultipleUnit': 'False', 'name': "<div class='title' title='Dallas Oasis, Perfect for Families, Couples...'>Dallas Oasis, Perfect for Families, Couples...</div>", 'lat': 32.7283216, 'long': -96.8075738, 'MDesc': '<ul><li>Fully Furnished</li><li>Utilities Included</li><li>1 bedroom</li></ul>', 'MDesc2': '$1,900/month ', 'PType': 'Apartment', 'PF': 'False', 'image': 'https://www.furnishedfinder.com/_pserp_/694858/1/694858_1_56006852-full.png', 'PILink': 'https://www.furnishedfinder.com/property/694858_1', 'Icon': '', 'Price': '$1,900', 'Avail': 'Contact for Availability', 'AvailDate': '10/8/2024', 'Preferred': 'True', 'Verified': 'True', 'BathType': 'Private Bath', 'LNRLink': '', 'camount': '0', 'BedRoomCount': '1', 'Furnished': 'True', 'IsDPCEnabled': 'False', 'Utilities': 'True', 'HotTub': 'False', 'FreeParking': 'True', 'HouseKeeping': 'False', 'Wifi': 'True', 'Pool': 'False', 'SmartTV': 'True', 'Laundry': 'True', 'KingSizeBed': 'False', 'EV': 'False', 'Kitchen': 'True', 'WheelChair': 'False', 'Gym': 'False', 'Score': '1000', 'PMScore': '0', 'QualityScore': '0', 'brand': '', 'Subbrand': ', ', 'ComplexPartner': 'False', 'Bedrooms': [{'title': 'Bedroom 1', 'sleep': '3', 'beds': 'Queen Bed'}]}

Upvotes: 0

Views: 25

Answers (0)

Related Questions