Federico
Federico

Reputation: 11

change url while being signed in splash

Hi everyone I managed to sign in embedding a lua script within my scrapy code. However when I redirect the scraper it does not keep signed in. How can i do that?

Here is my scrapy code:

class ExampleSpider(scrapy.Spider):
    name = 'example'
    start_urls = ["https://ident.familysearch.org/cis-web/oauth2/v3/authorization?client_secret=K7ymz%2FugczIps3SKrEjpp41QmXcDGK0BrGkSnGIunpHwQ0pq1rs%2FoAcsLaouJ0zbPaFtJ5eG7MdZ64LjiB3y5rMVFURm%2B18fjRPvje%2BeCwHiqBVqgu5i2gXzaHgaCi33ylnqSw%2F0h9uuLjo0myEJ%2FFDUhm8W9iTxDdKTonoYqbGkQ%2FwdfqPjKwLtXRjlEXq5mpkdkQZciVY4MDiSBaZJ3DfKZihyveKZSp3dC%2FJ3nH7Q%2FUjWpg8RP4fvR%2BN4ValGmrC9imWLOPlt5dwKOPsTF8z9AiOyaGOvxEn5rYzf4iNZ35Hr7X9p87BRhKvTRSuQJr24y4aPpC33EAr7QrmgZw%3D%3D&response_type=code&redirect_uri=https%3A%2F%2Fwww.familysearch.org%2Fauth%2Ffamilysearch%2Fcallback&state=%2F&client_id=3Z3L-Z4GK-J7ZS-YT3Z-Q4KY-YN66-ZX5K-176R"]
 
    def start_requests(self):                               
        script = """
        function main(splash,args)
            splash:set_viewport_size(1366, 768)
            splash:set_user_agent('Splash bot')  
            local url = splash.args.url
            splash:go(url)
            splash:wait(5)
            local form = splash: select('form[name=eventForm]')
                local values = {
                    userName = 'username',
                    password = 'password',
                }
            assert(form:fill(values))
            assert(form:submit())
            assert(splash:wait(10))
            
            return splash:html()
        end
        """
        yield SplashRequest( url=self.start_urls[0],
                            callback=self.after_login,
                            endpoint='execute',
                            args={
                            'lua_source': script,
                            'wait': 5})
           
    def after_login(self, response):
        # check login succeed before going on
        if "authentication failed" in response.text:
            self.logger.error("Login failed")
            return
        else:
            url = "https://www.familysearch.org/search/record/results?count=20&q.anyDate.from=1500&q.anyDate.to=1650&q.anyPlace=Lima"
            yield SplashRequest(url, self.parse_url, args={'wait':5})                                    

    def parse_url(self, response):
        print(response.text)

I need to do that because when it works I need to iterate over many different urls.

Thanks a lot!

Upvotes: 1

Views: 148

Answers (0)

Related Questions