Reputation: 11
Hi everyone I managed to sign in embedding a lua script within my scrapy code. However when I redirect the scraper it does not keep signed in. How can i do that?
Here is my scrapy code:
class ExampleSpider(scrapy.Spider):
name = 'example'
start_urls = ["https://ident.familysearch.org/cis-web/oauth2/v3/authorization?client_secret=K7ymz%2FugczIps3SKrEjpp41QmXcDGK0BrGkSnGIunpHwQ0pq1rs%2FoAcsLaouJ0zbPaFtJ5eG7MdZ64LjiB3y5rMVFURm%2B18fjRPvje%2BeCwHiqBVqgu5i2gXzaHgaCi33ylnqSw%2F0h9uuLjo0myEJ%2FFDUhm8W9iTxDdKTonoYqbGkQ%2FwdfqPjKwLtXRjlEXq5mpkdkQZciVY4MDiSBaZJ3DfKZihyveKZSp3dC%2FJ3nH7Q%2FUjWpg8RP4fvR%2BN4ValGmrC9imWLOPlt5dwKOPsTF8z9AiOyaGOvxEn5rYzf4iNZ35Hr7X9p87BRhKvTRSuQJr24y4aPpC33EAr7QrmgZw%3D%3D&response_type=code&redirect_uri=https%3A%2F%2Fwww.familysearch.org%2Fauth%2Ffamilysearch%2Fcallback&state=%2F&client_id=3Z3L-Z4GK-J7ZS-YT3Z-Q4KY-YN66-ZX5K-176R"]
def start_requests(self):
script = """
function main(splash,args)
splash:set_viewport_size(1366, 768)
splash:set_user_agent('Splash bot')
local url = splash.args.url
splash:go(url)
splash:wait(5)
local form = splash: select('form[name=eventForm]')
local values = {
userName = 'username',
password = 'password',
}
assert(form:fill(values))
assert(form:submit())
assert(splash:wait(10))
return splash:html()
end
"""
yield SplashRequest( url=self.start_urls[0],
callback=self.after_login,
endpoint='execute',
args={
'lua_source': script,
'wait': 5})
def after_login(self, response):
# check login succeed before going on
if "authentication failed" in response.text:
self.logger.error("Login failed")
return
else:
url = "https://www.familysearch.org/search/record/results?count=20&q.anyDate.from=1500&q.anyDate.to=1650&q.anyPlace=Lima"
yield SplashRequest(url, self.parse_url, args={'wait':5})
def parse_url(self, response):
print(response.text)
I need to do that because when it works I need to iterate over many different urls.
Thanks a lot!
Upvotes: 1
Views: 148