CODEWITHSUNDEEP
pythonweb-scrapingpython-requests
libolon
libolon

Reputation: 23

InvalidSchema: No connection adapters were found for

I am using beautiful soup with requests package in python 2.7 for web news scrapping.When I debug the below code I get the error.

#encoding:utf-8

import re
import socket
import requests
import httplib
import urllib2
from bs4 import BeautifulSoup

#headers = ('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0')
response = requests.get('http://www.mhi.com.my/')

class Crawler(object):
    """Crawler"""
    def __init__(self, url):
        self.url = url

    def getNextUrls(self):
        urls = []
        request = urllib2.Request(self.url)
        request.add_header('User-Agent',
        'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0')
        try:
            html = urllib2.urlopen(request)
        except socket.timeout, e:
            pass
        except urllib2.URLError,ee:
            pass
        except httplib.BadStatusLine:
            pass
            # analyse the txt have gotten
        soup = BeautifulSoup(response.text,'lxml')# slesct and return a list 
        pattern = 'http://www\.mhi\.com\.my/.*\.html'
        links = soup.find_all('a', href=re.compile(pattern))
        for link in links:
            urls.append(link)
        return urls

def getNews(url):
    print url
    xinwen = ''
    request = requests.get(url)
    request.add_header('User-Agent',
        'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0')
    try:
        html = urllib2.urlopen(request)
    except urllib2.HTTPError, e:
        print e.code

    soup = BeautifulSoup(html, 'html.parser')
    for news in soup.select('p.para'):
        xinwen += news.get_text().decode('utf-8')
    return xinwen

class News(object):
    """
    source:from where 
    title:title of news  
    time:published time of news
    content:content of news 
    type:type of news    
    """
    def __init__(self, title, time, content, type):
        self.title = title
        self.time = time
        self.content = content
        self.type = type

file = open('C:/MyFold/kiki.json', 'a')
url = "http://www.mhi.com.my"
print url
s = Crawler(url)
for newsUrl in s.getNextUrls():
    file.write(getNews(newsUrl))
    file.write("\n")
    print "---------------------------"

file.close()

This is returned error.

C:\Python27\python.exe C:/MyFold/CodeTest/file1.py
http://www.mhi.com.my
Traceback (most recent call last):
  File "C:/MyFold/CodeTest/file1.py", line 74, in <module>
    file.write(getNews(newsUrl))
  File "C:/MyFold/CodeTest/file1.py", line 42, in getNews
    request = requests.get(url)
  File "C:\Python27\lib\site-packages\requests\api.py", line 70, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Python27\lib\site-packages\requests\api.py", line 56, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Python27\lib\site-packages\requests\sessions.py", line 488, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Python27\lib\site-packages\requests\sessions.py", line 603, in send
    adapter = self.get_adapter(url=request.url)
  File "C:\Python27\lib\site-packages\requests\sessions.py", line 685, in get_adapter
    raise InvalidSchema("No connection adapters were found for '%s'" % url)
requests.exceptions.InvalidSchema: No connection adapters were found for '<a class="glow" href="http://www.mhi.com.my/akhbar2016.html" style="text-decoration: none;"></a>'
<a class="glow" href="http://www.mhi.com.my/akhbar2016.html" style="text-decoration: none;"></a>

Is it a problem about my loop? Can anyone help me with it?

Upvotes: 2

Views: 6522

Answers (1)

Tiny.D
Tiny.D

Reputation: 6556

In your classCrawler, the function getNextUrls() return <a> list:

[<a class="glow" href="http://www.mhi.com.my/akhbar2016.html" style="text-decoration: none;"></a>]

when you loop it, it will pass whole <a> element to function getNews, but the parameter should be a url.

You could change your function getNextUrls():

from

urls.append(link)

to

urls.append(link.get('href'))

so that the function getNextUrls will return you url list instead of <a> element list: 

['http://www.mhi.com.my/akhbar2016.html']

Upvotes: 3

Related Questions