CODEWITHSUNDEEP
pythonpython-3.xweb-scrapingpyqt4
Venkatesh Panabaka
Venkatesh Panabaka

Reputation: 2154

Dynamic web page scraping using python

I am running below code, but i am getting an empty list. Can you please help me in find out the issue.

execution: xvfb-run python dynamic_scrapy.py

import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs


class Client(QWebPage):
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebPage.__init__(self)
        self.loadFinished.connect(self.on_page_load)
        self.mainFrame().load(QUrl(url))
        self.app.exec_()
    def on_page_load(self):
        self.app.quit()

url = "https://pythonprogramming.net/parsememcparseface/"
client_response = Client(url)
source = client_response.mainFrame().toHtml()
soup = bs.BeautifulSoup(source, 'lxml')
print(soup)
js_test = soup.find_all('p', class_='jstest')
print(js_test)

Upvotes: 0

Views: 142

Answers (1)

Satish Prakash Garg
Satish Prakash Garg

Reputation: 2233

You need to convert the QString into string to pass it into BeautifulSoup. You can do something like this :

import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs

class Client(QWebPage):
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebPage.__init__(self)
        self.loadFinished.connect(self.on_page_load)
        self.mainFrame().load(QUrl(url))
        self.app.exec_()
    def on_page_load(self):
        self.app.quit()

url = "https://pythonprogramming.net/parsememcparseface/"
client_response = Client(url)

source = client_response.mainFrame().toHtml()
source_utf = unicode(source.toUtf8(), encoding="UTF-8") # Added
soup = bs.BeautifulSoup(source_utf, 'lxml')
js_test = soup.find_all('p', class_='jstest')
print(js_test)

This will result in :

[<p class="jstest" id="yesnojs">Look at you shinin!</p>]

Upvotes: 1

Related Questions