CODEWITHSUNDEEP
pythonxpathweb-scraping
Joji k joy
Joji k joy

Reputation: 33

Using Xpath find element by text and print the next element text in python

for the search buyer in the method I need to get the price of that buyer

def getprice(search_buyer):
    try:
        # webscrape url
        url = 'http://econpy.pythonanywhere.com/ex/001.html'
        response = requests.get(url)
        print(response)

        tree = html.fromstring(response.content)

        buyers =     tree.xpath('//div[contains(text(),"'+search_buyer+'")]/following-sibling:://span[@class="item-price"]')
   for div in buyers:
            print(";;;;;",div)
   except Exception:
        print("No buyer found")
getprice("Ben D. Rules")

(myvir) Administrators-Mac-mini:RESTAPI_ python JavaDeveloper$ python3 test.py 
<Response [200]>
No buyer found

Upvotes: 2

Views: 616

Answers (3)

QHarr
QHarr

Reputation: 84465

Consider using bs4 4.7.1 and css pseudo class :contains. I find css syntax less brittle. Maybe useful to future readers.

import requests
from bs4 import BeautifulSoup as bs

def get_price(buyer_name, soup):    
    price = soup.select_one('div:contains("' + buyer_name + '") + .item-price')
    if price is None:
        price = 'Not found'
    else:
        price = price.text
    return price

buyers = ['Patty Cakes', 'Derri Anne Connecticut', 'Moe Dess']
r = requests.get('http://econpy.pythonanywhere.com/ex/001.html')
soup = bs(r.content, 'html.parser')

for buyer_name in buyers:
    print(get_price(buyer_name, soup))

More efficient, pass list of buyers in:

def get_prices(buyers):   

    import requests
    from bs4 import BeautifulSoup as bs

    r = requests.get('http://econpy.pythonanywhere.com/ex/001.html')
    soup = bs(r.content, 'html.parser')
    results = [None] * len(buyers)
    for index, buyer in enumerate(buyers):
        price = soup.select_one('div:contains("' + buyer + '") + .item-price')
        if price is None:
            price = 'Not found'
        else:
            price = price.text
        results[index] = price
    return list(zip(buyers,results))

buyers = ['Patty Cakes', 'Derri Anne Connecticut', 'Moe Dess']

print(get_prices(buyers))

Upvotes: 1

MITHU
MITHU

Reputation: 164

You can do that in number of different ways. Here is another approach:

import requests
from lxml.html import fromstring    

link = 'http://econpy.pythonanywhere.com/ex/001.html'

def get_price(url,buyer):
    res = requests.get(url)
    root = fromstring(res.text)
    price = root.xpath(f'//*[@title="buyer-info"][div[contains(.,"{buyer}")]]/*[@class="item-price"]')

    if price:
        print(buyer,price[0].text)
    else:
        print("No buyer found")

get_price(link,"Ben D. Rules")

Upvotes: 0

RomanPerekhrest
RomanPerekhrest

Reputation: 92894

I guess that page should logically contain a list of unique buyers (with their prices). Use the following approach:

from lxml import html    

def get_price(search_buyer):
    try:
        url = 'http://econpy.pythonanywhere.com/ex/001.html'
        tree = html.parse(url)
        price = tree.xpath(
            f'//div[@title="buyer-name"][contains(text(), '
            f'"{search_buyer}")]/following-sibling::span[@class="item-price"]/text()')[0]
        print(f"{search_buyer} price: {price}")
    except Exception:
        print("No buyer found")


get_price("Ben D. Rules")

The actual output:

Ben D. Rules price: $7.50

Upvotes: 0

Related Questions