CODEWITHSUNDEEP
pythonfeedparser
wormyworm
wormyworm

Reputation: 179

Detect duplicate titles using feedparser

def parseRSS(rss_url):
    parsed_feed = feedparser.parse(rss_url)
    return parsed_feed

def getHeadlines(rss_url,key):
    headlines = []
    feed = parseRSS(rss_url)
    for newsitem in feed['items']:
        if newsitem['title'] not in headlines:
            headlines.append([newsitem,key])
        else:
            print("-----------------------Duplicate title found----------------------")
    return headlines

def get_rss():
    allheadlines = []
    newsurls = {
    ('key1','source1'): 'https://news.google.com/news/rss/?hl=en&ned=us&gl=US',
    ('key2','source2'): 'https://news.google.com/news/rss/?hl=en&ned=us&gl=US',
    }
    for key,url in newsurls.items():
        allheadlines.extend(getHeadlines(url,key))

    return allheadlines

allheadlines = get_rss()

for hl in allheadlines:
    source = hl[1][0]
    key = hl[1][1]
    title = hl[0]['title']
    link = hl[0]['link']

I'm using the same rss feed for testing purposes. I'm checking if the title is already in headlines every time I append a new headline. However, it doesn't seem to detect duplicate titles. Duplicate title found is never being printed. What am I doing wrong?

Upvotes: 0

Views: 326

Answers (1)

Marcus.Aurelianus
Marcus.Aurelianus

Reputation: 1518

try this

def parseRSS(rss_url):
    parsed_feed = feedparser.parse(rss_url)
    return parsed_feed

def getHeadlines(rss_url,key,allheadlines,allitems):
    feed = parseRSS(rss_url)
    for newsitem in feed['items']:
        if newsitem['title'] not in allheadlines:
            allheadlines.append(newsitem['title'])
            allitems.append([newsitem,key])
        else:
            print("-----------------------Duplicate title found----------------------")
    return allheadlines,allitems

def get_rss():
    allheadlines = []
    allitems = []
    newsurls = {
    ('key1','source1'): 'https://news.google.com/news/rss/?hl=en&ned=us&gl=US',
    ('key2','source2'): 'https://news.google.com/news/rss/?hl=en&ned=us&gl=US',
    }
    for key,url in newsurls.items():
        allheadlines,allitems=(getHeadlines(url,key,allheadlines,allitems))

    return allitems

allheadlines = get_rss()

Upvotes: 1

Related Questions