CODEWITHSUNDEEP
python
n1c9
n1c9

Reputation: 2687

Replace a specific word

This script grabs headlines from a number of news sites and counts how many times words in the headlines come up.

I get words like "to", "for", and similar words that I didn't intend to snatch with this script.

I tried writing a str.translate(None, "to") to remove the word to, but it removed "greedily" - snatching out sections of Washington when all I meant for it to remove was the word "to".

import pprint
import feedparser
from collections import Counter

def feedGrabber(feed):
    parsed = feedparser.parse(feed)
    feed1 = []
    feed1.append(parsed.entries[0].title)
    feed1.append(parsed.entries[1].title)
    feed1.append(parsed.entries[3].title)
    feed1.append(parsed.entries[4].title)
    feed1.append(parsed.entries[5].title)
    feed1.append(parsed.entries[6].title)
    feed1.append(parsed.entries[7].title)
    feed1.append(parsed.entries[8].title)
    feed1.append(parsed.entries[9].title)
    feed1 = str(feed1)
    feedsplit = feed1
    feedsplit = feedsplit.translate(None, '\'')
    feedsplit = feedsplit.translate(None, 'u')
    feedsplit = feedsplit.translate(None, '[')
    feedsplit = feedsplit.translate(None, ']')
    feedsplit = str.lower(feedsplit)
    feedsplit = str.split(feedsplit)
    return(feedsplit)

reddit = feedGrabber("https://www.reddit.com/r/news/.rss")
cnn = feedGrabber('http://rss.cnn.com/rss/cnn_topstories.rss')
nyt = feedGrabber('http://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml')

one = Counter(reddit)
two = Counter(cnn)
three = Counter(nyt)
pprint.pprint(one + two + three)

Upvotes: 0

Views: 63

Answers (1)

LetzerWille
LetzerWille

Reputation: 5658

here is a list of common words, you can use a list comprehention to remove them from text 

text = [ x for x in  text if not isCommon(x)]


   def isCommon(word):

    commonWords = ["the", "be", "and", "of", "a", "in", "to", "have", "it",
        "i", "that", "for", "you", "he", "with", "on", "do", "say", "this",
        "they", "is", "an", "at", "but","we", "his", "from", "that", "not",
        "by", "she", "or", "as", "what", "go", "their","can", "who", "get",
        "if", "would", "her", "all", "my", "make", "about", "know", "will",
        "as", "up", "one", "time", "has", "been", "there", "year", "so",
        "think", "when", "which", "them", "some", "me", "people", "take",
        "out", "into", "just", "see", "him", "your", "come", "could", "now",
        "than", "like", "other", "how", "then", "its", "our", "two", "more",
        "these", "want", "way", "look", "first", "also", "new", "because",
        "day", "more", "use", "no", "man", "find", "here", "thing", "give",
        "many", "well"]

    if word in commonWords:
        return True
    return False

Upvotes: 2

Related Questions