Information is not written to text file from twitter in python 2.7

Question

I have a project in python that I am trying to run from IDLE using Python 2.7 When I run the program a text file does get created like I want, but is not getting any information written to it and I do not understand why this is happening. I am running it as a module by pressing the F5 key in IDLE on my Ubuntu 12.04 LTS laptop.

Here is the code:

import time
import MySQLdb
import tweepy
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream

# Go to http://dev.twitter.com and create an app.
# The consumer key and secret will be generated for you after
consumer_key=" # Omitted "
consumer_secret=" # Omitted "

# After the step above, you will be redirected to your app's page.
# Create an access token under the the "Your access token" section
access_token=" # Omitted "
access_token_secret=" # Omitted "

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

api = tweepy.API(auth)

# If the authentication was successful, you should
# see the name of the account print out
print api.me().name

class StdOutListener(StreamListener):
        """ A listener handles tweets are the received from the stream.
        This is a basic listener that just prints received tweets to stdout.

        """
        def on_data(self, data):
            print data
            return True

        def on_error(self, status):
            print status

if __name__ == '__main__':
        l = StdOutListener()

        stream = Stream(auth, l)    
        stream.filter(track=['#google'])

My file on github if anyone wants to work on it with me on github: stocktwitterdb.py

A streaming example using tweepy can be found here on github: tweepy streaming.py

Now that I have things coming across the shell I want to put them to a database or a text file.

Answer 1

There's been some confusion in the changes you need to make, so I'm going to post a hopefully-fixed version here, along with some comments about why things are the way they are.

import time
import MySQLdb # also not currently used
import tweepy
from textwrap import TextWrapper # not used currently
from getpass import getpass

# define the format for messages here to avoid repetition
OUT_STR = '''
Status : %(text)s
Author : %(author)s
Date/Time : %(date)s
Source : %(source)s
Geo : %(geo)s


-----------------------------------------------------------------------------


'''

class StockTweetListener(tweepy.StreamListener):
    def __init__(self, target):
        super(StockTweetListener, self).__init__();
        self.target = target
        # status_wrapper = TextWrapper(width=60, initial_indent=' ',
        #                             subsequent_indent=' ')
        # This isn't used in the current code. But, if you were going
        # to use it, you'd need to assign it to self.status_wrapper;
        # otherwise the variable would be local to this __init__ method
        # and inaccessible from anything else.

    def on_status(self, status):
        try:
            msg = OUT_STR % {
                'text': status.text,
                'author': status.author.screen_name,
                'date': status.created_at,
                'source': status.source,
                'geo': status.geo,
            }
            print msg
            self.target.write(msg)
            # use self.target here. self is one of the paramaters to this
            # method and refers to the object; because you assigned to its
            # .target attribute before, you can use it here.

        except UnicodeDecodeError:
            # Catch any unicode errors while printing to console
            # and just ignore them to avoid breaking application.
            print "Record Skipped"

    def on_error(self, status_code):
        print 'An error has occured! Status code = %s' % status_code
        return True # keep stream alive

    def on_timeout(self):
        print 'Snoozing Zzzzzz'

def main():
    username = raw_input('Twitter username: ')
    password = getpass('Twitter password: ')
    stock = raw_input('Name of Stocks(comma seperated): ')
    stock_list = [u for u in stock.split(',')]
    follow_list = None # ??? you don't seem to define this variable

    # open results.txt here and name it f locally. once code flow leaves
    # the with statement, in this case only through an exception happening
    # that jumps you out of the while loop, the file will be closed.
    with open('results.txt', 'w') as f:
         while True:
             stream = tweepy.Stream(
                            username, password,
                            StockTweetListener(f), # passes the file to __init__
                                                   # as the "target" argument
                            timeout=None)
             stream.filter(follow_list, stock_list)

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        quit()

Answer 2

import time
import MySQLdb
import tweepy
from textwrap import TextWrapper
from getpass import getpass

qParam = "Twitter"

def __init__(self, target):
    super(StockTweetListener, self).__init__();
    self.target = target

with open('results.txt', 'w') as f:
    while True:
        stream = tweepy.Stream(username, password, StockTweetListener(target), timeout=None);
        stream.filter(None, stock_list)
target.truncate()

def debug_print(text):
    """Print text if debugging mode is on"""
    if settings.debug:
        print text

class StockTweetListener(tweepy.StreamListener):

    status_wrapper = TextWrapper(width=60, initial_indent=' ', subsequent_indent=' ')

    def on_status(self, status):
        try:
            # print 'Status : %s' %(self.status_wrapper.fill(status.text))
            print '\nStatus : %s' %(status.text)            
            print '\nAuthor : %s' %(status.author.screen_name)
            print '\nDate/Time : %s' %(status.created_at)
            print '\nSource : %s' %(status.source)           
            print '\nGeo : %s' %(status.geo)
            print '\n\n\n-----------------------------------------------------------------------------\n\n\n'

            l1 = '\nStatus : %s' %(status.text)            
            l2 = '\nAuthor : %s' %(status.author.screen_name)
            l3 = '\nDate/Time : %s' %(status.created_at)
            l4 = '\nSource : %s' %(status.source)           
            l5 = '\nGeo : %s' %(status.geo)
            l6 = '\n\n\n-----------------------------------------------------------------------------\n\n\n'

            target.write(l1)
            target.write(l2)
            target.write(l3)
            target.write(l4)
            target.write(l5)
            target.write(l6)                                    

        except UnicodeDecodeError:
            # Catch any unicode errors while printing to console
            # and just ignore them to avoid breaking application.
            pass

    def on_error(self, status_code):
        print 'An error has occured! Status code = %s' % status_code
        target.close()
        return True # keep stream alive

    def on_timeout(self):
        print 'Snoozing Zzzzzz'
        target.close()





def main():
    username = raw_input('Twitter username: ')
    password = getpass('Twitter password: ')
    stock = raw_input('Name of Stocks(comma seperated): ')
    stock_list = [u for u in stock.split(',')]




    stream = tweepy.Stream(username, password, StockTweetListener(target), timeout=None)
#    follow_list = None
    stream.filter(None, stock_list)



if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        target.close()
        quit()

This is producing errors

Answer 3

Well, this looks like a prime example for why it's a bad idea to do a bare try/except.

First, if there does happen any UnicodeDecodeError during printing, nothing will be written to your file either because that part of the code will be skipped.

Second, if any other exception were to occur, you wouldn't even notice it because you silently catch (and ignore) all of them.

So at least do

except UnicodeDecodeError:
    print "Record skipped!"

and see which other (if any) exceptions might be occurring.