Python 2.7 - redirect instantly output to log file

Question

Python test.py script is collecting data non-stop. To exit script - ctrl/c is expected.

File test.log is empty while script is running. Output to test.log is created only after script is finished.

It is running on windows 2008 server.

How to save ouput "on the fly", so I can check test.log and see the progress?

from time import sleep
import sys

class Logger(object):
 def __init__(self, filename="Default.log"):
     self.terminal = sys.stdout
     self.log = open(filename, "a")

 def write(self, message):
     self.terminal.write(message)
     self.log.write(message)

def main():

 signaled_to_stop = False

 sheep_counter       = 1 

 sys.stdout = Logger("test.log")

 print "Running.. (press CTRL-C to stop)"

 while True:
     try:
         # Wait 3 seconds between each step.

         sleep(3)
         print "Counting sheeps... %s" % sheep_counter
         sheep_counter = sheep_counter + 1
         # If signaled to stop then - Create a peepfile and stop the loop
         if signaled_to_stop:

             print "Stop requested..."
             if signaled_to_stop:
                 break

     except KeyboardInterrupt:
         print "\nStopping (might take some seconds).."
         signaled_to_stop = True

     except SystemExit as err:
         raise err

 print "Process has finished."

# If this file is executed directly then call the main() method
if __name__ == "__main__":
 main()

Output is like:

python test.py

Running.. (press CTRL-C to stop)
Counting sheeps... 1
Counting sheeps... 2
Counting sheeps... 3

Stopping (might take some seconds)..
Counting sheeps... 4
Stop requested...
Process has finished.

Answer 1

You need to close the file before the file update happens on the system. you cannot read or wright to an open file.

def log(self, exampletext):
    with open(self.filename) as fn:
        fn.write(exampletext)

In this example the file will be automatically closed once the line has been written.

Here is what i use to create my own log files with the same end result you are looking for.

class Log: #class to write to log file with time stamps
def __init__(self):
    import os, traceback, time
    self.traceback = traceback
    self.os = os
    self.time = time
    if getattr(sys, 'frozen', False): #windows path fix
        self.exe = self.os.path.dirname(sys.executable)
    elif __file__:
        self.exe = self.os.path.dirname(__file__)
    if not os.path.exists(os.path.dirname(str(os.environ['USERPROFILE'])+"\\Documents\\AppName\\")):
        os.makedirs(str(os.environ['USERPROFILE'])+"\\Documents\\AppName\\")
    self.fname = str(os.environ['USERPROFILE'])+"\\Documents\\AppName\\debug.log"
    self.logfile = None
def error(self, error):
    exc_type, exc_obj, exc_tb = sys.exc_info()
    trace_stack = self.traceback.extract_tb(exc_tb)[-1]
    trace_format = "Error in file "+str(trace_stack[0])+"\r     on line "+str(trace_stack[1])+", from module '"+str(trace_stack[2])+"'\r        "+str(trace_stack[3])
    try:
        self.logfile = open(self.fname, "a+")
    except:
        self.logfile = open(self.fname, "w+")
    strtime = str(self.time.strftime("%d-%m-%Y,(%z),%H:%M:%S"))
    self.logfile.write("error: %s, %s, %s\r" %(strtime, error, trace_format))
    self.logfile.close()
    self.logfile = None
def log(self, log):
    try:
        self.logfile = open(self.fname, "a+")
    except:
        self.logfile = open(self.fname, "w+")
    strtime = str(self.time.strftime("%d-%m-%Y,(%z),%H:%M:%S"))
    self.logfile.write("log: %s, %s\r" %(strtime, log))
    self.logfile.close()
    self.logfile = None

and here is how i use it in my applicaton

try:
    self.log.log("This is a standard log")
except Exception as err:
    exc_type, exc_obj, exc_tb = sys.exc_info()
    self.log.error("create_options failed\n%s, %s, %s, %s" %(err, exc_type, exc_obj, traceback.print_tb(exc_tb)))

EDIT: for a faster (simple) log here is how i would do it.

#logger.py
class log:
    def __init__(self, message):
        f = open("default.log", "a+")
        f.write(message+"\r")
        f.close()

#maincode.py
from logger import log
for i in range(10):
    log("Hello World %s" %i)

you can simply replace all print statements with log statements instead.

Answer 2

Solution was to use sys.stdout.flush(). It updates log file "on the fly".

I can just redirect output using ">"

python test.py > result.log

test.py

from time import sleep
import sys

signaled_to_stop = False

sheep_counter       = 1 

print "Running.. (press CTRL-C to stop)"

while True:
    try:
        # Wait 3 seconds between each step.

        sleep(3)
        print "Counting sheeps... %s" % sheep_counter
        sys.stdout.flush()
        sheep_counter += 1
        # If signaled to stop then - stop the loop
        if signaled_to_stop:

            print "Stop requested..."
            if signaled_to_stop:
                break

    except KeyboardInterrupt:
        print "\nStopping (might take some seconds).."
        signaled_to_stop = True

    except SystemExit as err:
        raise err

print "Process has finished."