Python log rotation and compression

Question

My script allows me to import photos into a specific folder and update my database MongoDB via a JSON file import.

I set up a logging system with log rotation and compression.

I have different problems:

• Some actions are not loggued file, but in the console. (subprocess.call)
• the logger creates me three files instead of a one file

python script:

def moveFTPFiles(serverName,userName,passWord,remotePath,localPath,deleteRemoteFiles=False,onlyDiff=False):
    """Connect to an FTP server and bring down files to a local directory"""
    import os
    import sys
    import glob
    from sets import Set
    import ftplib 

    logger.info(' Deleting Files ')

    os.chdir(localDirectoryPath)
    files=glob.glob('*.*')
    for filename in files:
        os.unlink(filename)

    logger.info(' Retreiving Files ')

    try:
        ftp = ftplib.FTP(serverName)
        ftp.login(userName,passWord)
        ftp.cwd(remotePath)
        logger.info(' Connecting ')
        if onlyDiff:
            lFileSet = Set(os.listdir(localPath))
            rFileSet = Set(ftp.nlst())
            transferList = list(rFileSet - lFileSet)
            logger.info(' Missing ' + str(len(transferList)))
        else:
            transferList = ftp.nlst()
        delMsg = "" 
        filesMoved = 0
        for fl in transferList:
            # create a full local filepath
            localFile = localPath + fl
            # print "Create a full local filepath: " + localFile
            grabFile = True
            if grabFile:                
                #open a the local file
                fileObj = open(localFile, 'wb')
                # Download the file a chunk at a time using RETR
                ftp.retrbinary('RETR ' + fl, fileObj.write)
                # Close the file
                # print "Close the file "
                fileObj.close()
                filesMoved += 1
                #print "Uploaded: " + str(filesMoved)
                #sys.stdout.write(str(filesMoved)+' ')
                #sys.stdout.flush()

            # Delete the remote file if requested
            if deleteRemoteFiles:
                ftp.delete(fl)
                delMsg = " and Deleted"         

        logger.info('Files Moved' + delMsg + ': ' + str(filesMoved) + ' on ' + timeStamp())
    except ftplib.all_errors as e:
        logger.error('We have a problem on moveFTPFiles' + '%s' % e)
    ftp.close() # Close FTP connection
    ftp = None

def timeStamp():
    """returns a formatted current time/date"""
    import time
    return str(time.strftime("%a %d %b %Y %I:%M:%S %p"))

def importData(serverName,userName,passWord,directory,filematch,source,destination):
    import socket
    import ftplib
    import os
    import subprocess
    import json

    try:
        ftp = ftplib.FTP(serverName)
        ftp.login(userName,passWord)
        ftp.cwd(directory)
        logger.info(' Connecting ')
        # Loop through matching files and download each one individually
        for filename in ftp.nlst(filematch):
            fhandle = open(filename, 'wb')
            logger.info(' Getting ' + filename)
            ftp.retrbinary('RETR ' + filename, fhandle.write)
            fhandle.close()
        #convert xml to json
        logger.info(' Convert ' + filename + ' to .json ')
        subprocess.call('xml2json -t xml2json -o stockvo.json stockvo.xml --strip_text', shell=True)
        #remove xml file
        logger.info(' Delete ' + filename)
        os.unlink(source+'stockvo.xml')
        #modify json file
        logger.info(' Modify .json file')
        data = json.loads(open("stockvo.json").read())
        with open("stockvo.json", "w") as outfile:
                json.dump(data["Stock"]["Vehicule"], outfile)

        #import json file to MongoDB
        logger.info(' Import json file to MongoDB')
        subprocess.call('mongoimport --db AutoPrivilege -c cars stockvo.json --jsonArray --upsert --drop',shell=True)

        #remove old json file
        logger.info('Delete old .json file')
        ## if file exists, delete it ##
        myfile=destination+"stockvo.json"
        if os.path.isfile(myfile):
            os.remove(myfile)
        #os.unlink(destination+'stockvo.json')

        #move json file
        logger.info('Move .json')
        os.system('mv %s %s' % (source+'stockvo.json', destination+'stockvo.json'))

    except ftplib.all_errors as e:
        logger.error('We have a problem on importData' + '%s' % e)
    ftp.close() # Close FTP connection
    ftp = None

import time
import re
import os
import stat
import logging
import logging.handlers as handlers

class SizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler):
    """
    Handler for logging to a set of files, which switches from one file
    to the next when the current file reaches a certain size, or at certain
    timed intervals
    """
    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None,
                 delay=0, when='h', interval=1, utc=False):
        # If rotation/rollover is wanted, it doesn't make sense to use another
        # mode. If for example 'w' were specified, then if there were multiple
        # runs of the calling application, the logs from previous runs would be
        # lost if the 'w' is respected, because the log file would be truncated
        # on each run.
        if maxBytes > 0:
            mode = 'a'
        handlers.TimedRotatingFileHandler.__init__(
            self, filename, when, interval, backupCount, encoding, delay, utc)
        self.maxBytes = maxBytes

    def shouldRollover(self, record):
        """
        Determine if rollover should occur.

        Basically, see if the supplied record would cause the file to exceed
        the size limit we have.
        """
        if self.stream is None:                 # delay was set...
            self.stream = self._open()
        if self.maxBytes > 0:                   # are we rolling over?
            msg = "%s\n" % self.format(record)
            self.stream.seek(0, 2)  #due to non-posix-compliant Windows feature
            if self.stream.tell() + len(msg) >= self.maxBytes:
                return 1
        t = int(time.time())
        if t >= self.rolloverAt:
            return 1
        return 0

if __name__ == '__main__':      

    #log to a file
    log_filename='/opt/log/importData.log'
    logger=logging.getLogger('importData')
    logger.setLevel(logging.DEBUG)
    handler=SizedTimedRotatingFileHandler(
        log_filename, maxBytes=100, backupCount=5,
        when='s',interval=10,
        # encoding='bz2',  # uncomment for bz2 compression
    )
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    #--- constant connection values
    ftpServerName = "xxxxx.xxxxxx"
    ftpU = "xxxxxxxx"
    ftpP = "xxxxxx"
    remoteDirectoryPath = "/xxxxxx/xxxxxx/xxxxxx/xxxxxxx/"
    localDirectoryPath = "/xxxxx/xxxxxxxxx/xxxxxxxx/xxxxxxx/"

    directory = '/xxxxxxx/'
    filematch = '*.xml'
    source='/xxxxxx/xxxxxxxx/'
    destination='/xxxxxxxx/xxxxxx/'

    deleteAfterCopy = False     #set to true if you want to clean out the remote directory
    onlyNewFiles = True         #set to true to grab & overwrite all files locally
    importData(ftpServerName,ftpU,ftpP,directory,filematch,source,destination)
    moveFTPFiles(ftpServerName,ftpU,ftpP,remoteDirectoryPath,localDirectoryPath,deleteAfterCopy,onlyNewFiles)

importData.log:

2015-04-23 11:33:57,408 INFO Files Moved: 1145 on Thu 23 Apr 2015 11:33:57 AM

importData.log.2015-04-23_11-33-40:

2015-04-23 11:33:40,896 INFO Deleting Files 2015-04-23 11:33:40,956 INFO Retreiving Files

importData.log2015-04-23_11-33-41:

2015-04-23 11:33:41,386 INFO Connecting 2015-04-23 11:33:41,825 INFO Missing 1145

Can anyone suggest a way in python to solve my problems .?

Answer 1

SizedTimedRotatingFileHandler(log_filename, maxBytes=100, ...

Does exactly what it's configured to do - a maximum of 100 bytes will be logged into one file. Increase the max size to a few (hundred?) megabytes and you'll get infrequent rotation.

Regarding only partial logging to the file, you define the handler only for the 'importData' module. Other modules will write to the default handler (likely the console).

Regarding the subprocess.call() itself, it's not actually logging anything. Unless you capture the output using the stdout argument, the output will be printed to the normal stdout. You need to setup a pipe and read from it into the logger instead.