Monday, April 11, 2011

compressed RotatingFileHandler for python

class CompressedRotatingFileHandler(RotatingFileHandler):
    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0):
        RotatingFileHandler.__init__(self, filename, mode, maxBytes, backupCount, encoding, delay)

    def doRollover(self):
        self.stream.close()
        if self.backupCount > 0:
            for i in range(self.backupCount - 1, 0, -1):
                sfn = "%s.%d.gz" % (self.baseFilename, i)
                dfn = "%s.%d.gz" % (self.baseFilename, i + 1)
                if os.path.exists(sfn):
                    #print "%s -> %s" % (sfn, dfn)
                    if os.path.exists(dfn):
                        os.remove(dfn)
                    os.rename(sfn, dfn)
            dfn = self.baseFilename + ".1.gz"
            if os.path.exists(dfn):
                os.remove(dfn)
            import gzip
            try:
                f_in = open(self.baseFilename, 'rb')
                f_out = gzip.open(dfn, 'wb')
                f_out.writelines(f_in)
            except:
                if not os.path.exists(dfn):
                    if os.path.exists(self.baseFilename):
                        os.rename(self.baseFilename, dfn)
            finally:
                if "f_out" in dir() and f_out is not None:
                    f_out.close()
                if "f_in" in dir() and f_in is not None:
                    f_in.close()
            if os.path.exists(self.baseFilename):
                os.remove(self.baseFilename)
            #os.rename(self.baseFilename, dfn)
            #print "%s -> %s" % (self.baseFilename, dfn)
        self.mode = 'w'
        self.stream = self._open()

Friday, April 1, 2011

other thread pool options used in pylons configuration

[server:main]
use = egg:Paste#http
host = 127.0.0.1:8081
# These options make it easier to trigger the thread pool catches
# (i.e., threads are hung fast, killed fast, spawn fast, and the
# whole process dies quickly due to zombies)
threadpool_workers = 3
threadpool_hung_thread_limit = 10
threadpool_kill_thread_limit = 20
threadpool_spawn_if_under = 2
threadpool_max_zombie_threads_before_die = 2
threadpool_hung_check_period = 1
threadpool_dying_limit = 10

extend default thread pool workers of paster in pylons

add a option named threadpool_workers in server:main section in development.ini as below:

[server:main]
use = egg:Paste#http
host = 127.0.0.1
port = 5000
threadpool_workers = 20

then you will get this message from log:
11:09:33,678 INFO [paste.httpserver.ThreadPool] kill_hung_threads status: 20 threads (0 working, 20 idle, 0 starting) ave time N/A, max time 0.00sec, killed 0 workers