From 66013d7a6c8594d435e6d3ff0b468b51c40318b6 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 21 Nov 2011 16:51:43 +0000 Subject: Made ready for 0.3.2. --- NEWS.txt | 9 +++++ doc/conf.py | 2 +- doc/index.rst | 1 + doc/redis.rst | 11 ++++++ logutils/__init__.py | 2 +- logutils/queue.py | 11 +++++- logutils/redis.py | 72 ++++++++++++++++++++++++++++++++++++ setup.py | 21 +---------- tests/logutil_tests.py | 4 ++ tests/mytest.py | 9 +++++ tests/test_dictconfig.py | 39 ++++++++++++++++++++ tests/test_redis.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 255 insertions(+), 22 deletions(-) create mode 100644 doc/redis.rst create mode 100644 logutils/redis.py create mode 100644 tests/mytest.py create mode 100644 tests/test_redis.py diff --git a/NEWS.txt b/NEWS.txt index 8b235b6..7cfe005 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -5,6 +5,15 @@ What's New in logutils ====================== +Version 0.3.2 +------------- + +- Improvements in QueueListener implementation. +- Added redis module with RedisQueueHandler and + RedisQueueListener. +- Added unit test for a handler in a module + where absolute imports are used. + Version 0.3.1 ------------- diff --git a/doc/conf.py b/doc/conf.py index c526df8..2eb049a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,7 +51,7 @@ copyright = u'2010-2011, Vinay Sajip' # The short X.Y version. version = '0.3' # The full version, including alpha/beta/rc tags. -release = '0.3' +release = '0.3.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/index.rst b/doc/index.rst index 095d0e8..1fc321b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -17,6 +17,7 @@ tasks you may want to perform: libraries queue + redis testing dictconfig adapter diff --git a/doc/redis.rst b/doc/redis.rst new file mode 100644 index 0000000..8293c30 --- /dev/null +++ b/doc/redis.rst @@ -0,0 +1,11 @@ +Working with Redis queues +========================= + +:class:`~logutils.queue.QueueHandler` and :class:`~logutils.queue.QueueListener` classes are provided to facilitate interfacing with Redis. + +.. autoclass:: logutils.redis.RedisQueueHandler + :members: + +.. autoclass:: logutils.redis.RedisQueueListener + :members: + diff --git a/logutils/__init__.py b/logutils/__init__.py index 377da43..f9a677c 100644 --- a/logutils/__init__.py +++ b/logutils/__init__.py @@ -10,7 +10,7 @@ of Python, and so are packaged here. import logging from string import Template -__version__ = '0.3.1' +__version__ = '0.3.2' class NullHandler(logging.Handler): """ diff --git a/logutils/queue.py b/logutils/queue.py index 03ed570..39be637 100644 --- a/logutils/queue.py +++ b/logutils/queue.py @@ -214,6 +214,15 @@ class QueueListener(object): except queue.Empty: break + def enqueue_sentinel(self): + """ + Writes a sentinel to the queue to tell the listener to quit. This + implementation uses ``put_nowait()``. You may want to override this + method if you want to use timeouts or work with custom queue + implementations. + """ + self.queue.put_nowait(self._sentinel) + def stop(self): """ Stop the listener. @@ -223,7 +232,7 @@ class QueueListener(object): may be some records still left on the queue, which won't be processed. """ self._stop.set() - self.queue.put_nowait(self._sentinel) + self.enqueue_sentinel() self._thread.join() self._thread = None diff --git a/logutils/redis.py b/logutils/redis.py new file mode 100644 index 0000000..2d7c6bf --- /dev/null +++ b/logutils/redis.py @@ -0,0 +1,72 @@ +""" +This module contains classes which help you work with Redis queues. +""" + +from logutils.queue import QueueHandler, QueueListener +try: + import cPickle as pickle +except ImportError: + import pickle + +class RedisQueueHandler(QueueHandler): + """ + A QueueHandler implementation which pushes pickled + records to a Redis queue using a specified key. + + :param key: The key to use for the queue. Defaults to + "python.logging". + :param redis: If specified, this instance is used to + communicate with a Redis instance. + :param limit: If specified, the queue is restricted to + have only this many elements. + """ + def __init__(self, key='python.logging', redis=None, limit=0): + if redis is None: + from redis import Redis + redis = Redis() + self.key = key + assert limit >= 0 + self.limit = limit + QueueHandler.__init__(self, redis) + + def enqueue(self, record): + s = pickle.dumps(vars(record)) + self.queue.rpush(self.key, s) + if self.limit: + self.queue.ltrim(self.key, -self.limit, -1) + +class RedisQueueListener(QueueListener): + """ + A QueueListener implementation which fetches pickled + records from a Redis queue using a specified key. + + :param key: The key to use for the queue. Defaults to + "python.logging". + :param redis: If specified, this instance is used to + communicate with a Redis instance. + """ + def __init__(self, *handlers, **kwargs): + redis = kwargs.get('redis') + if redis is None: + from redis import Redis + redis = Redis() + self.key = kwargs.get('key', 'python.logging') + QueueListener.__init__(self, redis, *handlers) + + def dequeue(self, block): + """ + Dequeue and return a record. + """ + if block: + s = self.queue.blpop(self.key)[1] + else: + s = self.queue.lpop(self.key) + if not s: + record = None + else: + record = pickle.loads(s) + return record + + def enqueue_sentinel(self): + self.queue.rpush(self.key, '') + diff --git a/setup.py b/setup.py index b0e95a9..3665a13 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,4 @@ # -*- coding: utf-8 -*- -# -# Copyright (C) 2010 Vinay Sajip. All Rights Reserved. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose and without fee is hereby granted, -# provided that the above copyright notice appear in all copies and that -# both that copyright notice and this permission notice appear in -# supporting documentation, and that the name of Vinay Sajip -# not be used in advertising or publicity pertaining to distribution -# of the software without specific, written prior permission. -# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING -# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL -# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR -# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER -# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# import distutils.core import logutils @@ -60,9 +43,9 @@ distutils.core.setup( url='http://code.google.com/p/logutils/', description='Logging utilities', long_description = description(), - license='New BSD', + license='Copyright (C) 2010-2011 by Vinay Sajip. All Rights Reserved. See LICENSE.txt for license.', classifiers=[ - 'Development Status :: 5 - Production', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', diff --git a/tests/logutil_tests.py b/tests/logutil_tests.py index 8f1bfb6..330594c 100644 --- a/tests/logutil_tests.py +++ b/tests/logutil_tests.py @@ -4,6 +4,10 @@ from test_dictconfig import ConfigDictTest from test_queue import QueueTest from test_formatter import FormatterTest from test_messages import MessageTest +try: + from test_redis import RedisQueueTest +except ImportError: + pass # The adapter won't work in < 2.5 because the "extra" parameter used by it # only appeared in 2.5 :-( diff --git a/tests/mytest.py b/tests/mytest.py new file mode 100644 index 0000000..1c3fdd6 --- /dev/null +++ b/tests/mytest.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from logutils.testing import TestHandler, Matcher + +class MyTestHandler(TestHandler): + def __init__(self): + TestHandler.__init__(self, Matcher()) + + diff --git a/tests/test_dictconfig.py b/tests/test_dictconfig.py index d333808..42647a2 100644 --- a/tests/test_dictconfig.py +++ b/tests/test_dictconfig.py @@ -483,6 +483,39 @@ class ConfigDictTest(unittest.TestCase): }, } + # As config10, but declaring a handler in a module using + # absolute imports + config11 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'filters' : { + 'filt1' : { + 'name' : 'compiler.parser', + }, + }, + 'handlers' : { + 'hand1' : { + '()': 'mytest.MyTestHandler', + 'formatter': 'form1', + 'filters' : ['filt1'], + } + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'filters' : ['filt1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + def apply_config(self, conf): dictConfig(conf) @@ -660,3 +693,9 @@ class ConfigDictTest(unittest.TestCase): dict(levelname='ERROR', message='4'), ])) + def test_config_11_ok(self): + self.apply_config(self.config11) + h = logging.getLogger().handlers[0] + self.assertEqual(h.__module__, 'mytest') + self.assertEqual(h.__class__.__name__, 'MyTestHandler') + diff --git a/tests/test_redis.py b/tests/test_redis.py new file mode 100644 index 0000000..cc319cb --- /dev/null +++ b/tests/test_redis.py @@ -0,0 +1,96 @@ +import logging +from logutils.testing import TestHandler, Matcher +from logutils.redis import RedisQueueHandler, RedisQueueListener +from redis import Redis +import socket +import subprocess +import time +import unittest + +class QueueListener(RedisQueueListener): + def dequeue(self, block): + record = RedisQueueListener.dequeue(self, block) + if record: + record = logging.makeLogRecord(record) + return record + +class RedisQueueTest(unittest.TestCase): + def setUp(self): + self.handler = h = TestHandler(Matcher()) + self.logger = l = logging.getLogger() + self.server = subprocess.Popen(['redis-server'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.wait_for_server() + self.queue = q = Redis() + self.qh = qh = RedisQueueHandler(redis=q) + self.ql = ql = QueueListener(h, redis=q) + ql.start() + l.addHandler(qh) + + def tearDown(self): + self.logger.removeHandler(self.qh) + self.qh.close() + self.handler.close() + self.server.terminate() + + def wait_for_server(self): + maxtime = time.time() + 2 # 2 seconds to wait for server + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + while time.time() < maxtime: + try: + sock.connect(('localhost', 6379)) + break + except socket.error: + pass + if time.time() >= maxtime: + raise Exception('unable to connect to Redis server') + sock.close() + + def test_simple(self): + "Simple test of queue handling and listening." + # Just as a demo, let's log some messages. + # Only one should show up in the log. + self.logger.debug("This won't show up.") + self.logger.info("Neither will this.") + self.logger.warning("But this will.") + self.ql.stop() #ensure all records have come through. + h = self.handler + #import pdb; pdb.set_trace() + self.assertTrue(h.matches(levelno=logging.WARNING)) + self.assertFalse(h.matches(levelno=logging.DEBUG)) + self.assertFalse(h.matches(levelno=logging.INFO)) + + def test_partial(self): + "Test of partial matching through queues." + # Just as a demo, let's log some messages. + # Only one should show up in the log. + self.logger.debug("This won't show up.") + self.logger.info("Neither will this.") + self.logger.warning("But this will.") + self.ql.stop() #ensure all records have come through. + h = self.handler + self.assertTrue(h.matches(msg="ut th")) # from "But this will" + self.assertTrue(h.matches(message="ut th")) # from "But this will" + self.assertFalse(h.matches(message="either")) + self.assertFalse(h.matches(message="won't")) + + def test_multiple(self): + "Test of matching multiple values through queues." + # Just as a demo, let's log some messages. + # Only one should show up in the log. + self.logger.debug("This won't show up.") + self.logger.info("Neither will this.") + self.logger.warning("But this will.") + self.logger.error("And so will this.") + self.ql.stop() #ensure all records have come through. + h = self.handler + self.assertTrue(h.matches(levelno=logging.WARNING, + message='ut thi')) + self.assertTrue(h.matches(levelno=logging.ERROR, + message='nd so wi')) + self.assertFalse(h.matches(levelno=logging.INFO)) + +if __name__ == '__main__': + unittest.main() + -- cgit v1.2.1