From 55bff3c77905fee488d2fbff084622e4d0afc15f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 31 May 2015 17:12:37 +0300 Subject: Move memory dumping method to util.py --- cliapp/__init__.py | 1 + cliapp/app.py | 68 ++------------------------------------ cliapp/util.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ without-tests | 1 + 4 files changed, 99 insertions(+), 66 deletions(-) create mode 100644 cliapp/util.py diff --git a/cliapp/__init__.py b/cliapp/__init__.py index 05e8185..a8e4f78 100644 --- a/cliapp/__init__.py +++ b/cliapp/__init__.py @@ -30,6 +30,7 @@ Homepage: http://liw.fi/cliapp/ __version__ = '1.20150305' +from .util import MemoryProfileDumper from .fmt import TextFormat from .app import Application, AppException from .settings import (Settings, log_group_name, config_group_name, diff --git a/cliapp/app.py b/cliapp/app.py index ddc273a..9812f0d 100644 --- a/cliapp/app.py +++ b/cliapp/app.py @@ -16,7 +16,6 @@ import errno -import gc import inspect import logging import logging.handlers @@ -24,7 +23,6 @@ import os import StringIO import sys import traceback -import time import platform import textwrap @@ -115,9 +113,7 @@ class Application(object): self.plugin_subdir = 'plugins' - # For meliae memory dumps. - self.memory_dump_counter = 0 - self.last_memory_dump = 0 + self.memory_profile_dumper = cliapp.MemoryProfileDumper(self.settings) # For process duration. self._started = os.times()[-1] @@ -638,65 +634,5 @@ class Application(object): def runcmd_unchecked(self, *args, **kwargs): # pragma: no cover return cliapp.runcmd_unchecked(*args, **kwargs) - def _vmrss(self): # pragma: no cover - '''Return current resident memory use, in KiB.''' - if platform.system() != 'Linux': - return 0 - try: - f = open('/proc/self/status') - except IOError: - return 0 - rss = 0 - for line in f: - if line.startswith('VmRSS'): - rss = line.split()[1] - f.close() - return rss - def dump_memory_profile(self, msg): # pragma: no cover - '''Log memory profiling information. - - Get the memory profiling method from the dump-memory-profile - setting, and log the results at DEBUG level. ``msg`` is a - message the caller provides to identify at what point the profiling - happens. - - ''' - - kind = self.settings['dump-memory-profile'] - interval = self.settings['memory-dump-interval'] - - if kind == 'none': - return - - now = time.time() - if self.last_memory_dump + interval > now: - return - self.last_memory_dump = now - - # Log wall clock and CPU times for self, children. - utime, stime, cutime, cstime, elapsed_time = os.times() - duration = elapsed_time - self._started - logging.debug('process duration: %s s', duration) - logging.debug('CPU time, in process: %s s', utime) - logging.debug('CPU time, in system: %s s', stime) - logging.debug('CPU time, in children: %s s', cutime) - logging.debug('CPU time, in system for children: %s s', cstime) - - logging.debug('dumping memory profiling data: %s', msg) - logging.debug('VmRSS: %s KiB', self._vmrss()) - - if kind == 'simple': - return - - # These are fairly expensive operations, so we only log them - # if we're doing expensive stuff anyway. - logging.debug('# objects: %d', len(gc.get_objects())) - logging.debug('# garbage: %d', len(gc.garbage)) - - if kind == 'meliae': - filename = 'obnam-%d.meliae', self.memory_dump_counter - logging.debug('memory profile: see %s', filename) - from meliae import scanner - scanner.dump_all_objects(filename) - self.memory_dump_counter += 1 + self.memory_profile_dumper.dump_memory_profile(msg) diff --git a/cliapp/util.py b/cliapp/util.py new file mode 100644 index 0000000..bbdb051 --- /dev/null +++ b/cliapp/util.py @@ -0,0 +1,95 @@ +# Copyright 2011-2015 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import gc +import logging +import os +import platform +import time + + +class MemoryProfileDumper(object): + + def __init__(self, settings): + self.settings = settings + self.last_memory_dump = 0 + self.memory_dump_counter = 0 + self.started = time.time() + + def dump_memory_profile(self, msg): + '''Log memory profiling information. + + Get the memory profiling method from the dump-memory-profile + setting, and log the results at DEBUG level. ``msg`` is a + message the caller provides to identify at what point the profiling + happens. + + ''' + + kind = self.settings['dump-memory-profile'] + interval = self.settings['memory-dump-interval'] + + if kind == 'none': + return + + now = time.time() + if self.last_memory_dump + interval > now: + return + self.last_memory_dump = now + + # Log wall clock and CPU times for self, children. + utime, stime, cutime, cstime, elapsed_time = os.times() + duration = elapsed_time - self.started + logging.debug('process duration: %s s', duration) + logging.debug('CPU time, in process: %s s', utime) + logging.debug('CPU time, in system: %s s', stime) + logging.debug('CPU time, in children: %s s', cutime) + logging.debug('CPU time, in system for children: %s s', cstime) + + logging.debug('dumping memory profiling data: %s', msg) + logging.debug('VmRSS: %s KiB', self._vmrss()) + + if kind == 'simple': + return + + # These are fairly expensive operations, so we only log them + # if we're doing expensive stuff anyway. + logging.debug('# objects: %d', len(gc.get_objects())) + logging.debug('# garbage: %d', len(gc.garbage)) + + if kind == 'meliae': + filename = 'obnam-%d.meliae', self.memory_dump_counter + logging.debug('memory profile: see %s', filename) + from meliae import scanner + scanner.dump_all_objects(filename) + self.memory_dump_counter += 1 + + def _vmrss(self): # pragma: no cover + '''Return current resident memory use, in KiB.''' + if platform.system() != 'Linux': + return 0 + try: + f = open('/proc/self/status') + except IOError: + return 0 + rss = 0 + for line in f: + if line.startswith('VmRSS'): + rss = line.split()[1] + f.close() + return rss diff --git a/without-tests b/without-tests index c6dc59d..2747120 100644 --- a/without-tests +++ b/without-tests @@ -1,4 +1,5 @@ ./cliapp/__init__.py +./cliapp/util.py ./example.py ./example2.py ./setup.py -- cgit v1.2.1