diff options
author | Ric Holland <richard.holland@codethink.co.uk> | 2013-03-15 10:17:23 +0000 |
---|---|---|
committer | Ric Holland <richard.holland@codethink.co.uk> | 2013-03-15 10:17:23 +0000 |
commit | 7389d9836deaba6acb9cd3032351da7b6a0227ff (patch) | |
tree | e1aa511f85963a98c8c17d4289de2394a465fd18 | |
parent | 4b4b0b47372543572c3d2911c8bde87e4db8745b (diff) | |
parent | 9b8176d4c122d15eec80f23ebe70954045e7bbde (diff) | |
download | cliapp-7389d9836deaba6acb9cd3032351da7b6a0227ff.tar.gz |
Merge remote-tracking branch 'origin/trunk' into baserock/richardholland/cliapp-updatebaserock/richardholland/cliapp-update
-rw-r--r-- | NEWS | 62 | ||||
-rw-r--r-- | README | 3 | ||||
-rw-r--r-- | cliapp.5 | 57 | ||||
-rw-r--r-- | cliapp/__init__.py | 8 | ||||
-rw-r--r-- | cliapp/app.py | 234 | ||||
-rw-r--r-- | cliapp/app_tests.py | 28 | ||||
-rw-r--r-- | cliapp/fmt.py | 125 | ||||
-rw-r--r-- | cliapp/fmt_tests.py | 59 | ||||
-rw-r--r-- | cliapp/genman.py | 30 | ||||
-rw-r--r-- | cliapp/runcmd.py | 77 | ||||
-rw-r--r-- | cliapp/runcmd_tests.py | 20 | ||||
-rw-r--r-- | cliapp/settings.py | 223 | ||||
-rw-r--r-- | cliapp/settings_tests.py | 12 | ||||
-rw-r--r-- | debian/changelog | 12 | ||||
-rw-r--r-- | example.py | 4 | ||||
-rw-r--r-- | example3.py | 48 | ||||
-rw-r--r-- | example4.py | 53 | ||||
-rw-r--r-- | without-tests | 2 |
18 files changed, 901 insertions, 156 deletions
@@ -1,14 +1,74 @@ NEWS for cliapp =============== -Version UNRELEASED +Version 1.20130313 ------------------ +* Add `cliapp.Application.compute_setting_values` method. This allows + the application to have settings with values that are computed after + configuration files and the command line are parsed. +* Cliapp now logs the Python version at startup, to aid debugging. +* `cliapp.runcmd` now logs much less during execution of a command. The + verbose logging was useful while developing pipeline support, but has + now not been useful for months. +* More default settings and options have an option group now, making + `--help` output prettier. +* The `--help` output and the output of the `help` subcommand now only + list summaries for subcommands. The full documentation for a subcommand + can be seen by giving the name of the subcommand to `help`. +* Logging setup is now more overrideable. The `setup_logging` method + calls `setup_logging_handler_for_syslog`, + `setup_logging_handler_for_syslog`, or + `setup_logging_handler_to_file`, and the last one calls + `setup_logging_format` and `setup_logging_timestamp` to create the + format strings for messages and timestamps. This allows applications + to add, for example, more detailed timestamps easily. +* The process and system CPU times, and those of the child processes, + and the process wall clock duration, are now logged when the memory + profiling information is logged. +* Subcommands added with `add_subcommand` may now have aliases. + Subcommands defined using `Application` class methods named `cmd_*` + cannot have aliases. +* Settings and subcommands may now be hidden from `--help` and `help` + output. New option `--help-all` and new subcommand `help-all` show + everything. +* cliapp(5) now explains how `--generate-manpage` is used. Thanks to + Enrico Zini for the suggestion. +* New function `cliapp.ssh_runcmd` for executing a command remotely + over ssh. The function automatically shell-quotes the argv array + given to it so that arguments with spaces and other shell meta-characters + work over ssh. +* New function `cliapp.shell_quote` quotes strings for passing as shell + arguments. +* `cliapp.runcmd` now has a new keyword argument: `log_error`. If set to + false, errors are not logged. Defaults to true. + +Bug fixes: + +* The process title is now set only if `/proc/self/comm` exists. + Previously, on the kernel in Debian squeeze (2.6.32), setting the + process title would fail, and the error would be logged to the + terminal. Reported by William Boughton. +* A setting may no longer have a default value of None. + +Version 1.20121216 +------------------ + +* Options in option groups are now included in manual page SYNOPSIS and + OPTIONS sections. * `--log=syslog` message format improvement by Daniel Silverstone. No longer includes a timestamp, since syslog adds it anyway. Also, the process name is now set on Linux. * Make the default subcommand argument synopsis be an empty string, instead of None. Reported by Sam Thursfield. +* Meliae memory dumping support has been fixed. Reported by Joey Hess. +* Memory profiling reports can now be done at minimum intervals in + seconds, rather than every time the code asks for them. This can + reduce the overhead of memory profiling quite a lot. +* If there are any subcommands, cliapp now adds a subcommand called `help`, + unless one already exists. +* For every boolean setting foo, there will no be a --no-foo option to be + used on the command line. Version 1.20120929 ------------------ @@ -22,9 +22,6 @@ cliapp version numbers are of the form `API.DATE`, where `API` starts at 1, and gets incremented if the application API changes in an incompatible way. `DATE` is the date of the release. -The `cliapp.__version__` string is set to `1.DEVEL` in version control, -to differentiate development versions and release versions. - Legalese -------- @@ -130,6 +130,20 @@ attack-kittens = no .PP This turns the verbose setting on, but does not launch attack kittens. +.PP +For every boolean setting, +two command line options are added. +If the setting is called +.IR foo , +the option +.I \-\-foo +will turn the setting on, +and +.I \-\-no\-foo +will turn it off. +The negation is only usable on the command line: +its purpose is to allow the command line to override a setting from the +configuration file. .SS "Logging and log files" Programs using .B cliapp @@ -171,6 +185,49 @@ as set by the application code or determined by automatically. The value of the environment variable is the name of the file to which the resulting profile is to be written. +.SS "Manual page generation" +.B cliapp +can generate parts of a manual page: +the +.I SYNOPSIS +and +.I OPTIONS +sections. +It fills these in automatically based on the subcommand and settings +that a program supports. +Use the +.BR \-\-generate\-manpage =\fIFILE +option, +which is added automatically by +.BR cliapp . +The +.I FILE +is a manual page marked up using +the +.B -man +macros for +.BR troff (1). +It should have empty +.I SYNOPSIS +and +.I OPTIONS +sections, +and +.B cliapp +will fill them in. +The output it to the standard output. +.PP +For example: +.PP +.RS +foo --generate-manpage=foo.1.in > foo.1 +.RE +.PP +You would keep the source code for the manual page in +.I foo.1.in +and have your Makefile produce +.I foo.1 +as shown above. .SH FILES .B cliapp reads a list of configuration files at startup, diff --git a/cliapp/__init__.py b/cliapp/__init__.py index 4c92ef9..ea30ac2 100644 --- a/cliapp/__init__.py +++ b/cliapp/__init__.py @@ -15,11 +15,13 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -__version__ = '1.20120929' +__version__ = '1.20130313' -from settings import Settings -from runcmd import runcmd, runcmd_unchecked +from fmt import TextFormat +from settings import (Settings, log_group_name, config_group_name, + perf_group_name) +from runcmd import runcmd, runcmd_unchecked, shell_quote, ssh_runcmd from app import Application, AppException # The plugin system diff --git a/cliapp/app.py b/cliapp/app.py index 9e9e28b..3c07310 100644 --- a/cliapp/app.py +++ b/cliapp/app.py @@ -24,7 +24,9 @@ import os import StringIO import sys import traceback +import time import platform +import textwrap import cliapp @@ -99,6 +101,8 @@ class Application(object): self.cmd_synopsis = {} self.subcommands = {} + self.subcommand_aliases = {} + self.hidden_subcommands = set() for method_name in self._subcommand_methodnames(): cmd = self._unnormalize_cmd(method_name) self.subcommands[cmd] = getattr(self, method_name) @@ -110,6 +114,13 @@ class Application(object): self.plugin_subdir = 'plugins' + # For meliae memory dumps. + self.memory_dump_counter = 0 + self.last_memory_dump = 0 + + # For process duration. + self._started = os.times()[-1] + def add_settings(self): '''Add application specific settings.''' @@ -143,12 +154,10 @@ class Application(object): return ''.join(x.upper() if x in ok else '_' for x in basename) def _set_process_name(self): # pragma: no cover - if platform.system() == 'Linux': - try: - with open('/proc/self/comm', 'w', 0) as f: - f.write(self.settings.progname[:15]) - except IOError, e: - logging.warning(str(e)) + comm = '/proc/self/comm' + if platform.system() == 'Linux' and os.path.exists(comm): + with open('/proc/self/comm', 'w', 0) as f: + f.write(self.settings.progname[:15]) def _run(self, args=None, stderr=sys.stderr, log=logging.critical): try: @@ -163,6 +172,8 @@ class Application(object): # config file settings. self.setup() self.enable_plugins() + if self.subcommands: + self.add_default_subcommands() args = sys.argv[1:] if args is None else args self.parse_args(args, configs_only=True) self.settings.load_configs() @@ -215,7 +226,22 @@ class Application(object): logging.info('%s version %s ends normally' % (self.settings.progname, self.settings.version)) - def add_subcommand(self, name, func, arg_synopsis=None): + def compute_setting_values(self, settings): + '''Compute setting values after configs and options are parsed. + + You can override this method to implement a default value for + a setting that is dependent on another setting. For example, + you might have settings "url" and "protocol", where protocol + gets set based on the schema of the url, unless explicitly + set by the user. So if the user sets just the url, to + "http://www.example.com/", the protocol would be set to + "http". If the user sets both url and protocol, the protocol + does not get modified by compute_setting_values. + + ''' + + def add_subcommand( + self, name, func, arg_synopsis=None, aliases=None, hidden=False): '''Add a subcommand. Normally, subcommands are defined by add ``cmd_foo`` methods @@ -231,6 +257,43 @@ class Application(object): if name not in self.subcommands: self.subcommands[name] = func self.cmd_synopsis[name] = arg_synopsis + self.subcommand_aliases[name] = aliases or [] + if hidden: # pragma: no cover + self.hidden_subcommands.add(name) + + def add_default_subcommands(self): + if 'help' not in self.subcommands: + self.add_subcommand('help', self.help) + if 'help-all' not in self.subcommands: + self.add_subcommand('help-all', self.help_all) + + def _help_helper(self, args, show_all): # pragma: no cover + try: + width = int(os.environ.get('COLUMNS', '78')) + except ValueError: + width = 78 + + fmt = cliapp.TextFormat(width=width) + + if args: + usage = self._format_usage_for(args[0]) + description = fmt.format(self._format_subcommand_help(args[0])) + text = '%s\n\n%s' % (usage, description) + else: + usage = self._format_usage(all=show_all) + description = fmt.format(self._format_description(all=show_all)) + text = '%s\n\n%s' % (usage, description) + + text = self.settings.progname.join(text.split('%prog')) + self.output.write(text) + + def help(self, args): # pragma: no cover + '''Print help.''' + self._help_helper(args, False) + + def help_all(self, args): # pragma: no cover + '''Print help, including hidden subcommands.''' + self._help_helper(args, True) def _subcommand_methodnames(self): return [x @@ -245,63 +308,57 @@ class Application(object): assert method.startswith('cmd_') return method[len('cmd_'):].replace('_', '-') - def _format_usage(self): + def _format_usage(self, all=False): '''Format usage, possibly also subcommands, if any.''' if self.subcommands: lines = [] prefix = 'Usage:' for cmd in sorted(self.subcommands.keys()): - args = self.cmd_synopsis.get(cmd, '') or '' - lines.append('%s %%prog [options] %s %s' % (prefix, cmd, args)) - prefix = ' ' * len(prefix) + if all or cmd not in self.hidden_subcommands: + args = self.cmd_synopsis.get(cmd, '') or '' + lines.append( + '%s %%prog [options] %s %s' % (prefix, cmd, args)) + prefix = ' ' * len(prefix) return '\n'.join(lines) else: return None - def _format_description(self): + def _format_usage_for(self, cmd): # pragma: no cover + args = self.cmd_synopsis.get(cmd, '') or '' + return 'Usage: %%prog [options] %s %s' % (cmd, args) + + def _format_description(self, all=False): '''Format OptionParser description, with subcommand support.''' if self.subcommands: - paras = [] + summaries = [] for cmd in sorted(self.subcommands.keys()): - paras.append(self._format_subcommand_description(cmd)) - cmd_desc = '\n\n'.join(paras) - return '%s\n\n%s' % (self._description or '', cmd_desc) + if all or cmd not in self.hidden_subcommands: + summaries.append(self._format_subcommand_summary(cmd)) + cmd_desc = ''.join(summaries) + return '%s\n%s' % (self._description or '', cmd_desc) else: return self._description - def _format_subcommand_description(self, cmd): # pragma: no cover - - def remove_empties(lines): - while lines and not lines[0].strip(): - del lines[0] - - def split_para(lines): - para = [] - while lines and lines[0].strip(): - para.append(lines[0].strip()) - del lines[0] - return para - - indent = ' ' * 4 + def _format_subcommand_summary(self, cmd): # pragma: no cover method = self.subcommands[cmd] doc = method.__doc__ or '' lines = doc.splitlines() - remove_empties(lines) if lines: - heading = '* %s -- %s' % (cmd, lines[0]) - result = [heading] - del lines[0] - remove_empties(lines) - while lines: - result.append('') - para_lines = split_para(lines) - para_text = ' '.join(para_lines) - result.append(para_text) - remove_empties(lines) - return '\n'.join(result) + summary = lines[0].strip() else: - return '* %s' % cmd - + summary = '' + return '* %%prog %s: %s\n' % (cmd, summary) + + def _format_subcommand_help(self, cmd): # pragma: no cover + method = self.subcommands[cmd] + doc = method.__doc__ or '' + t = doc.split('\n', 1) + if len(t) == 1: + return doc + else: + first, rest = t + return first + '\n' + textwrap.dedent(rest) + def setup_logging(self): # pragma: no cover '''Set up logging.''' @@ -317,11 +374,7 @@ class Application(object): level = levels.get(level_name, logging.INFO) if self.settings['log'] == 'syslog': - handler = logging.handlers.SysLogHandler(address='/dev/log') - progname = '%%'.join(self.settings.progname.split('%')) - fmt = progname + ": %(levelname)s %(message)s" - formatter = logging.Formatter(fmt) - handler.setFormatter(formatter) + handler = self.setup_logging_handler_for_syslog() elif self.settings['log'] and self.settings['log'] != 'none': handler = LogHandler( self.settings['log'], @@ -334,7 +387,7 @@ class Application(object): formatter = logging.Formatter(fmt, datefmt) handler.setFormatter(formatter) else: - handler = logging.FileHandler('/dev/null') + handler = self.setup_logging_handler_to_none() # reduce amount of pointless I/O level = logging.FATAL @@ -342,6 +395,47 @@ class Application(object): logger.addHandler(handler) logger.setLevel(level) + def setup_logging_handler_for_syslog(self): # pragma: no cover + '''Setup a logging.Handler for logging to syslog.''' + + handler = logging.handlers.SysLogHandler(address='/dev/log') + progname = '%%'.join(self.settings.progname.split('%')) + fmt = progname + ": %(levelname)s %(message)s" + formatter = logging.Formatter(fmt) + handler.setFormatter(formatter) + + return handler + + def setup_logging_handler_to_none(self): # pragma: no cover + '''Setup a logging.Handler that does not log anything anywhere.''' + + handler = logging.FileHandler('/dev/null') + return handler + + def setup_logging_handler_to_file(self): # pragma: no cover + '''Setup a logging.Handler for logging to a named file.''' + + handler = LogHandler( + self.settings['log'], + perms=int(self.settings['log-mode'], 8), + maxBytes=self.settings['log-max'], + backupCount=self.settings['log-keep'], + delay=False) + fmt = self.setup_logging_format() + datefmt = self.setup_logging_timestamp() + formatter = logging.Formatter(fmt, datefmt) + handler.setFormatter(formatter) + + return handler + + def setup_logging_format(self): # pragma: no cover + '''Return format string for log messages.''' + return '%(asctime)s %(levelname)s %(message)s' + + def setup_logging_timestamp(self): # pragma: no cover + '''Return timestamp format string for log message.''' + return '%Y-%m-%d %H:%M:%S' + def log_config(self): logging.info('%s version %s starts' % (self.settings.progname, self.settings.version)) @@ -353,6 +447,7 @@ class Application(object): f = StringIO.StringIO() cp.write(f) logging.debug('Config:\n%s' % f.getvalue()) + logging.debug('Python version: %s' % sys.version) def app_directory(self): '''Return the directory where the application class is defined. @@ -390,9 +485,10 @@ class Application(object): ''' - return self.settings.parse_args(args, configs_only=configs_only, - arg_synopsis=self.arg_synopsis, - cmd_synopsis=self.cmd_synopsis) + return self.settings.parse_args( + args, configs_only=configs_only, arg_synopsis=self.arg_synopsis, + cmd_synopsis=self.cmd_synopsis, + compute_setting_values=self.compute_setting_values) def setup(self): '''Prepare for process_args. @@ -427,11 +523,18 @@ class Application(object): if self.subcommands: if not args: raise SystemExit('must give subcommand') - if args[0] in self.subcommands: - method = self.subcommands[args[0]] - method(args[1:]) - else: - raise SystemExit('unknown subcommand %s' % args[0]) + + cmd = args[0] + if cmd not in self.subcommands: + for name in self.subcommand_aliases: + if cmd in self.subcommand_aliases[name]: + cmd = name + break + else: + raise SystemExit('unknown subcommand %s' % args[0]) + + method = self.subcommands[cmd] + method(args[1:]) else: self.process_inputs(args) @@ -523,9 +626,24 @@ class Application(object): ''' kind = self.settings['dump-memory-profile'] + interval = self.settings['memory-dump-interval'] if kind == 'none': return + + now = time.time() + if self.last_memory_dump + interval > now: + return + self.last_memory_dump = now + + # Log wall clock and CPU times for self, children. + utime, stime, cutime, cstime, elapsed_time = os.times() + duration = elapsed_time - self._started + logging.debug('process duration: %s s' % duration) + logging.debug('CPU time, in process: %s s' % utime) + logging.debug('CPU time, in system: %s s' % stime) + logging.debug('CPU time, in children: %s s' % cutime) + logging.debug('CPU time, in system for children: %s s' % cstime) logging.debug('dumping memory profiling data: %s' % msg) logging.debug('VmRSS: %s KiB' % self._vmrss()) diff --git a/cliapp/app_tests.py b/cliapp/app_tests.py index 6783514..674e033 100644 --- a/cliapp/app_tests.py +++ b/cliapp/app_tests.py @@ -274,8 +274,8 @@ class ApplicationTests(unittest.TestCase): class DummySubcommandApp(cliapp.Application): - def cmd_help(self, args): - self.help_called = True + def cmd_foo(self, args): + self.foo_called = True class SubcommandTests(unittest.TestCase): @@ -285,10 +285,10 @@ class SubcommandTests(unittest.TestCase): self.trash = StringIO.StringIO() def test_lists_subcommands(self): - self.assertEqual(self.app._subcommand_methodnames(), ['cmd_help']) + self.assertEqual(self.app._subcommand_methodnames(), ['cmd_foo']) def test_normalizes_subcommand(self): - self.assertEqual(self.app._normalize_cmd('help'), 'cmd_help') + self.assertEqual(self.app._normalize_cmd('foo'), 'cmd_foo') self.assertEqual(self.app._normalize_cmd('foo-bar'), 'cmd_foo_bar') def test_raises_error_for_no_subcommand(self): @@ -300,8 +300,20 @@ class SubcommandTests(unittest.TestCase): stderr=self.trash, log=devnull) def test_calls_subcommand_method(self): - self.app.run(['help'], stderr=self.trash, log=devnull) - self.assert_(self.app.help_called) + self.app.run(['foo'], stderr=self.trash, log=devnull) + self.assert_(self.app.foo_called) + + def test_calls_subcommand_method_via_alias(self): + self.bar_called = False + def bar(*args): + self.bar_called = True + self.app.add_subcommand('bar', bar, aliases=['yoyo']) + self.app.run(['yoyo'], stderr=self.trash, log=devnull) + self.assertTrue(self.bar_called) + + def test_adds_default_subcommand_help(self): + self.app.run(['foo'], stderr=self.trash, log=devnull) + self.assertTrue('help' in self.app.subcommands) class ExtensibleSubcommandTests(unittest.TestCase): @@ -314,6 +326,6 @@ class ExtensibleSubcommandTests(unittest.TestCase): def test_adds_subcommand(self): help = lambda args: None - self.app.add_subcommand('help', help) - self.assertEqual(self.app.subcommands, {'help': help}) + self.app.add_subcommand('foo', help) + self.assertEqual(self.app.subcommands, {'foo': help}) diff --git a/cliapp/fmt.py b/cliapp/fmt.py new file mode 100644 index 0000000..72a9265 --- /dev/null +++ b/cliapp/fmt.py @@ -0,0 +1,125 @@ +# Copyright (C) 2013 Lars Wirzenius +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Simplistic text re-formatter. + +This module format text, paragraph by paragraph, so it is somewhat +nice-looking, with no line too long, and short lines joined. In +other words, like what the textwrap library does. However, it +extends textwrap by recognising bulleted lists. + +''' + + +import textwrap + + +class Paragraph(object): + + def __init__(self): + self._lines = [] + + def append(self, line): + self._lines.append(line) + + def _oneliner(self): + return ' '.join(' '.join(x.split()) for x in self._lines) + + def fill(self, width): + filled = textwrap.fill(self._oneliner(), width=width) + return filled + + +class BulletPoint(Paragraph): + + def fill(self, width): + text = self._oneliner() + assert text.startswith('* ') + filled = textwrap.fill(text[2:], width=width - 2) + lines = [' %s' % x for x in filled.splitlines(True)] + lines[0] = '* %s' % lines[0][2:] + return ''.join(lines) + + +class EmptyLine(Paragraph): + + def fill(self, width): + return '' + + +class TextFormat(object): + + def __init__(self, width=78): + self._width = width + + def format(self, text): + '''Return input string, but formatted nicely.''' + + filled_paras = [] + for para in self._paragraphs(text): + filled_paras.append(para.fill(self._width)) + filled = '\n'.join(filled_paras) + if text and not filled.endswith('\n'): + filled += '\n' + return filled + + def _paragraphs(self, text): + + def is_empty(line): + return line.strip() == '' + + def is_bullet(line): + return line.startswith('* ') + + def is_continuation(line): + return line.startswith(' ') + + current = None + in_list = False + for line in text.splitlines(True): + if in_list and is_continuation(line): + assert current is not None + current.append(line) + elif is_bullet(line): + if current: + yield current + if not in_list: + yield EmptyLine() + current = BulletPoint() + current.append(line) + in_list = True + elif is_empty(line): + if current: + yield current + yield EmptyLine() + current = None + in_list = False + else: + if in_list: + yield current + yield EmptyLine() + current = None + + if not current: + current = Paragraph() + current.append(line) + in_list = False + + if current: + yield current + + diff --git a/cliapp/fmt_tests.py b/cliapp/fmt_tests.py new file mode 100644 index 0000000..0e109e7 --- /dev/null +++ b/cliapp/fmt_tests.py @@ -0,0 +1,59 @@ +# Copyright (C) 2013 Lars Wirzenius +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import unittest + +import cliapp + + +class TextFormatTests(unittest.TestCase): + + def setUp(self): + self.fmt = cliapp.TextFormat(width=10) + + def test_returns_empty_string_for_empty_string(self): + self.assertEqual(self.fmt.format(''), '') + + def test_returns_short_one_line_paragraph_as_is(self): + self.assertEqual(self.fmt.format('foo bar'), 'foo bar\n') + + def test_collapse_multiple_spaces_into_one(self): + self.assertEqual(self.fmt.format('foo bar'), 'foo bar\n') + + def test_wraps_long_line(self): + self.assertEqual(self.fmt.format('foobar word'), 'foobar\nword\n') + + def test_handles_paragraphs(self): + self.assertEqual( + self.fmt.format('foo\nbar\n\nyo\nyo\n'), + 'foo bar\n\nyo yo\n') + + def test_collapses_more_than_two_empty_lines(self): + self.assertEqual( + self.fmt.format('foo\nbar\n\n\n\n\n\n\n\n\n\nyo\nyo\n'), + 'foo bar\n\nyo yo\n') + + def test_handles_bulleted_lists(self): + self.assertEqual( + self.fmt.format('foo\nbar\n\n* yo\n* a\n and b\n\nword'), + 'foo bar\n\n* yo\n* a and b\n\nword\n') + + def test_handles_bulleted_lists_without_surrounding_empty_lines(self): + self.assertEqual( + self.fmt.format('foo\nbar\n* yo\n* a\n and b\nword'), + 'foo bar\n\n* yo\n* a and b\n\nword\n') + diff --git a/cliapp/genman.py b/cliapp/genman.py index 8d604f8..efad53b 100644 --- a/cliapp/genman.py +++ b/cliapp/genman.py @@ -29,10 +29,16 @@ class ManpageGenerator(object): self.arg_synopsis = arg_synopsis self.cmd_synopsis = cmd_synopsis + def sort_options(self, options): + return sorted(options, + key=lambda o: (o._long_opts + o._short_opts)[0]) + + def option_list(self, container): + return self.sort_options(container.option_list) + @property def options(self): - return sorted(self.parser.option_list, - key=lambda o: (o._long_opts + o._short_opts)[0]) + return self.option_list(self.parser) def format_template(self): sections = (('SYNOPSIS', self.format_synopsis()), @@ -48,7 +54,10 @@ class ManpageGenerator(object): lines += ['.nh'] lines += ['.B %s' % self.esc_dashes(self.parser.prog)] - for option in self.options: + all_options = self.option_list(self.parser) + for group in self.parser.option_groups: + all_options += self.option_list(group) + for option in self.sort_options(all_options): for spec in self.format_option_for_synopsis(option): lines += ['.RB [ %s ]' % spec] @@ -75,8 +84,17 @@ class ManpageGenerator(object): yield '%s%s' % (self.esc_dashes(name), suffix) def format_options(self): - return ''.join(self.format_option_for_options(option) - for option in self.options) + lines = [] + + for option in self.sort_options(self.parser.option_list): + lines += self.format_option_for_options(option) + + for group in self.parser.option_groups: + lines += ['.SS "%s"' % group.title] + for option in self.sort_options(group.option_list): + lines += self.format_option_for_options(option) + + return ''.join('%s\n' % line for line in lines) def format_option_for_options(self, option): lines = [] @@ -90,7 +108,7 @@ class ManpageGenerator(object): for x in option._long_opts] lines += ['.BR ' + ' ", " '.join(shorts + longs)] lines += [self.esc_dots(self.expand_default(option).strip())] - return ''.join('%s\n' % line for line in lines) + return lines def expand_default(self, option): default = self.parser.defaults.get(option.dest) diff --git a/cliapp/runcmd.py b/cliapp/runcmd.py index f45d42b..c578059 100644 --- a/cliapp/runcmd.py +++ b/cliapp/runcmd.py @@ -40,19 +40,26 @@ def runcmd(argv, *args, **kwargs): ''' - if 'ignore_fail' in kwargs: - ignore_fail = kwargs['ignore_fail'] - del kwargs['ignore_fail'] - else: - ignore_fail = False + our_options = ( + ('ignore_fail', False), + ('log_error', True), + ) + opts = {} + for name, default in our_options: + opts[name] = default + if name in kwargs: + opts[name] = kwargs[name] + del kwargs[name] exit, out, err = runcmd_unchecked(argv, *args, **kwargs) if exit != 0: msg = 'Command failed: %s\n%s' % (' '.join(argv), err) - if ignore_fail: - logging.info(msg) + if opts['ignore_fail']: + if opts['log_error']: + logging.info(msg) else: - logging.error(msg) + if opts['log_error']: + logging.error(msg) raise cliapp.AppException(msg) return out @@ -124,12 +131,6 @@ def _build_pipeline(argvs, pipe_stdin, pipe_stdout, pipe_stderr, kwargs): def _run_pipeline(procs, feed_stdin, pipe_stdin, pipe_stdout, pipe_stderr): - logging.debug('PIPE=%d' % subprocess.PIPE) - logging.debug('STDOUT=%d' % subprocess.STDOUT) - logging.debug('pipe_stdin=%s' % repr(pipe_stdin)) - logging.debug('pipe_stdout=%s' % repr(pipe_stdout)) - logging.debug('pipe_stderr=%s' % repr(pipe_stderr)) - stdout_eof = False stderr_eof = False out = [] @@ -138,7 +139,6 @@ def _run_pipeline(procs, feed_stdin, pipe_stdin, pipe_stdout, pipe_stderr): io_size = 1024 def set_nonblocking(fd): - logging.debug('set nonblocking fd=%d' % fd) flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) flags = flags | os.O_NONBLOCK fcntl.fcntl(fd, fcntl.F_SETFL, flags) @@ -202,8 +202,53 @@ def _run_pipeline(procs, feed_stdin, pipe_stdin, pipe_stdout, pipe_stderr): while still_running(): for p in procs: if p.returncode is None: - logging.debug('Waiting for child pid=%d to terminate' % p.pid) p.wait() return procs[-1].returncode, ''.join(out), ''.join(err) + + +def shell_quote(s): + '''Return a shell-quoted version of s.''' + + lower_ascii = 'abcdefghijklmnopqrstuvwxyz' + upper_ascii = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + digits = '0123456789' + punctuation = '-_/=.,:' + safe = set(lower_ascii + upper_ascii + digits + punctuation) + + quoted = [] + for c in s: + if c in safe: + quoted.append(c) + elif c == "'": + quoted.append('"\'"') + else: + quoted.append("'%c'" % c) + + return ''.join(quoted) + + +def ssh_runcmd(target, argv, **kwargs): # pragma: no cover + '''Run command in argv on remote host target. + + This is similar to runcmd, but the command is run on the remote + machine. The command is given as an argv array; elements in the + array are automatically quoted so they get passed to the other + side correctly. + + The target is given as-is to ssh, and may use any syntax ssh + accepts. + + Environment variables may or may not be passed to the remote + machine: this is dependent on the ssh and sshd configurations. + Invoke env(1) explicitly to pass in the variables you need to + exist on the other end. + + Pipelines are not supported. + + ''' + + local_argv = ['ssh', target, '--'] + map(shell_quote, argv) + return runcmd(local_argv, **kwargs) + diff --git a/cliapp/runcmd_tests.py b/cliapp/runcmd_tests.py index 99dc98a..a8a15dc 100644 --- a/cliapp/runcmd_tests.py +++ b/cliapp/runcmd_tests.py @@ -31,7 +31,7 @@ def devnull(msg): pass -class ApplicationTests(unittest.TestCase): +class RuncmdTests(unittest.TestCase): def test_runcmd_executes_true(self): self.assertEqual(cliapp.runcmd(['true']), '') @@ -124,3 +124,21 @@ class ApplicationTests(unittest.TestCase): self.assertEqual(exit, 0) self.assertEqual(data, '') + +class ShellQuoteTests(unittest.TestCase): + + def test_returns_empty_string_for_empty_string(self): + self.assertEqual(cliapp.shell_quote(''), '') + + def test_returns_same_string_when_safe(self): + self.assertEqual(cliapp.shell_quote('abc123'), 'abc123') + + def test_quotes_space(self): + self.assertEqual(cliapp.shell_quote(' '), "' '") + + def test_quotes_double_quote(self): + self.assertEqual(cliapp.shell_quote('"'), "'\"'") + + def test_quotes_single_quote(self): + self.assertEqual(cliapp.shell_quote("'"), '"\'"') + diff --git a/cliapp/settings.py b/cliapp/settings.py index 5d52098..0c54d84 100644 --- a/cliapp/settings.py +++ b/cliapp/settings.py @@ -24,6 +24,18 @@ import sys import cliapp from cliapp.genman import ManpageGenerator + +log_group_name = 'Logging' +config_group_name = 'Configuration files and settings' +perf_group_name = 'Peformance' + +default_group_names = [ + log_group_name, + config_group_name, + perf_group_name, +] + + class Setting(object): action = 'store' @@ -31,12 +43,14 @@ class Setting(object): nargs = 1 choices = None - def __init__(self, names, default, help, metavar=None, group=None): + def __init__( + self, names, default, help, metavar=None, group=None, hidden=False): self.names = names self.set_value(default) self.help = help self.metavar = metavar or self.default_metavar() self.group = group + self.hidden = hidden def default_metavar(self): return None @@ -75,8 +89,10 @@ class StringListSetting(Setting): action = 'append' - def __init__(self, names, default, help, metavar=None, group=None): - Setting.__init__(self, names, [], help, metavar=metavar, group=group) + def __init__( + self, names, default, help, metavar=None, group=None, hidden=False): + Setting.__init__( + self, names, [], help, metavar=metavar, group=group, hidden=hidden) self.default = default self.using_default_value = True @@ -107,9 +123,11 @@ class ChoiceSetting(Setting): type = 'choice' - def __init__(self, names, choices, help, metavar=None, group=None): - Setting.__init__(self, names, choices[0], help, metavar=metavar, - group=group) + def __init__( + self, names, choices, help, metavar=None, group=None, hidden=False): + Setting.__init__( + self, names, choices[0], help, metavar=metavar, group=group, + hidden=hidden) self.choices = choices def default_metavar(self): @@ -196,22 +214,10 @@ class FormatHelpParagraphs(optparse.IndentedHelpFormatter): def _format_text(self, text): # pragma: no cover '''Like the default, except handle paragraphs.''' - - def format_para(lines): - para = '\n'.join(lines) - return optparse.IndentedHelpFormatter._format_text(self, para) - - paras = [] - cur = [] - for line in text.splitlines(): - if line.strip(): - cur.append(line) - elif cur: - paras.append(format_para(cur)) - cur = [] - if cur: - paras.append(format_para(cur)) - return '\n\n'.join(paras) + + fmt = cliapp.TextFormat(width=self.width) + formatted = fmt.format(text) + return formatted.rstrip('\n') class Settings(object): @@ -278,29 +284,35 @@ class Settings(object): 'write log entries to FILE (default is to not write log ' 'files at all); use "syslog" to log to system log, ' 'or "none" to disable logging', - metavar='FILE') + metavar='FILE', group=log_group_name) self.choice(['log-level'], ['debug', 'info', 'warning', 'error', 'critical', 'fatal'], 'log at LEVEL, one of debug, info, warning, ' 'error, critical, fatal (default: %default)', - metavar='LEVEL') + metavar='LEVEL', group=log_group_name) self.bytesize(['log-max'], 'rotate logs larger than SIZE, ' 'zero for never (default: %default)', - metavar='SIZE', default=0) + metavar='SIZE', default=0, group=log_group_name) self.integer(['log-keep'], 'keep last N logs (%default)', - metavar='N', default=10) + metavar='N', default=10, group=log_group_name) self.string(['log-mode'], 'set permissions of new log files to MODE (octal; ' 'default %default)', - metavar='MODE', default='0600') + metavar='MODE', default='0600', group=log_group_name) self.choice(['dump-memory-profile'], ['simple', 'none', 'meliae', 'heapy'], 'make memory profiling dumps using METHOD, which is one ' 'of: none, simple, meliae, or heapy ' '(default: %default)', - metavar='METHOD') + metavar='METHOD', + group=perf_group_name) + self.integer(['memory-dump-interval'], + 'make memory profiling dumps at least SECONDS apart', + metavar='SECONDS', + default=300, + group=perf_group_name) def _add_setting(self, setting): '''Add a setting to self._cp.''' @@ -398,11 +410,25 @@ class Settings(object): return name def build_parser(self, configs_only=False, arg_synopsis=None, - cmd_synopsis=None): + cmd_synopsis=None, deferred_last=[], all_options=False): '''Build OptionParser for parsing command line.''' + # Call a callback function unless we're in configs_only mode. maybe = lambda func: (lambda *args: None) if configs_only else func + + # Maintain lists of callback function calls that are deferred. + # We call them ourselves rather than have OptionParser call them + # directly so that we can do things like --dump-config only + # after the whole command line is parsed. + + def defer_last(func): # pragma: no cover + def callback(*args): + deferred_last.append(lambda: func(*args)) + return callback + + # Create the command line parser. + def getit(x): if x is None or type(x) in [str, unicode]: return x @@ -415,41 +441,77 @@ class Settings(object): usage=usage, description=description, epilog=self.epilog) + + # Create all OptionGroup objects. This way, the user code can + # add settings to built-in option groups. + + group_names = set(default_group_names) + for name in self._canonical_names: + s = self._settingses[name] + if s.group is not None: + group_names.add(s.group) + group_names = sorted(group_names) + + option_groups = {} + for name in group_names: + group = optparse.OptionGroup(p, name) + p.add_option_group(group) + option_groups[name] = group + + config_group = option_groups[config_group_name] + + # Return help text, unless setting/option is hidden, in which + # case return optparse.SUPPRESS_HELP. + + def help_text(text, hidden): + if all_options or not hidden: + return text + else: + return optparse.SUPPRESS_HELP + + # Add --dump-setting-names. def dump_setting_names(*args): # pragma: no cover for name in self._canonical_names: sys.stdout.write('%s\n' % name) sys.exit(0) - p.add_option('--dump-setting-names', + config_group.add_option('--dump-setting-names', action='callback', nargs=0, - callback=maybe(dump_setting_names), - help='write out all names of settings and quit') + callback=defer_last(maybe(dump_setting_names)), + help=help_text( + 'write out all names of settings and quit', True)) + + # Add --dump-config. def call_dump_config(*args): # pragma: no cover self.dump_config(sys.stdout) sys.exit(0) - p.add_option('--dump-config', + config_group.add_option('--dump-config', action='callback', nargs=0, - callback=maybe(call_dump_config), + callback=defer_last(maybe(call_dump_config)), help='write out the entire current configuration') + # Add --no-default-configs. + def reset_configs(option, opt_str, value, parser): self.config_files = [] - p.add_option('--no-default-configs', + config_group.add_option('--no-default-configs', action='callback', nargs=0, callback=reset_configs, help='clear list of configuration files to read') + # Add --config. + def append_to_configs(option, opt_str, value, parser): self.config_files.append(value) - p.add_option('--config', + config_group.add_option('--config', action='callback', nargs=1, type='string', @@ -457,16 +519,20 @@ class Settings(object): help='add FILE to config files', metavar='FILE') + # Add --list-config-files. + def list_config_files(*args): # pragma: no cover for filename in self.config_files: print filename sys.exit(0) - p.add_option('--list-config-files', + config_group.add_option('--list-config-files', action='callback', nargs=0, - callback=maybe(list_config_files), - help='list all possible config files') + callback=defer_last(maybe(list_config_files)), + help=help_text('list all possible config files', True)) + + # Add --generate-manpage. self._arg_synopsis = arg_synopsis self._cmd_synopsis = cmd_synopsis @@ -475,9 +541,29 @@ class Settings(object): nargs=1, type='string', callback=maybe(self._generate_manpage), - help='fill in manual page TEMPLATE', + help=help_text('fill in manual page TEMPLATE', True), metavar='TEMPLATE') + # Add --help-all. + + def help_all(*args): # pragma: no cover + pp = self.build_parser( + configs_only=configs_only, + arg_synopsis=arg_synopsis, + cmd_synopsis=cmd_synopsis, + all_options=True) + sys.stdout.write(pp.format_help()) + sys.exit(0) + + config_group.add_option( + '--help-all', + action='callback', + help='show all options', + callback=defer_last(maybe(help_all))) + + # Add other options, from the user-defined and built-in + # settingses. + def set_value(option, opt_str, value, parser, setting): if setting.action == 'append': if setting.using_default_value: @@ -490,6 +576,9 @@ class Settings(object): assert setting.action == 'store' setting.value = value + def set_false(option, opt_str, value, parser, setting): + setting.value = False + def add_option(obj, s): option_names = self._option_names(s.names) obj.add_option(*option_names, @@ -499,34 +588,42 @@ class Settings(object): type=s.type, nargs=s.nargs, choices=s.choices, - help=s.help, + help=help_text(s.help, s.hidden), metavar=s.metavar) + + def add_negation_option(obj, s): + option_names = self._option_names(s.names) + long_names = [x for x in option_names if x.startswith('--')] + neg_names = ['--no-' + x[2:] for x in long_names] + unused_names = [x for x in neg_names + if x[2:] not in self._settingses] + obj.add_option(*unused_names, + action='callback', + callback=maybe(set_false), + callback_args=(s,), + type=s.type, + help=help_text('', s.hidden)) + + # Add options for every setting. for name in self._canonical_names: s = self._settingses[name] if s.group is None: - add_option(p, s) - p.set_defaults(**{self._destname(name): s.value}) - - groups = {} - for name in self._canonical_names: - s = self._settingses[name] - if s.group is not None: - groups[s.group] = groups.get(s.group, []) + [(name, s)] - - groupnames = sorted(groups.keys()) - for groupname in groupnames: - group = optparse.OptionGroup(p, groupname) - p.add_option_group(group) - for name, s in groups[groupname]: - add_option(group, s) - p.set_defaults(**{self._destname(name): s.value}) + obj = p + else: + obj = option_groups[s.group] + + add_option(obj, s) + if type(s) is BooleanSetting: + add_negation_option(obj, s) + p.set_defaults(**{self._destname(name): s.value}) return p def parse_args(self, args, parser=None, suppress_errors=False, configs_only=False, arg_synopsis=None, - cmd_synopsis=None): + cmd_synopsis=None, compute_setting_values=None, + all_options=False): '''Parse the command line. Return list of non-option arguments. ``args`` would usually @@ -534,14 +631,22 @@ class Settings(object): ''' + deferred_last = [] + p = parser or self.build_parser(configs_only=configs_only, arg_synopsis=arg_synopsis, - cmd_synopsis=cmd_synopsis) + cmd_synopsis=cmd_synopsis, + deferred_last=deferred_last, + all_options=all_options) if suppress_errors: p.error = lambda msg: sys.exit(1) options, args = p.parse_args(args) + if compute_setting_values: # pragma: no cover + compute_setting_values(self) + for callback in deferred_last: # pragma: no cover + callback() return args @property diff --git a/cliapp/settings_tests.py b/cliapp/settings_tests.py index b831ad3..757811a 100644 --- a/cliapp/settings_tests.py +++ b/cliapp/settings_tests.py @@ -65,6 +65,16 @@ class SettingsTests(unittest.TestCase): self.assertEqual(self.settings['foo'], 'foovalue') self.assertEqual(self.settings['bar'], True) + def test_parses_boolean_negation_option(self): + self.settings.boolean(['bar'], 'bar help') + self.settings.parse_args(['--bar', '--no-bar']) + self.assertEqual(self.settings['bar'], False) + + def test_parses_boolean_negation_option_in_group(self): + self.settings.boolean(['bar'], 'bar help', group='bar') + self.settings.parse_args(['--bar', '--no-bar']) + self.assertEqual(self.settings['bar'], False) + def test_does_not_have_foo_setting_by_default(self): self.assertFalse('foo' in self.settings) @@ -132,7 +142,7 @@ class SettingsTests(unittest.TestCase): def test_adds_boolean_setting(self): self.settings.boolean(['foo'], 'foo help') self.assert_('foo' in self.settings) - + def test_boolean_setting_is_false_by_default(self): self.settings.boolean(['foo'], 'foo help') self.assertFalse(self.settings['foo']) diff --git a/debian/changelog b/debian/changelog index ef606d2..eed671f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +python-cliapp (1.20130313-1) unstable; urgency=low + + * New upstream version. + + -- Lars Wirzenius <liw@liw.fi> Wed, 13 Mar 2013 21:21:40 +0000 + +python-cliapp (1.20121216-1) unstable; urgency=low + + * New upstream release. + + -- Lars Wirzenius <liw@liw.fi> Sun, 16 Dec 2012 16:41:36 +0000 + python-cliapp (1.20120929-1) unstable; urgency=low * New upstream release. @@ -38,6 +38,10 @@ class ExampleApp(cliapp.Application): self.settings.boolean(['dummy'], 'this setting is ignored', group='Test Group') + self.settings.string(['yoyo'], 'yoyo', group=cliapp.config_group_name) + + self.settings.string(['nono'], 'nono', default=None) + # We override process_inputs to be able to do something after the last # input line. def process_inputs(self, args): diff --git a/example3.py b/example3.py new file mode 100644 index 0000000..5de5a11 --- /dev/null +++ b/example3.py @@ -0,0 +1,48 @@ +# Copyright (C) 2012 Lars Wirzenius +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Example for cliapp framework. + +Demonstrate the compute_setting_values method. + +''' + + +import cliapp +import urlparse + + +class ExampleApp(cliapp.Application): + + '''A little fgrep-like tool.''' + + def add_settings(self): + self.settings.string(['url'], 'a url') + self.settings.string(['protocol'], 'the protocol') + + def compute_setting_values(self, settings): + if not self.settings['protocol']: + schema = urlparse.urlparse(self.settings['url'])[0] + self.settings['protocol'] = schema + + def process_args(self, args): + return + + +app = ExampleApp() +app.run() + diff --git a/example4.py b/example4.py new file mode 100644 index 0000000..14386df --- /dev/null +++ b/example4.py @@ -0,0 +1,53 @@ +# Copyright (C) 2013 Lars Wirzenius +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import logging + + +class ExampleApp(cliapp.Application): + + def setup(self): + self.add_subcommand('insult', self.insult, hidden=True) + + def add_settings(self): + self.settings.string(['yoyo'], 'yoyo help', hidden=True) + self.settings.boolean(['blip'], 'blip help', hidden=True) + + def cmd_greet(self, args): + '''Greet the user. + + The user is treated to a a courteus, + but terse form of greeting. + + ''' + for arg in args: + self.output.write('greetings, %s\n' % arg) + + def insult(self, args): + '''Insult the user. (hidden command) + + Sometimes, though rarely, it happens that a user is really a bit of + a prat, and needs to be told off. This is the command for that. + + ''' + for arg in args: + self.output.write('you suck, %s\n' % arg) + + +ExampleApp().run() + diff --git a/without-tests b/without-tests index 7c5b978..9f87e60 100644 --- a/without-tests +++ b/without-tests @@ -8,3 +8,5 @@ ./test-plugins/hello_plugin.py ./test-plugins/wrongversion_plugin.py ./test-plugins/aaa_hello_plugin.py +example3.py +example4.py |