add formattable columns concept

Some items that need to be treated as individual values are actually
complex data structures. Different formatters need to render those
values in different ways. For human-readable formatters like the table,
we may want to strip punctuation from data structures, for example,
while JSON formatters may need the values to be machine-readable and it
may not be possible to add extra punctuation or whitespace.

This patch adds the ability to all built-in formatters to have values
returned from commands be FormattableColumn instances, which can then be
interrogated for the two different forms of the data (human or machine
readable).

Change-Id: Ic84b05ae720188a63a11151e012997416544ac34
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

15 files changed, 305 insertions, 16 deletions

diff --git a/cliff/columns.py b/cliff/columns.py
new file mode 100644
index 0000000..3b6c026
--- /dev/null
+++ b/cliff/columns.py
@@ -0,0 +1,28 @@
+"""Formattable column tools.
+"""
+
+import abc
+
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class FormattableColumn(object):
+
+    def __init__(self, value):
+        self._value = value
+
+    @abc.abstractmethod
+    def human_readable(self):
+        """Return a basic human readable version of the data.
+        """
+
+    def machine_readable(self):
+        """Return a raw data structure using only Python built-in types.
+
+        It must be possible to serialize the return value directly
+        using a formatter like JSON, and it will be up to the
+        formatter plugin to decide how to make that transformation.
+
+        """
+        return self._value
diff --git a/cliff/formatters/base.py b/cliff/formatters/base.py
index c477a54..79e8920 100644
--- a/cliff/formatters/base.py
+++ b/cliff/formatters/base.py
@@ -26,11 +26,18 @@ class ListFormatter(Formatter):
     def emit_list(self, column_names, data, stdout, parsed_args):
         """Format and print the list from the iterable data source.
 
+        Data values can be primitive types like ints and strings, or
+        can be an instance of a :class:`FormattableColumn` for
+        situations where the value is complex, and may need to be
+        handled differently for human readable output vs. machine
+        readable output.
+
         :param column_names: names of the columns
         :param data: iterable data source, one tuple per object
                      with values in order of column names
         :param stdout: output stream where data should be written
         :param parsed_args: argparse namespace from our local options
+
         """
 
 
@@ -43,6 +50,12 @@ class SingleFormatter(Formatter):
     def emit_one(self, column_names, data, stdout, parsed_args):
         """Format and print the values associated with the single object.
 
+        Data values can be primitive types like ints and strings, or
+        can be an instance of a :class:`FormattableColumn` for
+        situations where the value is complex, and may need to be
+        handled differently for human readable output vs. machine
+        readable output.
+
         :param column_names: names of the columns
         :param data: iterable data source with values in order of column names
         :param stdout: output stream where data should be written
diff --git a/cliff/formatters/commaseparated.py b/cliff/formatters/commaseparated.py
index c545f28..ce66d2a 100644
--- a/cliff/formatters/commaseparated.py
+++ b/cliff/formatters/commaseparated.py
@@ -5,6 +5,9 @@ import os
 import sys
 
 from .base import ListFormatter
+from cliff import columns
+
+import six
 
 if sys.version_info[0] == 3:
     import csv
@@ -35,8 +38,14 @@ class CSVLister(ListFormatter):
         writer = csv.writer(stdout,
                             quoting=self.QUOTE_MODES[parsed_args.quote_mode],
                             lineterminator=os.linesep,
+                            escapechar='\\',
                             )
         writer.writerow(column_names)
         for row in data:
-            writer.writerow(row)
+            writer.writerow(
+                [(six.text_type(c.machine_readable())
+                  if isinstance(c, columns.FormattableColumn)
+                  else c)
+                 for c in row]
+            )
         return
diff --git a/cliff/formatters/json_format.py b/cliff/formatters/json_format.py
index fc35da1..6782499 100644
--- a/cliff/formatters/json_format.py
+++ b/cliff/formatters/json_format.py
@@ -4,6 +4,7 @@
 import json
 
 from .base import ListFormatter, SingleFormatter
+from cliff import columns
 
 
 class JSONFormatter(ListFormatter, SingleFormatter):
@@ -20,11 +21,21 @@ class JSONFormatter(ListFormatter, SingleFormatter):
     def emit_list(self, column_names, data, stdout, parsed_args):
         items = []
         for item in data:
-            items.append(dict(zip(column_names, item)))
+            items.append(
+                {n: (i.machine_readable()
+                     if isinstance(i, columns.FormattableColumn)
+                     else i)
+                 for n, i in zip(column_names, item)}
+            )
         indent = None if parsed_args.noindent else 2
         json.dump(items, stdout, indent=indent)
 
     def emit_one(self, column_names, data, stdout, parsed_args):
-        one = dict(zip(column_names, data))
+        one = {
+            n: (i.machine_readable()
+                if isinstance(i, columns.FormattableColumn)
+                else i)
+            for n, i in zip(column_names, data)
+        }
         indent = None if parsed_args.noindent else 2
         json.dump(one, stdout, indent=indent)
diff --git a/cliff/formatters/shell.py b/cliff/formatters/shell.py
index e613c22..5525e22 100644
--- a/cliff/formatters/shell.py
+++ b/cliff/formatters/shell.py
@@ -2,6 +2,7 @@
 """
 
 from .base import SingleFormatter
+from cliff import columns
 
 import argparse
 import six
@@ -37,6 +38,9 @@ class ShellFormatter(SingleFormatter):
         desired_columns = parsed_args.variables
         for name, value in zip(variable_names, data):
             if name in desired_columns or not desired_columns:
+                value = (six.text_type(value.machine_readable())
+                         if isinstance(value, columns.FormattableColumn)
+                         else value)
                 if isinstance(value, six.string_types):
                     value = value.replace('"', '\\"')
                 stdout.write('%s%s="%s"\n' % (parsed_args.prefix, name, value))
diff --git a/cliff/formatters/table.py b/cliff/formatters/table.py
index 8e51859..b62d7e3 100644
--- a/cliff/formatters/table.py
+++ b/cliff/formatters/table.py
@@ -7,6 +7,18 @@ import os
 
 from cliff import utils
 from .base import ListFormatter, SingleFormatter
+from cliff import columns
+
+
+def _format_row(row):
+    new_row = []
+    for r in row:
+        if isinstance(r, columns.FormattableColumn):
+            r = r.human_readable()
+        if isinstance(r, six.string_types):
+            r = r.replace('\r\n', '\n').replace('\r', ' ')
+        new_row.append(r)
+    return new_row
 
 
 class TableFormatter(ListFormatter, SingleFormatter):
@@ -52,12 +64,9 @@ class TableFormatter(ListFormatter, SingleFormatter):
                 alignment = self.ALIGNMENTS.get(type(value), 'l')
                 x.align[name] = alignment
             # Now iterate over the data and add the rows.
-            x.add_row(first_row)
+            x.add_row(_format_row(first_row))
             for row in data_iter:
-                row = [r.replace('\r\n', '\n').replace('\r', ' ')
-                       if isinstance(r, six.string_types) else r
-                       for r in row]
-                x.add_row(row)
+                x.add_row(_format_row(row))
 
         # Choose a reasonable min_width to better handle many columns on a
         # narrow console. The table will overflow the console width in
@@ -80,9 +89,7 @@ class TableFormatter(ListFormatter, SingleFormatter):
         x.align['Field'] = 'l'
         x.align['Value'] = 'l'
         for name, value in zip(column_names, data):
-            value = (value.replace('\r\n', '\n').replace('\r', ' ') if
-                     isinstance(value, six.string_types) else value)
-            x.add_row((name, value))
+            x.add_row(_format_row((name, value)))
 
         # Choose a reasonable min_width to better handle a narrow
         # console. The table will overflow the console width in preference
diff --git a/cliff/formatters/value.py b/cliff/formatters/value.py
index 6cc6744..d28f928 100644
--- a/cliff/formatters/value.py
+++ b/cliff/formatters/value.py
@@ -5,6 +5,7 @@ import six
 
 from .base import ListFormatter
 from .base import SingleFormatter
+from cliff import columns
 
 
 class ValueFormatter(ListFormatter, SingleFormatter):
@@ -14,10 +15,19 @@ class ValueFormatter(ListFormatter, SingleFormatter):
 
     def emit_list(self, column_names, data, stdout, parsed_args):
         for row in data:
-            stdout.write(' '.join(map(six.text_type, row)) + u'\n')
+            stdout.write(
+                ' '.join(
+                    six.text_type(c.machine_readable()
+                                  if isinstance(c, columns.FormattableColumn)
+                                  else c)
+                    for c in row) + u'\n')
         return
 
     def emit_one(self, column_names, data, stdout, parsed_args):
         for value in data:
-            stdout.write('%s\n' % str(value))
+            stdout.write('%s\n' % six.text_type(
+                value.machine_readable()
+                if isinstance(value, columns.FormattableColumn)
+                else value)
+            )
         return
diff --git a/cliff/formatters/yaml_format.py b/cliff/formatters/yaml_format.py
index e6fe17d..083cf52 100644
--- a/cliff/formatters/yaml_format.py
+++ b/cliff/formatters/yaml_format.py
@@ -4,6 +4,7 @@
 import yaml
 
 from .base import ListFormatter, SingleFormatter
+from cliff import columns
 
 
 class YAMLFormatter(ListFormatter, SingleFormatter):
@@ -14,10 +15,19 @@ class YAMLFormatter(ListFormatter, SingleFormatter):
     def emit_list(self, column_names, data, stdout, parsed_args):
         items = []
         for item in data:
-            items.append(dict(zip(column_names, item)))
+            items.append(
+                {n: (i.machine_readable()
+                     if isinstance(i, columns.FormattableColumn)
+                     else i)
+                 for n, i in zip(column_names, item)}
+            )
         yaml.safe_dump(items, stream=stdout, default_flow_style=False)
 
     def emit_one(self, column_names, data, stdout, parsed_args):
         for key, value in zip(column_names, data):
-            dict_data = {key: value}
+            dict_data = {
+                key: (value.machine_readable()
+                      if isinstance(value, columns.FormattableColumn)
+                      else value)
+            }
             yaml.safe_dump(dict_data, stream=stdout, default_flow_style=False)
diff --git a/cliff/tests/test_columns.py b/cliff/tests/test_columns.py
new file mode 100644
index 0000000..cbb8217
--- /dev/null
+++ b/cliff/tests/test_columns.py
@@ -0,0 +1,18 @@
+from cliff import columns
+
+
+class FauxColumn(columns.FormattableColumn):
+
+    def human_readable(self):
+        return u'I made this string myself: {}'.format(self._value)
+
+
+def test_faux_column_machine():
+    c = FauxColumn(['list', 'of', 'values'])
+    assert c.machine_readable() == ['list', 'of', 'values']
+
+
+def test_faux_column_human():
+    c = FauxColumn(['list', 'of', 'values'])
+    assert c.human_readable() == \
+        u"I made this string myself: ['list', 'of', 'values']"
diff --git a/cliff/tests/test_formatters_csv.py b/cliff/tests/test_formatters_csv.py
index cd2e4cf..510c790 100644
--- a/cliff/tests/test_formatters_csv.py
+++ b/cliff/tests/test_formatters_csv.py
@@ -6,6 +6,7 @@ import argparse
 import six
 
 from cliff.formatters import commaseparated
+from cliff.tests import test_columns
 
 
 def test_commaseparated_list_formatter():
@@ -40,6 +41,20 @@ def test_commaseparated_list_formatter_quoted():
     assert expected == actual
 
 
+def test_commaseparated_list_formatter_formattable_column():
+    sf = commaseparated.CSVLister()
+    c = ('a', 'b', 'c')
+    d1 = ('A', 'B', test_columns.FauxColumn(['the', 'value']))
+    data = [d1]
+    expected = 'a,b,c\nA,B,[\'the\'\\, \'value\']\n'
+    output = six.StringIO()
+    parsed_args = mock.Mock()
+    parsed_args.quote_mode = 'none'
+    sf.emit_list(c, data, output, parsed_args)
+    actual = output.getvalue()
+    assert expected == actual
+
+
 def test_commaseparated_list_formatter_unicode():
     sf = commaseparated.CSVLister()
     c = (u'a', u'b', u'c')
diff --git a/cliff/tests/test_formatters_json.py b/cliff/tests/test_formatters_json.py
index 26d6f5e..0ce902b 100644
--- a/cliff/tests/test_formatters_json.py
+++ b/cliff/tests/test_formatters_json.py
@@ -3,6 +3,7 @@ from six import StringIO
 import json
 
 from cliff.formatters import json_format
+from cliff.tests import test_columns
 
 import mock
 
@@ -38,6 +39,29 @@ def test_json_format_one():
     assert expected == actual
 
 
+def test_json_format_formattablecolumn_one():
+    sf = json_format.JSONFormatter()
+    c = ('a', 'b', 'c', 'd')
+    d = ('A', 'B', 'C', test_columns.FauxColumn(['the', 'value']))
+    expected = {
+        'a': 'A',
+        'b': 'B',
+        'c': 'C',
+        'd': ['the', 'value'],
+    }
+    args = mock.Mock()
+    sf.add_argument_group(args)
+
+    args.noindent = True
+    output = StringIO()
+    sf.emit_one(c, d, output, args)
+    value = output.getvalue()
+    print(len(value.splitlines()))
+    assert 1 == len(value.splitlines())
+    actual = json.loads(value)
+    assert expected == actual
+
+
 def test_json_format_list():
     sf = json_format.JSONFormatter()
     c = ('a', 'b', 'c')
@@ -69,3 +93,24 @@ def test_json_format_list():
     assert 17 == len(value.splitlines())
     actual = json.loads(value)
     assert expected == actual
+
+
+def test_json_format_formattablecolumn_list():
+    sf = json_format.JSONFormatter()
+    c = ('a', 'b', 'c')
+    d = (
+        ('A1', 'B1', test_columns.FauxColumn(['the', 'value'])),
+    )
+    expected = [
+        {'a': 'A1', 'b': 'B1', 'c': ['the', 'value']},
+    ]
+    args = mock.Mock()
+    sf.add_argument_group(args)
+
+    args.noindent = True
+    output = StringIO()
+    sf.emit_list(c, d, output, args)
+    value = output.getvalue()
+    assert 1 == len(value.splitlines())
+    actual = json.loads(value)
+    assert expected == actual
diff --git a/cliff/tests/test_formatters_shell.py b/cliff/tests/test_formatters_shell.py
index 956e16b..2babfd4 100644
--- a/cliff/tests/test_formatters_shell.py
+++ b/cliff/tests/test_formatters_shell.py
@@ -5,6 +5,7 @@ import argparse
 from six import StringIO, text_type
 
 from cliff.formatters import shell
+from cliff.tests import test_columns
 
 import mock
 
@@ -38,6 +39,24 @@ def test_shell_formatter_args():
     assert expected == actual
 
 
+def test_shell_formatter_formattable_column():
+    sf = shell.ShellFormatter()
+    c = ('a', 'b', 'c')
+    d = ('A', 'B', test_columns.FauxColumn(['the', 'value']))
+    expected = '\n'.join([
+        'a="A"',
+        'b="B"',
+        'c="[\'the\', \'value\']"\n',
+    ])
+    output = StringIO()
+    args = mock.Mock()
+    args.variables = ['a', 'b', 'c']
+    args.prefix = ''
+    sf.emit_one(c, d, output, args)
+    actual = output.getvalue()
+    assert expected == actual
+
+
 def test_shell_formatter_with_non_string_values():
     sf = shell.ShellFormatter()
     c = ('a', 'b', 'c', 'd', 'e')
diff --git a/cliff/tests/test_formatters_table.py b/cliff/tests/test_formatters_table.py
index a9cd975..0e093eb 100644
--- a/cliff/tests/test_formatters_table.py
+++ b/cliff/tests/test_formatters_table.py
@@ -6,6 +6,7 @@ import os
 import argparse
 
 from cliff.formatters import table
+from cliff.tests import test_columns
 
 
 class args(object):
@@ -65,7 +66,6 @@ def test_table_formatter(tw):
 '''
     assert expected == _table_tester_helper(c, d)
 
-
 # Multi-line output when width is restricted to 42 columns
 expected_ml_val = '''\
 +-------+--------------------------------+
@@ -237,6 +237,24 @@ def test_table_list_formatter(tw):
     assert expected == _table_tester_helper(c, data)
 
 
+@mock.patch('cliff.utils.terminal_width')
+def test_table_formatter_formattable_column(tw):
+    tw.return_value = 0
+    c = ('a', 'b', 'c', 'd')
+    d = ('A', 'B', 'C', test_columns.FauxColumn(['the', 'value']))
+    expected = '''\
++-------+---------------------------------------------+
+| Field | Value                                       |
++-------+---------------------------------------------+
+| a     | A                                           |
+| b     | B                                           |
+| c     | C                                           |
+| d     | I made this string myself: ['the', 'value'] |
++-------+---------------------------------------------+
+'''
+    assert expected == _table_tester_helper(c, d)
+
+
 _col_names = ('one', 'two', 'three')
 _col_data = [(
     'one one one one one',
@@ -302,6 +320,22 @@ _expected_mv = {
 
 
 @mock.patch('cliff.utils.terminal_width')
+def test_table_list_formatter_formattable_column(tw):
+    tw.return_value = 80
+    c = ('a', 'b', 'c')
+    d1 = ('A', 'B', test_columns.FauxColumn(['the', 'value']))
+    data = [d1]
+    expected = '''\
++---+---+---------------------------------------------+
+| a | b | c                                           |
++---+---+---------------------------------------------+
+| A | B | I made this string myself: ['the', 'value'] |
++---+---+---------------------------------------------+
+'''
+    assert expected == _table_tester_helper(c, data)
+
+
+@mock.patch('cliff.utils.terminal_width')
 def test_table_list_formatter_max_width(tw):
     # no resize
     l = tw.return_value = 80
diff --git a/cliff/tests/test_formatters_value.py b/cliff/tests/test_formatters_value.py
index a19a4d2..6ba9e9d 100644
--- a/cliff/tests/test_formatters_value.py
+++ b/cliff/tests/test_formatters_value.py
@@ -2,6 +2,7 @@
 from six import StringIO
 
 from cliff.formatters import value
+from cliff.tests import test_columns
 
 
 def test_value_formatter():
@@ -15,6 +16,17 @@ def test_value_formatter():
     assert expected == actual
 
 
+def test_value_formatter_formattable_column():
+    sf = value.ValueFormatter()
+    c = ('a', 'b', 'c', 'd')
+    d = ('A', 'B', 'C', test_columns.FauxColumn(['the', 'value']))
+    expected = "A\nB\nC\n['the', 'value']\n"
+    output = StringIO()
+    sf.emit_one(c, d, output, None)
+    actual = output.getvalue()
+    assert expected == actual
+
+
 def test_value_list_formatter():
     sf = value.ValueFormatter()
     c = ('a', 'b', 'c')
@@ -26,3 +38,15 @@ def test_value_list_formatter():
     sf.emit_list(c, data, output, None)
     actual = output.getvalue()
     assert expected == actual
+
+
+def test_value_list_formatter_formattable_column():
+    sf = value.ValueFormatter()
+    c = ('a', 'b', 'c')
+    d1 = ('A', 'B', test_columns.FauxColumn(['the', 'value']))
+    data = [d1]
+    expected = "A B ['the', 'value']\n"
+    output = StringIO()
+    sf.emit_list(c, data, output, None)
+    actual = output.getvalue()
+    assert expected == actual
diff --git a/cliff/tests/test_formatters_yaml.py b/cliff/tests/test_formatters_yaml.py
index ef8805f..d64d1b7 100644
--- a/cliff/tests/test_formatters_yaml.py
+++ b/cliff/tests/test_formatters_yaml.py
@@ -3,6 +3,7 @@ from six import StringIO
 import yaml
 
 from cliff.formatters import yaml_format
+from cliff.tests import test_columns
 
 import mock
 
@@ -24,6 +25,28 @@ def test_yaml_format_one():
     assert expected == actual
 
 
+def test_yaml_format_formattablecolumn_one():
+    sf = yaml_format.YAMLFormatter()
+    c = ('a', 'b', 'c', 'd')
+    d = ('A', 'B', 'C', test_columns.FauxColumn(['the', 'value']))
+    expected = {
+        'a': 'A',
+        'b': 'B',
+        'c': 'C',
+        'd': ['the', 'value'],
+    }
+    args = mock.Mock()
+    sf.add_argument_group(args)
+
+    args.noindent = True
+    output = StringIO()
+    sf.emit_one(c, d, output, args)
+    value = output.getvalue()
+    print(len(value.splitlines()))
+    actual = yaml.safe_load(output.getvalue())
+    assert expected == actual
+
+
 def test_yaml_format_list():
     sf = yaml_format.YAMLFormatter()
     c = ('a', 'b', 'c')
@@ -43,3 +66,22 @@ def test_yaml_format_list():
     sf.emit_list(c, d, output, args)
     actual = yaml.safe_load(output.getvalue())
     assert expected == actual
+
+
+def test_yaml_format_formattablecolumn_list():
+    sf = yaml_format.YAMLFormatter()
+    c = ('a', 'b', 'c')
+    d = (
+        ('A1', 'B1', test_columns.FauxColumn(['the', 'value'])),
+    )
+    expected = [
+        {'a': 'A1', 'b': 'B1', 'c': ['the', 'value']},
+    ]
+    args = mock.Mock()
+    sf.add_argument_group(args)
+
+    args.noindent = True
+    output = StringIO()
+    sf.emit_list(c, d, output, args)
+    actual = yaml.safe_load(output.getvalue())
+    assert expected == actual