diff options
| -rw-r--r-- | .hgignore | 3 | ||||
| -rw-r--r-- | py/convert/__init__.py | 0 | ||||
| -rw-r--r-- | py/convert/from_csv.py | 46 | ||||
| -rw-r--r-- | py/convert/html.py | 120 | ||||
| -rw-r--r-- | setup.py | 6 | ||||
| -rw-r--r-- | test/test_convert.py | 61 | ||||
| -rw-r--r-- | test/test_util.py | 212 | ||||
| -rw-r--r-- | test/test_utiltohtml.py | 104 | ||||
| -rw-r--r-- | tox.ini | 2 |
9 files changed, 6 insertions, 548 deletions
@@ -12,3 +12,6 @@ build .tox .cache README.pdf +ruamel +convert +cmd diff --git a/py/convert/__init__.py b/py/convert/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/py/convert/__init__.py +++ /dev/null diff --git a/py/convert/from_csv.py b/py/convert/from_csv.py deleted file mode 100644 index 6e1bd09..0000000 --- a/py/convert/from_csv.py +++ /dev/null @@ -1,46 +0,0 @@ -# coding: utf-8 - -from __future__ import print_function - -import dateutil.parser -import ruamel.yaml - -class CSV2YAML(object): - def __init__(self, args=None): - # self.flatten = not getattr(args, 'no_flatten', False) - self.delimiter = getattr(args, 'delimiter', None) - - def __call__(self, csv_file_name): - import csv - - data = [] - with open(csv_file_name) as inf: - for line in csv.reader(inf, delimiter=self.delimiter): - data.append(self.process_line(line)) - print(ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper)) - - def process_line(self, line): - """convert lines, trying, int, float, date""" - ret_val = [] - for elem in line: - try: - res = int(elem) - ret_val.append(res) - continue - except ValueError: - pass - try: - res = float(elem) - ret_val.append(res) - continue - except ValueError: - pass - try: - res = dateutil.parser.parse(elem) - ret_val.append(res) - continue - except TypeError: - pass - ret_val.append(elem) - return ret_val - diff --git a/py/convert/html.py b/py/convert/html.py deleted file mode 100644 index ed92f9d..0000000 --- a/py/convert/html.py +++ /dev/null @@ -1,120 +0,0 @@ -# coding: utf-8 - -from __future__ import print_function - -""" -conversion helpers for yaml utility to/from HTML - -html/xml to yaml inspired by -http://programmers.stackexchange.com/questions/157395/yaml-translating-free-flowing-text-from-xml - -""" - -import sys - - -class HTML2YAML(object): - def __init__(self, args=None): - self.flatten = not getattr(args, 'no_flatten', False) - self.strip = getattr(args, 'strip', False) - self.no_body = getattr(args, 'no_body', False) - - def __call__(self, html): - d = self.html_to_data(html) - if self.no_body: - d = d['html']['body'] - return self.data_to_yaml(d) - - def data_to_yaml(self, d): - import ruamel.yaml - return ruamel.yaml.dump( - d, - Dumper=ruamel.yaml.RoundTripDumper, - allow_unicode=True, - ) - - def html_to_data(self, html): - try: - import bs4 - except ImportError: - print("For HTML conversion you need to install BeautifulSoup") - print("e.g. using (pip install beautifulsoup4)") - sys.exit(1) - - soup = bs4.BeautifulSoup(html) - data = self._convert_node(soup) - return data - - def _convert_node(self, node, depth=0): - try: - import bs4 - except ImportError: - print("For HTML conversion you need to install BeautifulSoup") - print("e.g. using (pip install beautifulsoup4)") - sys.exit(1) - from ruamel.yaml.comments import CommentedMap - from ruamel.yaml.scalarstring import PreservedScalarString - ret_val = [] - if node.attrs: - ret_val.append({'.attribute': node.attrs}) - for data in node.contents: - if isinstance(data, bs4.Tag): - kv = CommentedMap() - #print data.name, data.attrs - # convert the intenals of the tag - kv[data.name] = self._convert_node(data, depth+1) - ret_val.append(kv) - elif isinstance(data, bs4.NavigableString): - s, nl = self._strip(data) - if not s: - continue - if nl: - ret_val.append(PreservedScalarString(s)) - continue - ret_val.append(s) - else: - print('unknow type', type(data)) - if self.flatten and len(ret_val) == 1: - return ret_val[0] - return ret_val - - def _strip(self, data): - import textwrap - # multiline strings might be nicely formatted so don't - # use .strip() immediately - if self.strip: - s = data.strip() - else: - s = data.rstrip() - if not s: - return None, False - first_nl_pos = s.find(u'\n') - if first_nl_pos < 0: - return s, False - if not s[:first_nl_pos].strip(): # i.e. space until first newline - if u'\n' not in s[first_nl_pos+1:]: - print(repr(data), repr(s)) - # single line of text preceded and followed by nl - return s.strip(), False - # use data here, removing the final newline would get your |- as marker - s = textwrap.dedent(data[first_nl_pos+1:]) - return s, True - - -class YAML2HTML(object): - def __init__(self, args=None): - pass - - def __call__(self, yaml): - d = self.yaml_to_data(yaml) - return self.data_to_html(d) - - def data_to_html(self, d): - if isinstance(d, dict): - pass - - def yaml_to_data(self, yaml): - import ruamel.yaml - return ruamel.yaml.load(yaml) - return data - @@ -160,8 +160,8 @@ def check_extensions(): def main(): install_requires = [ - "ruamel.base==0.3", - "ruamel.std.argparse==0.5.2", + "ruamel.base>=1.0.0", + "ruamel.std.argparse", ] # use fast ordereddict for !!omap if sys.version_info[0] == 2 and \ @@ -188,7 +188,7 @@ def main(): namespace_packages=[name_space], packages=packages, ext_modules=check_extensions(), - entry_points=mk_entry_points(full_package_name), + #entry_points=mk_entry_points(full_package_name), cmdclass={'install_lib': MyInstallLib}, classifiers=[ 'Development Status :: 4 - Beta', diff --git a/test/test_convert.py b/test/test_convert.py deleted file mode 100644 index a4ad70d..0000000 --- a/test/test_convert.py +++ /dev/null @@ -1,61 +0,0 @@ - -from ruamel.yaml.convert.html import HTML2YAML -from textwrap import dedent - -class Bin: - pass - -class TestH2Y: - sample1 = dedent("""\ - text: - - |- - This is an example text, spanning multiple lines, and it has embedded elements - like - - a: - - .attribute: - p: value - - this - - and - - b: this - - '. There is also a list:' - - quote: - - text: - - |- - The text of the quote, spanning multiple lines, and it has - embedded elements like - - c: - - .attribute: - p: value - - this - - and - - b: this - - author: The Author of this quote - - Text continues here. - """) - - - def test_00(self): - b = Bin() - b.strip = True - h2y = HTML2YAML(b) - d = h2y.html_to_data(dedent("""\ - <text> - This is an example text, spanning multiple lines, and it has embedded elements - like <a p="value">this</a> and <b>this</b>. There is also a list: - <quote> - <text>The text of the quote, spanning multiple lines, and it has - embedded elements like <c p="value">this</c> and <b>this</b></text> - <author>The Author of this quote</author> - </quote> - Text continues here. - </text> - """)) - if 'html' in d: - d = d['html']['body'] - res = h2y.data_to_yaml(d) - assert res == self.sample1 - - - def XXtest_01(self): - y2h = YAML2HTML(b) - d = y2h.yaml_to_data(self.sample1) diff --git a/test/test_util.py b/test/test_util.py deleted file mode 100644 index 15bb7a9..0000000 --- a/test/test_util.py +++ /dev/null @@ -1,212 +0,0 @@ - -from __future__ import print_function - -import io -import sys -import subprocess -try: - _ = subprocess.check_output - - def check_output(*args, **kw): - try: - res = subprocess.check_output(*args, **kw) - except subprocess.CalledProcessError as e: - print("subprocess.CalledProcessError\n", e.output, sep='') - res = e.output - if PY3: - res = res.decode('utf-8') - return res -except AttributeError: - def check_output(*args, **kw): - process = subprocess.Popen(stdout=subprocess.PIPE, *args, **kw) - output, unused_err = process.communicate() - if PY3: - output = output.decode('utf-8') - return output - -import pytest - -import ruamel.yaml -from ruamel.yaml.compat import PY3 -from roundtrip import dedent - - -def call_util(s, file_name, cmd, mp, td): - """call the utilitiy yaml. if exit != 0 or if somethhing goes wrong - return error output""" - mp.chdir(td) - with io.open(file_name, 'w') as fp: - fp.write(dedent(s)) - res = check_output(cmd, stderr=subprocess.STDOUT) - return res.replace('\r\n', '\n') - - -def rt_test(s, file_name, mp, td): - return call_util(s, file_name, ['yaml', 'rt', "-v", file_name], mp, td) - - -class TestUtil: - - def test_version(self, capsys): - res = check_output( - ['yaml', '--version'], stderr=subprocess.STDOUT) - assert res.replace('\r\n', '\n') == \ - u"version: {0}\n".format(ruamel.yaml.__version__) - - def test_ok(self, tmpdir, monkeypatch): - file_name = "00_ok.yaml" - res = rt_test(u""" - - abc - - ghi # some comment - - klm - """, file_name, mp=monkeypatch, td=tmpdir) - assert res == "{0}: ok\n".format(file_name) - - def test_not_ok(self, tmpdir, monkeypatch): - file_name = "01_second_rt_ok.yaml" - res = rt_test(u""" - - abc - - ghi # some comment - - klm - """, file_name, mp=monkeypatch, td=tmpdir) - assert res.replace('\r\n', '\n') == dedent(""" - {file_name}: - stabilizes on second round trip, ok without comments - --- 01_second_rt_ok.yaml - +++ round trip YAML - @@ -1,3 +1,3 @@ - - abc - -- ghi # some comment - +- ghi # some comment - - klm - """).format(**dict(file_name=file_name)) - - def test_from_configobj(self, tmpdir, monkeypatch): - file_name = "02_from_ini.yaml" - res = call_util( - u""" - # initial comment - - keyword1 = value1 - keyword2 = value2 - - [section 1] - keyword1 = value1 - keyword2 = value2 - - [[sub-section]] - # this is in section 1 - keyword1 = value1 - keyword2 = value2 - - [[[nested section]]] - # this is in sub section - keyword1 = value1 - keyword2 = value2 - - [[sub-section2]] - # this is in section 1 again - keyword1 = value1 - keyword2 = value2 - - [[sub-section3]] - # this is also in section 1, indentation is misleading here - keyword1 = value1 - keyword2 = value2 - - # final comment - """, file_name, ['yaml', 'from-ini', file_name], - mp=monkeypatch, td=tmpdir) - print(res) - assert res.replace('\r\n', '\n') == dedent(""" - # initial comment - keyword1: value1 - keyword2: value2 - section 1: - keyword1: value1 - keyword2: value2 - sub-section: - # this is in section 1 - keyword1: value1 - keyword2: value2 - nested section: - # this is in sub section - keyword1: value1 - keyword2: value2 - sub-section2: - # this is in section 1 again - keyword1: value1 - keyword2: value2 - sub-section3: - # this is also in section 1, indentation is misleading here - keyword1: value1 - keyword2: value2 - # final comment - """) - - def test_from_configobj_extra_comments(self, tmpdir, monkeypatch): - file_name = "02_from_ini.yaml" - res = call_util( - u""" - # initial comment - keyword1 = value1 - keyword2 = value2 # eol comment kw2 - - [section 1] - keyword1 = value1 # and here more comment - # comment s1kw2 - keyword2 = value2 # eol comment s1kw2 - - [[sub-section]] # eol on section - # this is in section 1 - keyword1 = value1 - keyword2 = value2 - - [[[nested section]]] - # this is in sub section - keyword1 = value1 - keyword2 = value2 - # after nested - - [[sub-section2]] - # this is in section 1 again - keyword1 = value1 - keyword2 = value2 - - [[sub-section3]] # comment on section key - # this is also in section 1, indentation is misleading here - keyword1 = value1 - keyword2 = value2 - - # final comment - """, - file_name, ['yaml', 'from-ini', file_name], - mp=monkeypatch, td=tmpdir) - print(res) - assert res.replace('\r\n', '\n') == dedent(""" - # initial comment - keyword1: value1 - keyword2: value2 # eol comment kw2 - section 1: - keyword1: value1 # and here more comment - # comment s1kw2 - keyword2: value2 # eol comment s1kw2 - sub-section: # eol on section - # this is in section 1 - keyword1: value1 - keyword2: value2 - nested section: - # this is in sub section - keyword1: value1 - keyword2: value2 - # after nested - sub-section2: - # this is in section 1 again - keyword1: value1 - keyword2: value2 - sub-section3: # comment on section key - # this is also in section 1, indentation is misleading here - keyword1: value1 - keyword2: value2 - # final comment - """) diff --git a/test/test_utiltohtml.py b/test/test_utiltohtml.py deleted file mode 100644 index 316a620..0000000 --- a/test/test_utiltohtml.py +++ /dev/null @@ -1,104 +0,0 @@ - -from __future__ import print_function - -import io -import sys -import subprocess - -from roundtrip import dedent - -from test_util import check_output, call_util - - -def to_html(s, file_name, mp, td, options=None): - cmd = ['yaml', 'htmltable', file_name] - if options: - cmd.extend(options) - return call_util(s, file_name, cmd, mp, td) - - -class TestUtilToHTML: - def test_html_01(self, tmpdir, monkeypatch): - file_name = "html.yaml" - res = to_html(u""" - - 1 - - 2 - """, file_name, mp=monkeypatch, td=tmpdir, options=["--level"]) - assert res == "levels: 1\n" - - def test_html_02(self, tmpdir, monkeypatch): - file_name = "html.yaml" - res = to_html(u""" - abc: - - 1 - - 2 - ghi: - - 3 - - 4 - """, file_name, mp=monkeypatch, td=tmpdir, options=["--level"]) - assert res == "levels: 2\n".format(file_name) - - def test_html_03(self, tmpdir, monkeypatch): - file_name = "html.yaml" - res = to_html(u""" - - - - 1 - - 2 - - - - 3 - - 4 - """, file_name, mp=monkeypatch, td=tmpdir, options=["--level"]) - assert res == "levels: 2\n" - - def test_html_03(self, tmpdir, monkeypatch): - file_name = "html.yaml" - res = to_html(u""" - - - abc: - - 1 - - 2 - def: - - 3 - - 4 - """, file_name, mp=monkeypatch, td=tmpdir, options=["--level"]) - assert res == "levels: 3\n" - - def test_html_04(self, tmpdir, monkeypatch): - file_name = "html.yaml" - res = to_html(u""" - title: - - fruit - - legume - local: - - apple - - sprouts - import: - - orange - - broccoli - """, file_name, mp=monkeypatch, td=tmpdir) - assert res == dedent("""\ - <HTML> - <HEAD> - </HEAD> - <BODY> - <TABLE> - <TR> - <TD>title</TD> - <TD>fruit</TD> - <TD>legume</TD> - </TR> - <TR> - <TD>local</TD> - <TD>apple</TD> - <TD>sprouts</TD> - </TR> - <TR> - <TD>import</TD> - <TD>orange</TD> - <TD>broccoli</TD> - </TR> - <TABLE> - </BODY> - </HTML> - - """) @@ -6,8 +6,6 @@ commands = py.test test deps = pytest - configobj - beautifulsoup4 [pytest] norecursedirs = test/lib .tox |