remove support for 2.7, prepare for f-strings0.17.0

6 files changed, 72 insertions, 35 deletions

diff --git a/_test/lib/canonical.py b/_test/lib/canonical.py
index 1cab1ff..8a00772 100644
--- a/_test/lib/canonical.py
+++ b/_test/lib/canonical.py
@@ -339,7 +339,9 @@ ruamel.yaml.CanonicalLoader = CanonicalLoader
 
 
 def canonical_scan(stream):
-    return ruamel.yaml.scan(stream, Loader=CanonicalLoader)
+    yaml = ruamel.yaml.YAML()
+    yaml.scanner = CanonicalScanner
+    return yaml.scan(stream)
 
 
 ruamel.yaml.canonical_scan = canonical_scan
diff --git a/_test/lib/test_tokens.py b/_test/lib/test_tokens.py
index 1483db8..575e95c 100644
--- a/_test/lib/test_tokens.py
+++ b/_test/lib/test_tokens.py
@@ -1,5 +1,5 @@
 
-import ruamel.yaml as yaml
+import ruamel.yaml
 import pprint
 
 # Tokens mnemonic:
@@ -22,24 +22,24 @@ import pprint
 # value:                :
 
 _replaces = {
-    yaml.DirectiveToken: '%',
-    yaml.DocumentStartToken: '---',
-    yaml.DocumentEndToken: '...',
-    yaml.AliasToken: '*',
-    yaml.AnchorToken: '&',
-    yaml.TagToken: '!',
-    yaml.ScalarToken: '_',
-    yaml.BlockSequenceStartToken: '[[',
-    yaml.BlockMappingStartToken: '{{',
-    yaml.BlockEndToken: ']}',
-    yaml.FlowSequenceStartToken: '[',
-    yaml.FlowSequenceEndToken: ']',
-    yaml.FlowMappingStartToken: '{',
-    yaml.FlowMappingEndToken: '}',
-    yaml.BlockEntryToken: ',',
-    yaml.FlowEntryToken: ',',
-    yaml.KeyToken: '?',
-    yaml.ValueToken: ':',
+    ruamel.yaml.DirectiveToken: '%',
+    ruamel.yaml.DocumentStartToken: '---',
+    ruamel.yaml.DocumentEndToken: '...',
+    ruamel.yaml.AliasToken: '*',
+    ruamel.yaml.AnchorToken: '&',
+    ruamel.yaml.TagToken: '!',
+    ruamel.yaml.ScalarToken: '_',
+    ruamel.yaml.BlockSequenceStartToken: '[[',
+    ruamel.yaml.BlockMappingStartToken: '{{',
+    ruamel.yaml.BlockEndToken: ']}',
+    ruamel.yaml.FlowSequenceStartToken: '[',
+    ruamel.yaml.FlowSequenceEndToken: ']',
+    ruamel.yaml.FlowMappingStartToken: '{',
+    ruamel.yaml.FlowMappingEndToken: '}',
+    ruamel.yaml.BlockEntryToken: ',',
+    ruamel.yaml.FlowEntryToken: ',',
+    ruamel.yaml.KeyToken: '?',
+    ruamel.yaml.ValueToken: ':',
 }
 
 
@@ -48,9 +48,10 @@ def test_tokens(data_filename, tokens_filename, verbose=False):
     with open(tokens_filename, 'r') as fp:
         tokens2 = fp.read().split()
     try:
+        yaml = ruamel.yaml.YAML(typ='unsafe', pure=True)
         with open(data_filename, 'rb') as fp1:
             for token in yaml.scan(fp1):
-                if not isinstance(token, (yaml.StreamStartToken, yaml.StreamEndToken)):
+                if not isinstance(token, (ruamel.yaml.StreamStartToken, ruamel.yaml.StreamEndToken)):
                     tokens1.append(_replaces[token.__class__])
     finally:
         if verbose:
@@ -68,6 +69,7 @@ def test_scanner(data_filename, canonical_filename, verbose=False):
     for filename in [data_filename, canonical_filename]:
         tokens = []
         try:
+            yaml = ruamel.yaml.YAML(typ='unsafe', pure=False)
             with open(filename, 'rb') as fp:
                 for token in yaml.scan(fp):
                     tokens.append(token.__class__.__name__)
diff --git a/_test/lib/test_yaml_ext.py b/_test/lib/test_yaml_ext.py
index fd2d4ed..8cba7e5 100644
--- a/_test/lib/test_yaml_ext.py
+++ b/_test/lib/test_yaml_ext.py
@@ -176,10 +176,12 @@ def test_c_version(verbose=False):
 
 
 def _compare_scanners(py_data, c_data, verbose):
-    py_tokens = list(ruamel.yaml.scan(py_data, Loader=ruamel.yaml.PyLoader))
+    yaml = ruamel.yaml.YAML(typ='unsafe', pure=True)
+    py_tokens = list(yaml.scan(py_data, Loader=ruamel.yaml.PyLoader))
     c_tokens = []
     try:
-        for token in ruamel.yaml.scan(c_data, Loader=ruamel.yaml.CLoader):
+        yaml = ruamel.yaml.YAML(typ='unsafe', pure=False)
+        for token in yaml.scan(c_data, Loader=ruamel.yaml.CLoader):
             c_tokens.append(token)
         assert len(py_tokens) == len(c_tokens), (len(py_tokens), len(c_tokens))
         for py_token, c_token in zip(py_tokens, c_tokens):
diff --git a/_test/roundtrip.py b/_test/roundtrip.py
index f8f4b21..8b87380 100644
--- a/_test/roundtrip.py
+++ b/_test/roundtrip.py
@@ -6,7 +6,7 @@ helper routines for testing round trip of commented YAML data
 import sys
 import textwrap
 import io
-from ruamel.std.pathlib import Path
+from pathlib import Path
 
 unset = object()
 
@@ -46,7 +46,7 @@ def round_trip_load_all(inp, preserve_quotes=None, version=None):
 
 def round_trip_dump(
     data,
-    stream=None, *,
+    stream=None,  # *,
     indent=None,
     block_seq_indent=None,
     default_flow_style=unset,
@@ -79,7 +79,7 @@ def round_trip_dump(
 
 def round_trip_dump_all(
     data,
-    stream=None, *,
+    stream=None,  # *,
     indent=None,
     block_seq_indent=None,
     default_flow_style=unset,
@@ -326,12 +326,13 @@ def save_and_run(program, base_dir=None, output=None, file_name=None, optimized=
     file_name.write_text(dedent(program))
 
     try:
-        cmd = [sys.executable]
+        cmd = [sys.executable, '-Wd']
         if optimized:
             cmd.append('-O')
         cmd.append(str(file_name))
         print('running:', *cmd)
-        res = check_output(cmd, stderr=STDOUT, universal_newlines=True)
+        # 3.5 needs strings
+        res = check_output(cmd, stderr=STDOUT, universal_newlines=True, cwd=str(base_dir))
         if output is not None:
             if '__pypy__' in sys.builtin_module_names:
                 res = res.splitlines(True)
diff --git a/_test/test_api_change.py b/_test/test_api_change.py
index 4ce61b6..22250b8 100644
--- a/_test/test_api_change.py
+++ b/_test/test_api_change.py
@@ -7,7 +7,7 @@ testing of anchors and the aliases referring to them
 import sys
 import textwrap
 import pytest
-from ruamel.std.pathlib import Path
+from pathlib import Path
 
 
 class TestNewAPI:
diff --git a/_test/test_z_data.py b/_test/test_z_data.py
index dcd6bd0..e48e9d5 100644
--- a/_test/test_z_data.py
+++ b/_test/test_z_data.py
@@ -1,10 +1,12 @@
 # coding: utf-8
 
 import sys
+import os
 import pytest  # NOQA
 import warnings  # NOQA
+from pathlib import Path
 
-from ruamel.std.pathlib import Path
+from ruamel.yaml.compat import _F
 
 base_path = Path('data')  # that is ruamel.yaml.data
 
@@ -82,6 +84,11 @@ def pytest_generate_tests(metafunc):
     paths = sorted(base_path.glob('**/*.yaml'))
     idlist = []
     for path in paths:
+        # while developing tests put them in data/debug and run:
+        #   auto -c "pytest _test/test_z_data.py" data/debug/*.yaml *.py _test/*.py
+        if os.environ.get('RUAMELAUTOTEST') == '1':
+            if path.parent.stem != 'debug':
+                continue
         stem = path.stem
         if stem.startswith('.#'):  # skip emacs temporary file
             continue
@@ -144,13 +151,32 @@ class TestYAMLData(object):
                 print(line)
                 exec(line)
 
-    def run_python(self, python, data, tmpdir):
+    def run_python(self, python, data, tmpdir, input=None):
         from roundtrip import save_and_run
 
+        if input is not None:
+            (tmpdir / 'input.yaml').write_text(input.value, encoding='utf-8')
         assert save_and_run(python.value, base_dir=tmpdir, output=data.value) == 0
 
-    # this is executed by pytest the methods with names not starting with test_
-    # are helpers
+    def insert_comments(self, data, actions):
+        """this is to automatically insert based on:
+          path (a.1.b),
+          position (before, after, between), and
+          offset (absolute/relative)
+        """
+        raise NotImplementedError
+        expected = []
+        for line in data.value.splitlines(True):
+            idx = line.index['?']
+            if idx < 0:
+                expected.append(line)
+                continue
+            assert line.lstrip()[0] == '#'  # it has to be comment line
+        print(data)
+        assert ''.join(expected) == data.value
+
+    # this is executed by pytest the methods with names not starting with
+    # test_ are helper methods
     def test_yaml_data(self, yaml, tmpdir):
         from collections.abc import Mapping
 
@@ -197,11 +223,15 @@ class TestYAMLData(object):
         if typ == 'rt':
             self.round_trip(data, output, yaml_version=yaml_version)
         elif typ == 'python_run':
-            self.run_python(python, output if output is not None else data, tmpdir)
+            inp = None if output is None or data is None else data
+            self.run_python(python, output if output is not None else data, tmpdir, input=inp)
         elif typ == 'load_assert':
             self.load_assert(data, confirm, yaml_version=yaml_version)
+        elif typ == 'comment':
+            actions = []
+            self.insert_comments(data, actions)
         else:
-            print('\nrun type unknown:', typ)
+            _F('\n>>>>>> run type unknown: "{typ}" <<<<<<\n')
             raise AssertionError()