update setup.py for 3.8 ast imports0.8.3

reported by John Vandenberg @ issue 2

9 files changed, 320 insertions, 254 deletions

diff --git a/.hgignore b/.hgignore
index 350f799..b05417b 100644
--- a/.hgignore
+++ b/.hgignore
@@ -1,16 +1,5 @@
 
 syntax: glob
 
-*.pyc
-*~
-*.bak
-*.o
-*.orig
-README.pdf
-dist
-build
-*.egg-info
-.tox
+
 ruamel
-.ruamel
-.cache
diff --git a/LICENSE b/LICENSE
index f2e500f..4f74f81 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
  The MIT License (MIT)
 
- Copyright (c) 2007-2017 Anthon van der Neut, Ruamel bvba
+ Copyright (c) 2007-2020 Anthon van der Neut, Ruamel bvba
 
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 12eb858..0000000
--- a/Makefile
+++ /dev/null
@@ -1,20 +0,0 @@
-
-UTILNAME:=argparse
-PKGNAME:=ruamel.std.argparse
-VERSION:=$$(python setup.py --version)
-REGEN:=/usr/local/bin/ruamel_util_new util --std Argparse --skip-util --skip-hg
-
-include ~/.config/ruamel_util_new/Makefile.inc
-
-clean:
-	rm -rf build .tox $(PKGNAME).egg-info/ README.pdf __pycache__
-	find . -name "*.pyc" -exec rm {} +
-
-updatereadme:
-	updatereadme
-
-layout:	pdf
-	cp README.pdf /data0/tmp/pdf
-
-owl:	sdist
-	make owl_copy owl_devpi
diff --git a/__init__.py b/__init__.py
index 967150f..aa007d8 100644
--- a/__init__.py
+++ b/__init__.py
@@ -4,8 +4,8 @@ from __future__ import print_function
 
 _package_data = dict(
     full_package_name='ruamel.std.argparse',
-    version_info=(0, 8, 1),
-    __version__='0.8.1',
+    version_info=(0, 8, 3),
+    __version__='0.8.3',
     author='Anthon van der Neut',
     author_email='a.van.der.neut@ruamel.eu',
     description='Enhancements to argparse: extra actions, subparser aliases, smart formatter, a decorator based wrapper',  # NOQA
@@ -14,6 +14,8 @@ _package_data = dict(
     keywords='argparse enhanced',
     install_requires=[],
     universal=True,
+    tox=dict(env='*p'),
+    print_allowed=True,
 )
 
 version_info = _package_data['version_info']
@@ -25,6 +27,7 @@ import argparse                      # NOQA
 from argparse import ArgumentParser  # NOQA
 import glob                          # NOQA
 from importlib import import_module  # NOQA
+from argparse import SUPPRESS  # NOQA
 
 PY3 = sys.version_info[0] == 3
 
@@ -75,6 +78,7 @@ class SubParsersAction(argparse._SubParsersAction):
 
 from .action.checksinglestore import CheckSingleStoreAction  # NOQA
 from .action.count import CountAction                        # NOQA
+from .action.date import DateAction                          # NOQA
 from .action.splitappend import SplitAppendAction            # NOQA
 
 
@@ -240,7 +244,7 @@ class ProgramBase(object):
                     if k == 'parser':
                         v = 'ArgumentParser()'
                     elif k == 'sp':
-                            v = '_SubParserAction()'
+                        v = '_SubParserAction()'
                     else:
                         v = info[k]
                     print('       ' + '  ' * level, k, '->', v)
diff --git a/_test/test_argparse.py b/_test/test_argparse.py
index 72b8d7a..9dc8a7e 100644
--- a/_test/test_argparse.py
+++ b/_test/test_argparse.py
@@ -43,7 +43,7 @@ def test_argparse(capsys):
     parser.parse_args(['--help'])
     out, err = capsys.readouterr()
     full_help = dedent("""\
-    usage: py.test [-h] [--verbose] [--list LIST] [--oneline]
+    usage: pytest [-h] [--verbose] [--list LIST] [--oneline]
 
     {0}
     optional arguments:
@@ -90,7 +90,7 @@ def test_argparse_default(capsys):
     parser.parse_args(['--help'])
     out, err = capsys.readouterr()
     full_help = dedent("""\
-    usage: py.test [-h] [--verbose] [--list LIST] [--oneline]
+    usage: pytest [-h] [--verbose] [--list LIST] [--oneline]
 
     {0}
     optional arguments:
diff --git a/action/checksinglestore.py b/action/checksinglestore.py
index 3a37792..706cd78 100644
--- a/action/checksinglestore.py
+++ b/action/checksinglestore.py
@@ -8,12 +8,18 @@ import argparse
 
 class CheckSingleStoreAction(argparse.Action):
     """issue a warning when the store action is called multiple times"""
+
     def __call__(self, parser, namespace, values, option_string=None):
         if getattr(namespace, self.dest, None) is not None:
             print(
-                'WARNING: previous optional argument "' + option_string + " " +
-                str(getattr(namespace, self.dest)) + '" overwritten by "' +
-                str(option_string) +
-                " " + str(values) +
-                '"')
+                'WARNING: previous optional argument "'
+                + option_string
+                + ' '
+                + str(getattr(namespace, self.dest))
+                + '" overwritten by "'
+                + str(option_string)
+                + ' '
+                + str(values)
+                + '"'
+            )
         setattr(namespace, self.dest, values)
diff --git a/action/date.py b/action/date.py
new file mode 100644
index 0000000..4b73a30
--- /dev/null
+++ b/action/date.py
@@ -0,0 +1,29 @@
+# coding: utf-8
+# Copyright Ruamel bvba 2007-2018
+
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+import argparse
+import datetime
+
+
+class DateAction(argparse.Action):
+    """argparse action for parsing dates with or without dashes
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--verbose', '-v', action=DateAction)
+    """
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        if nargs != 1 and nargs not in [None, '?', '*']:
+            raise ValueError("DateAction can only have one argument")
+        super(DateAction, self).__init__(option_strings, dest, nargs=nargs, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if values is None:
+            return None
+        s = values
+        for c in './-_':
+            s = s.replace(c, '')
+        val = datetime.datetime.strptime(s, '%Y%m%d').date()
+        #    val = self.const
+        setattr(namespace, self.dest, val)
diff --git a/setup.py b/setup.py
index e540f93..f22dceb 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
 # # header
 # coding: utf-8
+# dd: 20200125
 
 from __future__ import print_function, absolute_import, division, unicode_literals
 
@@ -8,15 +9,24 @@ from __future__ import print_function, absolute_import, division, unicode_litera
 import sys
 import os
 import datetime
-sys.path = [path for path in sys.path if path not in [os.getcwd(), '']]
-import platform          # NOQA
-from _ast import *       # NOQA
-from ast import parse    # NOQA
+import traceback
+
+sys.path = [path for path in sys.path if path not in [os.getcwd(), ""]]
+import platform  # NOQA
+from _ast import *  # NOQA
+from ast import parse  # NOQA
 
 from setuptools import setup, Extension, Distribution  # NOQA
-from setuptools.command import install_lib             # NOQA
-from setuptools.command.sdist import sdist as _sdist   # NOQA
+from setuptools.command import install_lib  # NOQA
+from setuptools.command.sdist import sdist as _sdist  # NOQA
 
+try:
+    from setuptools.namespaces import Installer as NameSpaceInstaller # NOQA
+except ImportError:
+    msg = ('You should use the latest setuptools. The namespaces.py file that this setup.py'
+           ' uses was added in setuptools 28.7.0 (Oct 2016)')
+    print(msg)
+    sys.exit()
 
 if __name__ != '__main__':
     raise NotImplementedError('should never include setup.py')
@@ -25,39 +35,62 @@ if __name__ != '__main__':
 
 full_package_name = None
 
-if __name__ != '__main__':
-    raise NotImplementedError('should never include setup.py')
-
-if sys.version_info < (3, ):
+if sys.version_info < (3,):
     string_type = basestring
 else:
     string_type = str
 
 
 if sys.version_info < (3, 4):
-    class Bytes():
+
+    class Bytes:
         pass
 
     class NameConstant:
         pass
 
-if sys.version_info < (3, ):
+
+if sys.version_info >= (3, 8):
+    from ast import Str, Num, Bytes, NameConstant  # NOQA
+
+
+if sys.version_info < (3,):
     open_kw = dict()
 else:
     open_kw = dict(encoding='utf-8')
 
 
 if sys.version_info < (2, 7) or platform.python_implementation() == 'Jython':
-    class Set():
+
+    class Set:
+        pass
+
+
+if os.environ.get('DVDEBUG', "") == "":
+
+    def debug(*args, **kw):
         pass
 
 
+else:
+
+    def debug(*args, **kw):
+        with open(os.environ['DVDEBUG'], 'a') as fp:
+            kw1 = kw.copy()
+            kw1['file'] = fp
+            print('{:%Y-%d-%mT%H:%M:%S}'.format(datetime.datetime.now()), file=fp, end=' ')
+            print(*args, **kw1)
+
+
 def literal_eval(node_or_string):
     """
     Safely evaluate an expression node or a string containing a Python
     expression.  The string or node provided may only consist of the following
     Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
     sets, booleans, and None.
+
+    Even when passing in Unicode, the resulting Str types parsed are 'str' in Python 2.
+    I don't now how to set 'unicode_literals' on parse -> Str is explicitly converted.
     """
     _safe_names = {'None': None, 'True': True, 'False': False}
     if isinstance(node_or_string, string_type):
@@ -65,10 +98,14 @@ def literal_eval(node_or_string):
     if isinstance(node_or_string, Expression):
         node_or_string = node_or_string.body
     else:
-        raise TypeError("only string or AST nodes supported")
+        raise TypeError('only string or AST nodes supported')
 
     def _convert(node):
-        if isinstance(node, (Str, Bytes)):
+        if isinstance(node, Str):
+            if sys.version_info < (3,) and not isinstance(node.s, unicode):
+                return node.s.decode('utf-8')
+            return node.s
+        elif isinstance(node, Bytes):
             return node.s
         elif isinstance(node, Num):
             return node.n
@@ -79,25 +116,28 @@ def literal_eval(node_or_string):
         elif isinstance(node, Set):
             return set(map(_convert, node.elts))
         elif isinstance(node, Dict):
-            return dict((_convert(k), _convert(v)) for k, v
-                        in zip(node.keys, node.values))
+            return dict((_convert(k), _convert(v)) for k, v in zip(node.keys, node.values))
         elif isinstance(node, NameConstant):
             return node.value
         elif sys.version_info < (3, 4) and isinstance(node, Name):
             if node.id in _safe_names:
                 return _safe_names[node.id]
-        elif isinstance(node, UnaryOp) and \
-             isinstance(node.op, (UAdd, USub)) and \
-             isinstance(node.operand, (Num, UnaryOp, BinOp)):  # NOQA
+        elif (
+            isinstance(node, UnaryOp)
+            and isinstance(node.op, (UAdd, USub))
+            and isinstance(node.operand, (Num, UnaryOp, BinOp))
+        ):  # NOQA
             operand = _convert(node.operand)
             if isinstance(node.op, UAdd):
-                return + operand
+                return +operand
             else:
-                return - operand
-        elif isinstance(node, BinOp) and \
-             isinstance(node.op, (Add, Sub)) and \
-             isinstance(node.right, (Num, UnaryOp, BinOp)) and \
-             isinstance(node.left, (Num, UnaryOp, BinOp)):  # NOQA
+                return -operand
+        elif (
+            isinstance(node, BinOp)
+            and isinstance(node.op, (Add, Sub))
+            and isinstance(node.right, (Num, UnaryOp, BinOp))
+            and isinstance(node.left, (Num, UnaryOp, BinOp))
+        ):  # NOQA
             left = _convert(node.left)
             right = _convert(node.right)
             if isinstance(node.op, Add):
@@ -121,6 +161,7 @@ def literal_eval(node_or_string):
         err.text = repr(node)
         err.node = node
         raise err
+
     return _convert(node_or_string)
 
 
@@ -133,23 +174,23 @@ def _package_data(fn):
         for line in fp.readlines():
             if sys.version_info < (3,):
                 line = line.decode('utf-8')
-            if line.startswith(u'_package_data'):
+            if line.startswith('_package_data'):
                 if 'dict(' in line:
                     parsing = 'python'
-                    lines.append(u'dict(\n')
-                elif line.endswith(u'= {\n'):
+                    lines.append('dict(\n')
+                elif line.endswith('= {\n'):
                     parsing = 'python'
-                    lines.append(u'{\n')
+                    lines.append('{\n')
                 else:
                     raise NotImplementedError
                 continue
             if not parsing:
                 continue
             if parsing == 'python':
-                if line.startswith(u')') or line.startswith(u'}'):
+                if line.startswith(')') or line.startswith('}'):
                     lines.append(line)
                     try:
-                        data = literal_eval(u''.join(lines))
+                        data = literal_eval("".join(lines))
                     except SyntaxError as e:
                         context = 2
                         from_line = e.lineno - (context + 1)
@@ -157,11 +198,16 @@ def _package_data(fn):
                         w = len(str(to_line))
                         for index, line in enumerate(lines):
                             if from_line <= index <= to_line:
-                                print(u"{0:{1}}: {2}".format(index, w, line).encode('utf-8'),
-                                      end=u'')
+                                print(
+                                    '{0:{1}}: {2}'.format(index, w, line).encode('utf-8'),
+                                    end="",
+                                )
                                 if index == e.lineno - 1:
-                                    print(u"{0:{1}}  {2}^--- {3}".format(
-                                        u' ', w, u' ' * e.offset, e.node))
+                                    print(
+                                        '{0:{1}}  {2}^--- {3}'.format(
+                                            ' ', w, ' ' * e.offset, e.node
+                                        )
+                                    )
                         raise
                     break
                 lines.append(line)
@@ -173,32 +219,29 @@ def _package_data(fn):
 # make sure you can run "python ../some/dir/setup.py install"
 pkg_data = _package_data(__file__.replace('setup.py', '__init__.py'))
 
-exclude_files = [
-    'setup.py',
-]
+exclude_files = ['setup.py']
 
 
 # # helper
 def _check_convert_version(tup):
     """Create a PEP 386 pseudo-format conformant string from tuple tup."""
     ret_val = str(tup[0])  # first is always digit
-    next_sep = "."  # separator for next extension, can be "" or "."
+    next_sep = '.'  # separator for next extension, can be "" or "."
     nr_digits = 0  # nr of adjacent digits in rest, to verify
     post_dev = False  # are we processig post/dev
     for x in tup[1:]:
         if isinstance(x, int):
             nr_digits += 1
             if nr_digits > 2:
-                raise ValueError("too many consecutive digits after " + ret_val)
+                raise ValueError('too many consecutive digits after ' + ret_val)
             ret_val += next_sep + str(x)
             next_sep = '.'
             continue
         first_letter = x[0].lower()
-        next_sep = ''
+        next_sep = ""
         if first_letter in 'abcr':
             if post_dev:
-                raise ValueError("release level specified after "
-                                 "post/dev: " + x)
+                raise ValueError('release level specified after ' 'post/dev: ' + x)
             nr_digits = 0
             ret_val += 'rc' if first_letter == 'r' else first_letter
         elif first_letter in 'pd':
@@ -220,8 +263,7 @@ version_str = _check_convert_version(version_info)
 class MyInstallLib(install_lib.install_lib):
     def install(self):
         fpp = pkg_data['full_package_name'].split('.')  # full package path
-        full_exclude_files = [os.path.join(*(fpp + [x]))
-                              for x in exclude_files]
+        full_exclude_files = [os.path.join(*(fpp + [x])) for x in exclude_files]
         alt_files = []
         outfiles = install_lib.install_lib.install(self)
         for x in outfiles:
@@ -251,7 +293,7 @@ class MySdist(_sdist):
 # try except so this doesn't bomb when you don't have wheel installed, implies
 # generation of wheels in ./dist
 try:
-    from wheel.bdist_wheel import bdist_wheel as _bdist_wheel   # NOQA
+    from wheel.bdist_wheel import bdist_wheel as _bdist_wheel  # NOQA
 
     class MyBdistWheel(_bdist_wheel):
         def initialize_options(self):
@@ -268,74 +310,6 @@ except ImportError:
     _bdist_wheel_available = False
 
 
-class InMemoryZipFile(object):
-    def __init__(self, file_name=None):
-        try:
-            from cStringIO import StringIO
-        except ImportError:
-            from io import BytesIO as StringIO
-        import zipfile
-        self.zip_file = zipfile
-        # Create the in-memory file-like object
-        self._file_name = file_name
-        self.in_memory_data = StringIO()
-        # Create the in-memory zipfile
-        self.in_memory_zip = self.zip_file.ZipFile(
-            self.in_memory_data, "w", self.zip_file.ZIP_DEFLATED, False)
-        self.in_memory_zip.debug = 3
-
-    def append(self, filename_in_zip, file_contents):
-        '''Appends a file with name filename_in_zip and contents of
-        file_contents to the in-memory zip.'''
-        self.in_memory_zip.writestr(filename_in_zip, file_contents)
-        return self   # so you can daisy-chain
-
-    def write_to_file(self, filename):
-        '''Writes the in-memory zip to a file.'''
-        # Mark the files as having been created on Windows so that
-        # Unix permissions are not inferred as 0000
-        for zfile in self.in_memory_zip.filelist:
-            zfile.create_system = 0
-        self.in_memory_zip.close()
-        with open(filename, 'wb') as f:
-            f.write(self.in_memory_data.getvalue())
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        if self._file_name is None:
-            return
-        self.write_to_file(self._file_name)
-
-    def delete_from_zip_file(self, pattern=None, file_names=None):
-        """
-        zip_file can be a string or a zipfile.ZipFile object, the latter will be closed
-        any name in file_names is deleted, all file_names provided have to be in the ZIP
-        archive or else an IOError is raised
-        """
-        if pattern and isinstance(pattern, string_type):
-            import re
-            pattern = re.compile(pattern)
-        if file_names:
-            if not isinstance(file_names, list):
-                file_names = [file_names]
-        else:
-            file_names = []
-        with self.zip_file.ZipFile(self._file_name) as zf:
-            for l in zf.infolist():
-                if l.filename in file_names:
-                    file_names.remove(l.filename)
-                    continue
-                if pattern and pattern.match(l.filename):
-                    continue
-                self.append(l.filename, zf.read(l))
-            if file_names:
-                raise IOError('[Errno 2] No such file{}: {}'.format(
-                    '' if len(file_names) == 1 else 's',
-                    ', '.join([repr(f) for f in file_names])))
-
-
 class NameSpacePackager(object):
     def __init__(self, pkg_data):
         assert isinstance(pkg_data, dict)
@@ -344,11 +318,16 @@ class NameSpacePackager(object):
         self._split = None
         self.depth = self.full_package_name.count('.')
         self.nested = self._pkg_data.get('nested', False)
+        if self.nested:
+            NameSpaceInstaller.install_namespaces = lambda x: None
         self.command = None
         self.python_version()
         self._pkg = [None, None]  # required and pre-installable packages
-        if sys.argv[0] == 'setup.py' and sys.argv[1] == 'install' and \
-           '--single-version-externally-managed' not in sys.argv:
+        if (
+            sys.argv[0] == 'setup.py'
+            and sys.argv[1] == 'install'
+            and '--single-version-externally-managed' not in sys.argv
+        ):
             if os.environ.get('READTHEDOCS', None) == 'True':
                 os.system('pip install .')
                 sys.exit(0)
@@ -369,7 +348,7 @@ class NameSpacePackager(object):
             break
 
     def pn(self, s):
-        if sys.version_info < (3, ) and isinstance(s, unicode):
+        if sys.version_info < (3,) and isinstance(s, unicode):
             return s.encode('utf-8')
         return s
 
@@ -377,16 +356,17 @@ class NameSpacePackager(object):
     def split(self):
         """split the full package name in list of compontents traditionally
         done by setuptools.find_packages. This routine skips any directories
-        with __init__.py that start with "_" or ".", or contain a
+        with __init__.py, for which the name starts with "_" or ".", or contain a
         setup.py/tox.ini (indicating a subpackage)
         """
+        skip = []
         if self._split is None:
             fpn = self.full_package_name.split('.')
             self._split = []
             while fpn:
                 self._split.insert(0, '.'.join(fpn))
                 fpn = fpn[:-1]
-            for d in os.listdir('.'):
+            for d in sorted(os.listdir('.')):
                 if not os.path.isdir(d) or d == self._split[0] or d[0] in '._':
                     continue
                 # prevent sub-packages in namespace from being included
@@ -394,16 +374,22 @@ class NameSpacePackager(object):
                 if os.path.exists(x):
                     pd = _package_data(x)
                     if pd.get('nested', False):
+                        skip.append(d)
                         continue
                     self._split.append(self.full_package_name + '.' + d)
-            if sys.version_info < (3, ):
-                self._split = [(y.encode('utf-8') if isinstance(y, unicode) else y)
-                               for y in self._split]
+            if sys.version_info < (3,):
+                self._split = [
+                    (y.encode('utf-8') if isinstance(y, unicode) else y) for y in self._split
+                ]
+        if skip:
+            # this interferes with output checking
+            # print('skipping sub-packages:', ', '.join(skip))
+            pass
         return self._split
 
     @property
     def namespace_packages(self):
-        return self.split[:self.depth]
+        return self.split[: self.depth]
 
     def namespace_directories(self, depth=None):
         """return list of directories where the namespace should be created /
@@ -421,8 +407,10 @@ class NameSpacePackager(object):
     def package_dir(self):
         d = {
             # don't specify empty dir, clashes with package_data spec
-            self.full_package_name: '.',
+            self.full_package_name: '.'
         }
+        if 'extra_packages' in self._pkg_data:
+            return d
         if len(self.split) > 1:  # only if package namespace
             d[self.split[0]] = self.namespace_directories(1)[0]
         return d
@@ -436,8 +424,9 @@ class NameSpacePackager(object):
             for d in directories:
                 os.mkdir(d)
                 with open(os.path.join(d, '__init__.py'), 'w') as fp:
-                    fp.write('import pkg_resources\n'
-                             'pkg_resources.declare_namespace(__name__)\n')
+                    fp.write(
+                        'import pkg_resources\n' 'pkg_resources.declare_namespace(__name__)\n'
+                    )
 
     def python_version(self):
         supported = self._pkg_data.get('supported')
@@ -493,12 +482,10 @@ class NameSpacePackager(object):
                     if self.command == 'develop':
                         raise InstallationError(
                             'Cannot mix develop (pip install -e),\nwith '
-                            'non-develop installs for package name {0}'.format(
-                                fn))
+                            'non-develop installs for package name {0}'.format(fn)
+                        )
                 elif fn == prefix:
-                    raise InstallationError(
-                        'non directory package {0} in {1}'.format(
-                            fn, p))
+                    raise InstallationError('non directory package {0} in {1}'.format(fn, p))
                 for pre in [x + '.' for x in prefixes]:
                     if fn.startswith(pre):
                         break
@@ -507,7 +494,8 @@ class NameSpacePackager(object):
                 if fn.endswith('-link') and self.command == 'install':
                     raise InstallationError(
                         'Cannot mix non-develop with develop\n(pip install -e)'
-                        ' installs for package name {0}'.format(fn))
+                        ' installs for package name {0}'.format(fn)
+                    )
 
     def entry_points(self, script_name=None, package_name=None):
         """normally called without explicit script_name and package name
@@ -521,13 +509,15 @@ class NameSpacePackager(object):
         if the ep entry is a simple string without "=", that is assumed to be
         the name of the script.
         """
+
         def pckg_entry_point(name):
             return '{0}{1}:main'.format(
-                name,
-                '.__main__' if os.path.exists('__main__.py') else '',
+                name, '.__main__' if os.path.exists('__main__.py') else ""
             )
 
         ep = self._pkg_data.get('entry_points', True)
+        if isinstance(ep, dict):
+            return ep
         if ep is None:
             return None
         if ep not in [True, 1]:
@@ -541,25 +531,29 @@ class NameSpacePackager(object):
             package_name = self.full_package_name
         if not script_name:
             script_name = package_name.split('.')[-1]
-        return {'console_scripts': [
-            '{0} = {1}'.format(script_name, pckg_entry_point(package_name)),
-        ]}
+        return {
+            'console_scripts': [
+                '{0} = {1}'.format(script_name, pckg_entry_point(package_name))
+            ]
+        }
 
     @property
     def url(self):
-        if self.full_package_name.startswith('ruamel.'):
-            sp = self.full_package_name.split('.', 1)
-        else:
-            sp = ['ruamel', self.full_package_name]
-        return 'https://bitbucket.org/{0}/{1}'.format(*sp)
+        url = self._pkg_data.get('url')
+        if url:
+            return url
+        sp = self.full_package_name
+        for ch in '_.':
+            sp = sp.replace(ch, '-')
+        return 'https://sourceforge.net/p/{0}/code/ci/default/tree'.format(sp)
 
     @property
     def author(self):
-        return self._pkg_data['author']
+        return self._pkg_data['author']  # no get needs to be there
 
     @property
     def author_email(self):
-        return self._pkg_data['author_email']
+        return self._pkg_data['author_email']  # no get needs to be there
 
     @property
     def license(self):
@@ -568,7 +562,7 @@ class NameSpacePackager(object):
         if lic is None:
             # lic_fn = os.path.join(os.path.dirname(__file__), 'LICENSE')
             # assert os.path.exists(lic_fn)
-            return "MIT license"
+            return 'MIT license'
         return lic
 
     def has_mit_lic(self):
@@ -576,34 +570,50 @@ class NameSpacePackager(object):
 
     @property
     def description(self):
-        return self._pkg_data['description']
+        return self._pkg_data['description']  # no get needs to be there
 
     @property
     def status(self):
         # αβ
-        status = self._pkg_data.get('status', u'β').lower()
-        if status in [u'α', u'alpha']:
+        status = self._pkg_data.get('status', 'β').lower()
+        if status in ['α', 'alpha']:
             return (3, 'Alpha')
-        elif status in [u'β', u'beta']:
+        elif status in ['β', 'beta']:
             return (4, 'Beta')
-        elif u'stable' in status.lower():
+        elif 'stable' in status.lower():
             return (5, 'Production/Stable')
         raise NotImplementedError
 
     @property
     def classifiers(self):
-        return [
-            'Development Status :: {0} - {1}'.format(*self.status),
-            'Intended Audience :: Developers',
-            'License :: ' + ('OSI Approved :: MIT' if self.has_mit_lic()
-                             else 'Other/Proprietary') + ' License',
-            'Operating System :: OS Independent',
-            'Programming Language :: Python',
-        ] + [self.pn(x) for x in self._pkg_data.get('classifiers', [])]
+        """this needs more intelligence, probably splitting the classifiers from _pkg_data
+        and only adding defaults when no explicit entries were provided.
+        Add explicit Python versions in sync with tox.env generation based on python_requires?
+        """
+        attr = '_' + sys._getframe().f_code.co_name
+        if not hasattr(self, attr):
+            setattr(self, attr, self._setup_classifiers())
+        return getattr(self, attr)
+
+    def _setup_classifiers(self):
+        return sorted(
+            set(
+                [
+                    'Development Status :: {0} - {1}'.format(*self.status),
+                    'Intended Audience :: Developers',
+                    'License :: '
+                    + ('OSI Approved :: MIT' if self.has_mit_lic() else 'Other/Proprietary')
+                    + ' License',
+                    'Operating System :: OS Independent',
+                    'Programming Language :: Python',
+                ]
+                + [self.pn(x) for x in self._pkg_data.get('classifiers', [])]
+            )
+        )
 
     @property
     def keywords(self):
-        return self.pn(self._pkg_data.get('keywords'))
+        return self.pn(self._pkg_data.get('keywords', []))
 
     @property
     def install_requires(self):
@@ -637,7 +647,7 @@ class NameSpacePackager(object):
         # 'any' for all builds, 'py27' etc for specifics versions
         packages = ir.get('any', [])
         if isinstance(packages, string_type):
-            packages = packages.split()    # assume white space separated string
+            packages = packages.split()  # assume white space separated string
         if self.nested:
             # parent dir is also a package, make sure it is installed (need its .pth file)
             parent_pkg = self.full_package_name.rsplit('.', 1)[0]
@@ -647,7 +657,7 @@ class NameSpacePackager(object):
         if implementation == 'CPython':
             pyver = 'py{0}{1}'.format(*sys.version_info)
         elif implementation == 'PyPy':
-            pyver = 'pypy' if sys.version_info < (3, ) else 'pypy3'
+            pyver = 'pypy' if sys.version_info < (3,) else 'pypy3'
         elif implementation == 'Jython':
             pyver = 'jython'
         packages.extend(ir.get(pyver, []))
@@ -673,14 +683,14 @@ class NameSpacePackager(object):
         ep = self._pkg_data.get('extras_require')
         return ep
 
-    @property
-    def data_files(self):
-        df = self._pkg_data.get('data_files', [])
-        if self.has_mit_lic():
-            df.append('LICENSE')
-        if not df:
-            return None
-        return [('.', df), ]
+    # @property
+    # def data_files(self):
+    #     df = self._pkg_data.get('data_files', [])
+    #     if self.has_mit_lic():
+    #         df.append('LICENSE')
+    #     if not df:
+    #         return None
+    #     return [('.', df)]
 
     @property
     def package_data(self):
@@ -690,14 +700,42 @@ class NameSpacePackager(object):
             df.append('LICENSE')
             # but don't install it
             exclude_files.append('LICENSE')
-        if not df:
-            return {}
-        return {self.full_package_name: df}
+        if self._pkg_data.get('binary_only', False):
+            exclude_files.append('__init__.py')
+        debug('testing<<<<<')
+        if 'Typing :: Typed' in self.classifiers:
+            debug('appending')
+            df.append('py.typed')
+        pd = self._pkg_data.get('package_data', {})
+        if df:
+            pd[self.full_package_name] = df
+        if sys.version_info < (3,):
+            # python2 doesn't seem to like unicode package names as keys
+            # maybe only when the packages themselves are non-unicode
+            for k in pd:
+                if isinstance(k, unicode):
+                    pd[str(k)] = pd.pop(k)
+            # for k in pd:
+            #     pd[k] = [e.encode('utf-8') for e in pd[k]]  # de-unicode
+        return pd
+
+    @property
+    def packages(self):
+        s = self.split
+        # fixed this in package_data, the keys there must be non-unicode for py27
+        # if sys.version_info < (3, 0):
+        #     s = [x.encode('utf-8') for x in self.split]
+        return s + self._pkg_data.get('extra_packages', [])
+
+    @property
+    def python_requires(self):
+        return self._pkg_data.get('python_requires', None)
 
     @property
     def ext_modules(self):
-        """check if all modules specified in the value for 'ext_modules' can be build
-        that value (if not None) is a list of dicts with 'name', 'src', 'lib'
+        """
+        Check if all modules specified in the value for 'ext_modules' can be build.
+        That value (if not None) is a list of dicts with 'name', 'src', 'lib'
         Optional 'test' can be used to make sure trying to compile will work on the host
 
         creates and return the external modules as Extensions, unless that
@@ -712,11 +750,6 @@ class NameSpacePackager(object):
             return None
         if platform.python_implementation() == 'Jython':
             return None
-        if sys.platform == "win32" and not self._pkg_data.get('win32bin'):
-            return None
-        if sys.platform == "win32":
-            if os.getenv("RUAMEL_FORCE_EXT_BUILD") is None:
-                return None
         try:
             plat = sys.argv.index('--plat-name')
             if 'win' in sys.argv[plat + 1]:
@@ -754,9 +787,15 @@ class NameSpacePackager(object):
                 sources=[self.pn(x) for x in target['src']],
                 libraries=[self.pn(x) for x in target.get('lib')],
             )
-            if 'test' not in target:  # no test just hope it works
+            # debug('test1 in target', 'test' in target, target)
+            if 'test' not in target:  # no test, just hope it works
                 self._ext_modules.append(ext)
                 continue
+            if sys.version_info[:2] == (3, 4) and platform.system() == 'Windows':
+                # this is giving problems on appveyor, so skip
+                if 'FORCE_C_BUILD_TEST' not in os.environ:
+                    self._ext_modules.append(ext)
+                    continue
             # write a temporary .c file to compile
             c_code = dedent(target['test'])
             try:
@@ -773,53 +812,58 @@ class NameSpacePackager(object):
                 distutils.sysconfig.customize_compiler(compiler)
                 # make sure you can reach header files because compile does change dir
                 compiler.add_include_dir(os.getcwd())
-                if sys.version_info < (3, ):
+                if sys.version_info < (3,):
                     tmp_dir = tmp_dir.encode('utf-8')
                 # used to be a different directory, not necessary
                 compile_out_dir = tmp_dir
                 try:
                     compiler.link_executable(
-                        compiler.compile(
-                            [file_name],
-                            output_dir=compile_out_dir,
-                        ),
+                        compiler.compile([file_name], output_dir=compile_out_dir),
                         bin_file_name,
                         output_dir=tmp_dir,
                         libraries=ext.libraries,
                     )
                 except CompileError:
+                    debug('compile error:', file_name)
                     print('compile error:', file_name)
                     continue
                 except LinkError:
-                    print('libyaml link error', file_name)
+                    debug('link error', file_name)
+                    print('link error', file_name)
                     continue
                 self._ext_modules.append(ext)
             except Exception as e:  # NOQA
+                debug('Exception:', e)
                 print('Exception:', e)
-                pass
+                if sys.version_info[:2] == (3, 4) and platform.system() == 'Windows':
+                    traceback.print_exc()
             finally:
                 shutil.rmtree(tmp_dir)
         return self._ext_modules
 
+    @property
+    def test_suite(self):
+        return self._pkg_data.get('test_suite')
+
     def wheel(self, kw, setup):
         """temporary add setup.cfg if creating a wheel to include LICENSE file
         https://bitbucket.org/pypa/wheel/issues/47
         """
         if 'bdist_wheel' not in sys.argv:
-            return
+            return False
         file_name = 'setup.cfg'
         if os.path.exists(file_name):  # add it if not in there?
-            return
+            return False
         with open(file_name, 'w') as fp:
             if os.path.exists('LICENSE'):
                 fp.write('[metadata]\nlicense-file = LICENSE\n')
             else:
-                print("\n\n>>>>>> LICENSE file not found <<<<<\n\n")
+                print('\n\n>>>>>> LICENSE file not found <<<<<\n\n')
             if self._pkg_data.get('universal'):
                 fp.write('[bdist_wheel]\nuniversal = 1\n')
         try:
             setup(**kw)
-        except:
+        except Exception:
             raise
         finally:
             os.remove(file_name)
@@ -833,6 +877,7 @@ def main():
         import wheel
         import distutils
         import setuptools
+
         print('python:    ', sys.version)
         print('setuptools:', setuptools.__version__)
         print('distutils: ', distutils.__version__)
@@ -844,10 +889,7 @@ def main():
     if pkg_data.get('tarfmt'):
         MySdist.tarfmt = pkg_data.get('tarfmt')
 
-    cmdclass = dict(
-        install_lib=MyInstallLib,
-        sdist=MySdist,
-    )
+    cmdclass = dict(install_lib=MyInstallLib, sdist=MySdist)
     if _bdist_wheel_available:
         MyBdistWheel.nsp = nsp
         cmdclass['bdist_wheel'] = MyBdistWheel
@@ -856,7 +898,8 @@ def main():
         name=nsp.full_package_name,
         namespace_packages=nsp.namespace_packages,
         version=version_str,
-        packages=nsp.split,
+        packages=nsp.packages,
+        python_requires=nsp.python_requires,
         url=nsp.url,
         author=nsp.author,
         author_email=nsp.author_email,
@@ -871,19 +914,24 @@ def main():
         keywords=nsp.keywords,
         package_data=nsp.package_data,
         ext_modules=nsp.ext_modules,
+        test_suite=nsp.test_suite,
     )
 
     if '--version' not in sys.argv and ('--verbose' in sys.argv or dump_kw in sys.argv):
         for k in sorted(kw):
             v = kw[k]
             print('  "{0}": "{1}",'.format(k, v))
+    # if '--record' in sys.argv:
+    #     return
     if dump_kw in sys.argv:
         sys.argv.remove(dump_kw)
     try:
         with open('README.rst') as fp:
             kw['long_description'] = fp.read()
-    except:
+            kw['long_description_content_type'] = 'text/x-rst'
+    except Exception:
         pass
+
     if nsp.wheel(kw, setup):
         return
     for x in ['-c', 'egg_info', '--egg-base', 'pip-egg-info']:
@@ -894,6 +942,7 @@ def main():
         # until you match your/package/name for your.package.name
         for p in nsp.install_pre:
             import subprocess
+
             # search other source
             setup_path = os.path.join(*p.split('.') + ['setup.py'])
             try_dir = os.path.dirname(sys.executable)
@@ -908,15 +957,6 @@ def main():
                     break
                 try_dir = os.path.dirname(try_dir)
     setup(**kw)
-    if nsp.nested and sys.argv[:2] == ['-c', 'bdist_wheel']:
-        d = sys.argv[sys.argv.index('-d') + 1]
-        for x in os.listdir(d):
-            if x.endswith('.whl'):
-                # remove .pth file from the wheel
-                full_name = os.path.join(d, x)
-                with InMemoryZipFile(full_name) as imz:
-                    imz.delete_from_zip_file(nsp.full_package_name + '.*.pth')
-                break
 
 
 main()
diff --git a/tox.ini b/tox.ini
index e1ba3bc..3ee992c 100644..100755
--- a/tox.ini
+++ b/tox.ini
@@ -1,18 +1,36 @@
 [tox]
-envlist = pep8,py35,py27,py34,pypy
+# toxworkdir = /data1/DATA/tox/ruamel.std.argparse 
+envlist = cs,py38,py27,py37,py36,py35,py39,pypy
 
 [testenv]
 commands =
-    py.test _test
+    /bin/bash -c 'pytest _test/test_*.py'
 deps =
     pytest
-    flake8==2.5.5
+
+[testenv:cs]
+basepython = python3.6
+deps =
+    flake8
+    flake8-bugbear;python_version>="3.5"
+commands =
+    flake8 []{posargs}
 
 [testenv:pep8]
+basepython = python3.6
+deps =
+    flake8
+    flake8-bugbear;python_version>="3.5"
 commands =
-   flake8 {posargs}
+    flake8 []{posargs}
 
 [flake8]
 show-source = True
 max-line-length = 95
+ignore = W503,F405,E203
 exclude = .hg,.git,.tox,dist,.cache,__pycache__,ruamel.zip2tar.egg-info
+
+[pytest]
+filterwarnings =
+    error::DeprecationWarning
+    error::PendingDeprecationWarning