Fix iteration of BTrees.items() in pure-python; and 3.6 support

Also fix ``list(proxy_btree.items())`` (or a list comprehension of the
same) in Python 3, which wants the ``__len__`` for a hint.

This is a central place to make sure these all behave consistently.

Fixes #20

Also drop pypy3

As a 3.2 implementation, it's not supported by pip anymore. There is a
much more recent version, 3.5-beta, but it's not on Travis yet. The
3.3-alpha which is on Travis is a dead end.

7 files changed, 114 insertions, 50 deletions

diff --git a/.travis.yml b/.travis.yml
index b35a2dd..92655c1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,8 @@ matrix:
   include:
     - python: 3.5
       env: TOXENV=py35
+    - python: 3.6
+      env: TOXENV=py36
 env:
     - TOXENV=py27
     - TOXENV=py27-pure
@@ -11,12 +13,13 @@ env:
     - TOXENV=py33-pure
     - TOXENV=py34
     - TOXENV=pypy
-    - TOXENV=pypy3
     - TOXENV=coverage
     - TOXENV=docs
 install:
-    - pip install tox
+    - pip install -U pip
+    - pip install -U setuptools tox
 script:
     - tox
 notifications:
     email: false
+cache: pip
diff --git a/CHANGES.rst b/CHANGES.rst
index 2766bc7..8d2528e 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -9,7 +9,14 @@ Changes
 
 - Drop support for Python 2.6 and 3.2.
 
-- Add support for Python 3.5.
+- Add support for Python 3.5 and 3.6.
+
+- Fix iteration of pure-Python BTrees.items(). See `issue 20
+  <https://github.com/zopefoundation/zope.security/issues/20>`_.
+
+- Fix creating a list from a BTrees.items() on Python 3.  See `issue 20
+  <https://github.com/zopefoundation/zope.security/issues/20>`_.
+
 
 4.0.3 (2015-06-02)
 ------------------
diff --git a/appveyor.yml b/appveyor.yml
index 7bcc4c5..8ee443a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -9,18 +9,25 @@ environment:
     - python : 34-x64
     - python : 35
     - python : 35-x64
+    - python : 36
+    - python : 36-x64
     - { python: 27, PURE_PYTHON: 1 }
     - { python: 35, PURE_PYTHON: 1 }
 
 install:
   - "SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH%"
   - echo "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 > "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat"
-  - pip install -e .
+# We need to install the C extensions that BTrees setup-requires
+# separately because we've seen problems with the BTrees build cleanup step trying
+# to delete a .pyd that was still open.
+  - pip install persistent
+  - pip install BTrees
+  - pip install -e .[test]
 
 build: false
 
 test_script:
-  - python setup.py -q test -q
+  - python -m zope.testrunner --test-path=src
 
 on_success:
   - echo Build succesful!
diff --git a/setup.py b/setup.py
index 611eea0..bae765c 100644
--- a/setup.py
+++ b/setup.py
@@ -26,6 +26,7 @@ from setuptools import find_packages
 from setuptools import setup
 
 TESTS_REQUIRE = [
+    'BTrees',
     'zope.component',
     'zope.configuration',
     'zope.location',
@@ -50,7 +51,8 @@ def alltests():
 
 here = os.path.abspath(os.path.dirname(__file__))
 def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+    with open(os.path.join(os.path.dirname(__file__), *rnames)) as f:
+        return f.read()
 
 # Include directories for C extensions
 # Sniff the location of the headers in 'persistent' or fall back
@@ -114,8 +116,8 @@ setup(name='zope.security',
           + '\n\n' +
           read('CHANGES.rst')
           ),
-      keywords = "zope security policy principal permission",
-      classifiers = [
+      keywords="zope security policy principal permission",
+      classifiers=[
           'Development Status :: 5 - Production/Stable',
           'Environment :: Web Environment',
           'Intended Audience :: Developers',
@@ -127,12 +129,14 @@ setup(name='zope.security',
           'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
           'Programming Language :: Python :: Implementation :: CPython',
           'Programming Language :: Python :: Implementation :: PyPy',
           'Natural Language :: English',
           'Operating System :: OS Independent',
           'Topic :: Internet :: WWW/HTTP',
-          'Framework :: Zope3'],
+          'Framework :: Zope3',
+      ],
       url='http://pypi.python.org/pypi/zope.security',
       license='ZPL 2.1',
       packages=find_packages('src'),
@@ -140,14 +144,15 @@ setup(name='zope.security',
       namespace_packages=['zope'],
       setup_requires=setup_requires,
       ext_modules=ext_modules,
-      install_requires=['setuptools',
-                        'zope.component',
-                        'zope.i18nmessageid',
-                        'zope.interface',
-                        'zope.location',
-                        'zope.proxy >= 4.1.0',
-                        'zope.schema',
-                        ],
+      install_requires=[
+          'setuptools',
+          'zope.component',
+          'zope.i18nmessageid',
+          'zope.interface',
+          'zope.location',
+          'zope.proxy >= 4.1.0',
+          'zope.schema',
+      ],
       test_suite = '__main__.alltests',
       tests_require=TESTS_REQUIRE,
       extras_require = dict(
@@ -157,7 +162,7 @@ setup(name='zope.security',
           test=TESTS_REQUIRE,
           testing=TESTS_REQUIRE + ['nose', 'coverage'],
           docs=['Sphinx', 'repoze.sphinx.autointerface'],
-          ),
+      ),
       include_package_data = True,
       zip_safe = False,
       )
diff --git a/src/zope/security/checker.py b/src/zope/security/checker.py
index d5f261c..d75cfd5 100644
--- a/src/zope/security/checker.py
+++ b/src/zope/security/checker.py
@@ -770,6 +770,43 @@ if PYTHON2:
     _default_checkers[type({}.iterkeys())] = _iteratorChecker
     _default_checkers[type({}.itervalues())] = _iteratorChecker
 
+try:
+    import BTrees
+except ImportError: # pragma: no cover
+    pass
+else:
+    # The C implementation of BTree.items() is its own iterator
+    # and doesn't need any special entries to enable iteration.
+    # But the Python implementation has to call __iter__ to be able
+    # to do iteration. Whitelist it so that they behave the same.
+    # In addition, Python 3 will attempt to call __len__ on iterators
+    # for a length hint, so the C implementations also need to be
+    # added to the _iteratorChecker.
+    # We do this here so that all users of zope.security can benefit
+    # without knowing implementation details.
+    # See https://github.com/zopefoundation/zope.security/issues/20
+
+    def _fixup_btrees():
+        import BTrees._base
+        _default_checkers[BTrees._base._TreeItems] = _iteratorChecker
+
+        for name in ('IF', 'II', 'IO', 'OI', 'OO'):
+            for family_name in ('family32', 'family64'):
+                family = getattr(BTrees, family_name)
+                btree = getattr(family, name).BTree()
+
+                empty_type = type(btree.items())
+                if empty_type not in _default_checkers:
+                    _default_checkers[empty_type] = _iteratorChecker
+
+                btree[1] = 1
+                populated_type = type(btree.items())
+                if populated_type not in _default_checkers:
+                    _default_checkers[populated_type] = _iteratorChecker
+
+    _fixup_btrees()
+    del _fixup_btrees
+
 def _clear():
     _checkers.clear()
     _checkers.update(_default_checkers)
diff --git a/src/zope/security/tests/test_checker.py b/src/zope/security/tests/test_checker.py
index c5c01e7..d1bad49 100644
--- a/src/zope/security/tests/test_checker.py
+++ b/src/zope/security/tests/test_checker.py
@@ -17,13 +17,15 @@ import unittest
 
 def _skip_if_not_Py2(testfunc):
     import sys
-    from functools import update_wrapper
-    if sys.version_info[0] >= 3:
-        def dummy(self):
-            pass
-        update_wrapper(dummy, testfunc)
-        return dummy
-    return testfunc
+    return unittest.skipIf(sys.version_info[0] >= 3, "Needs Python 2")(testfunc)
+
+def _skip_if_no_btrees(testfunc):
+    try:
+        import BTrees
+    except ImportError:
+        return unittest.skip("BTrees is not installed")(testfunc)
+    else:
+        return testfunc
 
 class Test_ProxyFactory(unittest.TestCase):
 
@@ -389,6 +391,28 @@ class CheckerTestsBase(object):
         finally:
             _clear()
 
+    @_skip_if_no_btrees
+    def test_iteration_of_btree_items(self):
+        # iteration of BTree.items() is allowed by default.
+        from zope.security.proxy import Proxy
+        from zope.security.checker import Checker
+        from zope.security.checker import CheckerPublic
+        import BTrees
+
+        checker = Checker({'items': CheckerPublic})
+
+        for name in ('IF', 'II', 'IO', 'OI', 'OO'):
+            for family_name in ('family32', 'family64'):
+                family = getattr(BTrees, family_name)
+                btree = getattr(family, name).BTree()
+                proxy = Proxy(btree, checker)
+                # empty
+                self.assertEqual([], list(proxy.items()))
+
+                # With an object
+                btree[1] = 2
+                self.assertEqual([(1, 2)], list(proxy.items()))
+
 
 class CheckerPyTests(unittest.TestCase, CheckerTestsBase):
 
diff --git a/tox.ini b/tox.ini
index 40180bc..d9c069a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,19 +3,13 @@ envlist =
 # Jython support pending 2.7 support, due 2012-07-15 or so.  See:
 # http://fwierzbicki.blogspot.com/2012/03/adconion-to-fund-jython-27.html
 #   py27,pypy,jython,py33,coverage,docs
-    py27,py27-pure,pypy,py33,py33-pure,py34,py35,coverage,docs
+    py27,py27-pure,pypy,py33,py33-pure,py34,py35,py36,coverage,docs
 
 [testenv]
 deps =
-    zope.interface
-    zope.testing
-    zope.configuration
-    zope.component
-    zope.location
-    zope.proxy
-    zope.testrunner
+    .[test]
 commands =
-    python setup.py -q test -q
+    zope-testrunner --test-path=src --auto-progress --auto-color []
 
 [testenv:py27-pure]
 basepython =
@@ -43,28 +37,15 @@ commands =
     pip install -e .
     nosetests --with-xunit --with-xcoverage
 deps =
-    zope.interface
-    zope.testing
-    zope.configuration
-    zope.component
-    zope.location
-    zope.proxy
-    nose
+    .[testing]
     coverage
     nosexcover
 
 [testenv:docs]
 basepython =
-    python3.3
+    python3.4
 commands =
     sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html
     sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest
 deps =
-    zope.interface
-    zope.testing
-    zope.configuration
-    zope.component
-    zope.location
-    zope.proxy
-    Sphinx
-    repoze.sphinx.autointerface
+    .[docs,test]