try loading trusted certs from a list of fallbacks (#633)

* try loading trusted certs from a list of fallbacks

pyca/cryptography will shortly begin shipping a wheel. Since
SSL_CTX_set_default_verify_paths uses a hardcoded path compiled into the
library, this will start failing to load the proper certificates for
users on many linux distributions. To avoid this we can use the Go
solution of iterating over a list of potential candidates and loading
it when found.

* capath is lazy loaded so we need to do a lot more checks

This now checks to see if env vars are set as well as seeing if the
dir exists and has valid certs in it. If either of those are true (or
the number of certs is > 0) it won't load the fallback. If it does do
the fallback it will also attempt to load certs from a dir as a final
fallback

* remove an early return

* this shouldn't be commented out

* oops

* very limited testing

* sigh, can't use these py3 exceptions of course

* expand the tests a bit

* coverage!

* don't need this now

* change the approach to use a pyca/cryptography guard value

* test fix

* older python sometimes calls itself linux2

* flake8

* add changelog

* coverage

* slash opt

4 files changed, 168 insertions, 1 deletions

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 871b1d5..86f6466 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -36,6 +36,7 @@ Changes:
 - Added ``OpenSSL.crypto.X509Req.from_cryptography``, ``OpenSSL.crypto.X509Req.to_cryptography``, ``OpenSSL.crypto.CRL.from_cryptography``, and ``OpenSSL.crypto.CRL.to_cryptography`` for converting X.509 CSRs and CRLs to and from pyca/cryptography objects. `#645 <https://github.com/pyca/pyopenssl/pull/645>`_
 - Added ``OpenSSL.debug`` that allows to get an overview of used library versions (including linked OpenSSL) and other useful runtime information using ``python -m OpenSSL.debug``.
   `#620 <https://github.com/pyca/pyopenssl/pull/620>`_
+- Added a fallback path to `Context.set_default_verify_paths` to accommodate the upcoming release of ``cryptography`` ``manylinux1`` wheels. `#633 <https://github.com/pyca/pyopenssl/pull/633>`_
 
 
 ----
diff --git a/src/OpenSSL/SSL.py b/src/OpenSSL/SSL.py
index 85d6e76..b571a5e 100644
--- a/src/OpenSSL/SSL.py
+++ b/src/OpenSSL/SSL.py
@@ -1,3 +1,4 @@
+import os
 import socket
 from sys import platform
 from functools import wraps, partial
@@ -132,6 +133,22 @@ SSL_CB_CONNECT_EXIT = _lib.SSL_CB_CONNECT_EXIT
 SSL_CB_HANDSHAKE_START = _lib.SSL_CB_HANDSHAKE_START
 SSL_CB_HANDSHAKE_DONE = _lib.SSL_CB_HANDSHAKE_DONE
 
+# Taken from https://golang.org/src/crypto/x509/root_linux.go
+_CERTIFICATE_FILE_LOCATIONS = [
+    "/etc/ssl/certs/ca-certificates.crt",  # Debian/Ubuntu/Gentoo etc.
+    "/etc/pki/tls/certs/ca-bundle.crt",  # Fedora/RHEL 6
+    "/etc/ssl/ca-bundle.pem",  # OpenSUSE
+    "/etc/pki/tls/cacert.pem",  # OpenELEC
+    "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",  # CentOS/RHEL 7
+]
+
+_CERTIFICATE_PATH_LOCATIONS = [
+    "/etc/ssl/certs",  # SLES10/SLES11
+]
+
+_CRYPTOGRAPHY_MANYLINUX1_CA_DIR = "/opt/pyca/cryptography/openssl/certs"
+_CRYPTOGRAPHY_MANYLINUX1_CA_FILE = "/opt/pyca/cryptography/openssl/cert.pem"
+
 
 class Error(Exception):
     """
@@ -701,8 +718,69 @@ class Context(object):
 
         :return: None
         """
+        # SSL_CTX_set_default_verify_paths will attempt to load certs from
+        # both a cafile and capath that are set at compile time. However,
+        # it will first check environment variables and, if present, load
+        # those paths instead
         set_result = _lib.SSL_CTX_set_default_verify_paths(self._context)
         _openssl_assert(set_result == 1)
+        # After attempting to set default_verify_paths we need to know whether
+        # to go down the fallback path.
+        # First we'll check to see if any env vars have been set. If so,
+        # we won't try to do anything else because the user has set the path
+        # themselves.
+        dir_env_var = _ffi.string(
+            _lib.X509_get_default_cert_dir_env()
+        ).decode("ascii")
+        file_env_var = _ffi.string(
+            _lib.X509_get_default_cert_file_env()
+        ).decode("ascii")
+        if not self._check_env_vars_set(dir_env_var, file_env_var):
+            default_dir = _ffi.string(_lib.X509_get_default_cert_dir())
+            default_file = _ffi.string(_lib.X509_get_default_cert_file())
+            # Now we check to see if the default_dir and default_file are set
+            # to the exact values we use in our manylinux1 builds. If they are
+            # then we know to load the fallbacks
+            if (
+                default_dir == _CRYPTOGRAPHY_MANYLINUX1_CA_DIR and
+                default_file == _CRYPTOGRAPHY_MANYLINUX1_CA_FILE
+            ):
+                # This is manylinux1, let's load our fallback paths
+                self._fallback_default_verify_paths(
+                    _CERTIFICATE_FILE_LOCATIONS,
+                    _CERTIFICATE_PATH_LOCATIONS
+                )
+
+    def _check_env_vars_set(self, dir_env_var, file_env_var):
+        """
+        Check to see if the default cert dir/file environment vars are present.
+
+        :return: bool
+        """
+        return (
+            os.environ.get(file_env_var) is not None or
+            os.environ.get(dir_env_var) is not None
+        )
+
+    def _fallback_default_verify_paths(self, file_path, dir_path):
+        """
+        Default verify paths are based on the compiled version of OpenSSL.
+        However, when pyca/cryptography is compiled as a manylinux1 wheel
+        that compiled location can potentially be wrong. So, like Go, we
+        will try a predefined set of paths and attempt to load roots
+        from there.
+
+        :return: None
+        """
+        for cafile in file_path:
+            if os.path.isfile(cafile):
+                self.load_verify_locations(cafile)
+                break
+
+        for capath in dir_path:
+            if os.path.isdir(capath):
+                self.load_verify_locations(None, capath)
+                break
 
     def use_certificate_chain_file(self, certfile):
         """
diff --git a/tests/test_ssl.py b/tests/test_ssl.py
index 96efec8..fafffa3 100644
--- a/tests/test_ssl.py
+++ b/tests/test_ssl.py
@@ -20,6 +20,8 @@ from warnings import simplefilter
 
 import pytest
 
+from pretend import raiser
+
 from six import PY3, text_type
 
 from cryptography import x509
@@ -46,6 +48,7 @@ from OpenSSL.SSL import OP_SINGLE_DH_USE, OP_NO_SSLv2, OP_NO_SSLv3
 from OpenSSL.SSL import (
     VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_CLIENT_ONCE, VERIFY_NONE)
 
+from OpenSSL import SSL
 from OpenSSL.SSL import (
     SESS_CACHE_OFF, SESS_CACHE_CLIENT, SESS_CACHE_SERVER, SESS_CACHE_BOTH,
     SESS_CACHE_NO_AUTO_CLEAR, SESS_CACHE_NO_INTERNAL_LOOKUP,
@@ -57,7 +60,7 @@ from OpenSSL.SSL import (
     Context, ContextType, Session, Connection, ConnectionType, SSLeay_version)
 from OpenSSL.SSL import _make_requires
 
-from OpenSSL._util import lib as _lib
+from OpenSSL._util import ffi as _ffi, lib as _lib
 
 from OpenSSL.SSL import (
     OP_NO_QUERY_MTU, OP_COOKIE_EXCHANGE, OP_NO_TICKET, OP_NO_COMPRESSION,
@@ -1109,6 +1112,79 @@ class TestContext(object):
             context.load_verify_locations(object(), object())
 
     @pytest.mark.skipif(
+        not platform.startswith("linux"),
+        reason="Loading fallback paths is a linux-specific behavior to "
+        "accommodate pyca/cryptography manylinux1 wheels"
+    )
+    def test_fallback_default_verify_paths(self, monkeypatch):
+        """
+        Test that we load certificates successfully on linux from the fallback
+        path. To do this we set the _CRYPTOGRAPHY_MANYLINUX1_CA_FILE and
+        _CRYPTOGRAPHY_MANYLINUX1_CA_DIR vars to be equal to whatever the
+        current OpenSSL default is and we disable
+        SSL_CTX_SET_default_verify_paths so that it can't find certs unless
+        it loads via fallback.
+        """
+        context = Context(TLSv1_METHOD)
+        monkeypatch.setattr(
+            _lib, "SSL_CTX_set_default_verify_paths", lambda x: 1
+        )
+        monkeypatch.setattr(
+            SSL,
+            "_CRYPTOGRAPHY_MANYLINUX1_CA_FILE",
+            _ffi.string(_lib.X509_get_default_cert_file())
+        )
+        monkeypatch.setattr(
+            SSL,
+            "_CRYPTOGRAPHY_MANYLINUX1_CA_DIR",
+            _ffi.string(_lib.X509_get_default_cert_dir())
+        )
+        context.set_default_verify_paths()
+        store = context.get_cert_store()
+        sk_obj = _lib.X509_STORE_get0_objects(store._store)
+        assert sk_obj != _ffi.NULL
+        num = _lib.sk_X509_OBJECT_num(sk_obj)
+        assert num != 0
+
+    def test_check_env_vars(self, monkeypatch):
+        """
+        Test that we return True/False appropriately if the env vars are set.
+        """
+        context = Context(TLSv1_METHOD)
+        dir_var = "CUSTOM_DIR_VAR"
+        file_var = "CUSTOM_FILE_VAR"
+        assert context._check_env_vars_set(dir_var, file_var) is False
+        monkeypatch.setenv(dir_var, "value")
+        monkeypatch.setenv(file_var, "value")
+        assert context._check_env_vars_set(dir_var, file_var) is True
+        assert context._check_env_vars_set(dir_var, file_var) is True
+
+    def test_verify_no_fallback_if_env_vars_set(self, monkeypatch):
+        """
+        Test that we don't use the fallback path if env vars are set.
+        """
+        context = Context(TLSv1_METHOD)
+        monkeypatch.setattr(
+            _lib, "SSL_CTX_set_default_verify_paths", lambda x: 1
+        )
+        dir_env_var = _ffi.string(
+            _lib.X509_get_default_cert_dir_env()
+        ).decode("ascii")
+        file_env_var = _ffi.string(
+            _lib.X509_get_default_cert_file_env()
+        ).decode("ascii")
+        monkeypatch.setenv(dir_env_var, "value")
+        monkeypatch.setenv(file_env_var, "value")
+        context.set_default_verify_paths()
+
+        monkeypatch.setattr(
+            context,
+            "_fallback_default_verify_paths",
+            raiser(SystemError)
+        )
+        context.set_default_verify_paths()
+
+    @pytest.mark.skipif(
         platform == "win32",
         reason="set_default_verify_paths appears not to work on Windows.  "
         "See LP#404343 and LP#404344."
@@ -1141,6 +1217,17 @@ class TestContext(object):
         clientSSL.send(b"GET / HTTP/1.0\r\n\r\n")
         assert clientSSL.recv(1024)
 
+    def test_fallback_path_is_not_file_or_dir(self):
+        """
+        Test that when passed empty arrays or paths that do not exist no
+        errors are raised.
+        """
+        context = Context(TLSv1_METHOD)
+        context._fallback_default_verify_paths([], [])
+        context._fallback_default_verify_paths(
+            ["/not/a/file"], ["/not/a/dir"]
+        )
+
     def test_add_extra_chain_cert_invalid_cert(self):
         """
         `Context.add_extra_chain_cert` raises `TypeError` if called with an
diff --git a/tox.ini b/tox.ini
index 9248041..76e1c5b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,6 +8,7 @@ passenv = ARCHFLAGS CFLAGS LC_ALL LDFLAGS PATH LD_LIBRARY_PATH TERM
 deps =
     coverage>=4.2
     pytest>=3.0.1
+    pretend
     cryptographyMaster: git+https://github.com/pyca/cryptography.git
     cryptographyMinimum: cryptography<=1.9
 setenv =