ALPN: complete handshake without accepting a client's protocols. (#876)

* ALPN: complete handshake without accepting a client's protocols.

The callback passed to `SSL_CTX_set_alpn_select_cb` can return
`SSL_TLSEXT_ERR_NOACK` to allow the handshake to continue without
accepting any of the client's offered protocols.

This commit introduces `NO_OVERLAPPING_PROTOCOLS`, which the Python
callback passed to `Context.set_alpn_select_callback` can return to
achieve the same thing.

It does not change the previous meaning of an empty string, which
still terminates the handshake.

* Update src/OpenSSL/SSL.py

Co-Authored-By: Alex Gaynor <alex.gaynor@gmail.com>

* Address @alex's review.

* Use recorded value in test, fix lint error.

* Cover TypeError branch in _ALPNHelper.callback

1 files changed, 25 insertions, 10 deletions

diff --git a/src/OpenSSL/SSL.py b/src/OpenSSL/SSL.py
index adcfd8f..a228b73 100644
--- a/src/OpenSSL/SSL.py
+++ b/src/OpenSSL/SSL.py
@@ -428,6 +428,9 @@ class _NpnSelectHelper(_CallbackExceptionHelper):
         )
 
 
+NO_OVERLAPPING_PROTOCOLS = object()
+
+
 class _ALPNSelectHelper(_CallbackExceptionHelper):
     """
     Wrap a callback such that it can be used as an ALPN selection callback.
@@ -453,24 +456,32 @@ class _ALPNSelectHelper(_CallbackExceptionHelper):
                     instr = instr[encoded_len + 1:]
 
                 # Call the callback
-                outstr = callback(conn, protolist)
-
-                if not isinstance(outstr, _binary_type):
-                    raise TypeError("ALPN callback must return a bytestring.")
+                outbytes = callback(conn, protolist)
+                any_accepted = True
+                if outbytes is NO_OVERLAPPING_PROTOCOLS:
+                    outbytes = b''
+                    any_accepted = False
+                elif not isinstance(outbytes, _binary_type):
+                    raise TypeError(
+                        "ALPN callback must return a bytestring or the "
+                        "special NO_OVERLAPPING_PROTOCOLS sentinel value."
+                    )
 
                 # Save our callback arguments on the connection object to make
                 # sure that they don't get freed before OpenSSL can use them.
                 # Then, return them in the appropriate output parameters.
                 conn._alpn_select_callback_args = [
-                    _ffi.new("unsigned char *", len(outstr)),
-                    _ffi.new("unsigned char[]", outstr),
+                    _ffi.new("unsigned char *", len(outbytes)),
+                    _ffi.new("unsigned char[]", outbytes),
                 ]
                 outlen[0] = conn._alpn_select_callback_args[0][0]
                 out[0] = conn._alpn_select_callback_args[1]
-                return 0
+                if not any_accepted:
+                    return _lib.SSL_TLSEXT_ERR_NOACK
+                return _lib.SSL_TLSEXT_ERR_OK
             except Exception as e:
                 self._problems.append(e)
-                return 2  # SSL_TLSEXT_ERR_ALERT_FATAL
+                return _lib.SSL_TLSEXT_ERR_ALERT_FATAL
 
         self.callback = _ffi.callback(
             ("int (*)(SSL *, unsigned char **, unsigned char *, "
@@ -1476,8 +1487,12 @@ class Context(object):
 
         :param callback: The callback function.  It will be invoked with two
             arguments: the Connection, and a list of offered protocols as
-            bytestrings, e.g ``[b'http/1.1', b'spdy/2']``.  It should return
-            one of those bytestrings, the chosen protocol.
+            bytestrings, e.g ``[b'http/1.1', b'spdy/2']``.  It can return
+            one of those bytestrings to indicate the chosen protocol, the
+            empty bytestring to terminate the TLS connection, or the
+            :py:obj:`NO_OVERLAPPING_PROTOCOLS` to indicate that no offered
+            protocol was selected, but that the connection should not be
+            aborted.
         """
         self._alpn_select_helper = _ALPNSelectHelper(callback)
         self._alpn_select_callback = self._alpn_select_helper.callback