From 5822fc112a39f2620ffbb28c1341802bb6cb1536 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Fri, 3 Jun 2016 20:16:06 -0400 Subject: Clean up urlopen doc string. Clarifies what is returned when and that the methods are common between the two. Patch by Alexander Liu as part of #22797. --- Lib/urllib/request.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Lib/urllib/request.py') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 4a3daec5d0..333c3f245c 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -173,12 +173,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, The *cadefault* parameter is ignored. For http and https urls, this function returns a http.client.HTTPResponse - object which has the following HTTPResponse Objects methods. - - For ftp, file, and data urls and requests explicitly handled by legacy - URLopener and FancyURLopener classes, this function returns a - urllib.response.addinfourl object which can work as context manager and has - methods such as: + object which has the following HTTPResponse Objects methods: * geturl() - return the URL of the resource retrieved, commonly used to determine if a redirect was followed @@ -190,6 +185,11 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, * getcode() - return the HTTP status code of the response. Raises URLError on errors. + For ftp, file, and data urls and requests explicitly handled by legacy + URLopener and FancyURLopener classes, this function returns a + urllib.response.addinfourl object which can work as context manager and + also support the geturl(), info(), getcode() methods listed above. + Note that *None& may be returned if no handler handles the request (though the default installed global OpenerDirector uses UnknownHandler to ensure this never happens). -- cgit v1.2.1 From 2d9f46b35d978f5a5540cae40471e4d460d7d982 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 4 Jun 2016 05:06:25 +0000 Subject: More typo fixes for 3.6 --- Lib/urllib/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib/urllib/request.py') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 333c3f245c..b16b642458 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -187,7 +187,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, For ftp, file, and data urls and requests explicitly handled by legacy URLopener and FancyURLopener classes, this function returns a - urllib.response.addinfourl object which can work as context manager and + urllib.response.addinfourl object which can work as a context manager and also support the geturl(), info(), getcode() methods listed above. Note that *None& may be returned if no handler handles the request (though -- cgit v1.2.1 From ed84ab3214da74d437d7fa0211173bbd046560b0 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 4 Jun 2016 05:06:34 +0000 Subject: Issue #22797: Synchronize urlopen() doc string with RST documentation --- Lib/urllib/request.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'Lib/urllib/request.py') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b16b642458..67e73f9ef8 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -172,8 +172,8 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, The *cadefault* parameter is ignored. - For http and https urls, this function returns a http.client.HTTPResponse - object which has the following HTTPResponse Objects methods: + This function always returns an object which can work as a context + manager and has methods such as * geturl() - return the URL of the resource retrieved, commonly used to determine if a redirect was followed @@ -185,12 +185,17 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, * getcode() - return the HTTP status code of the response. Raises URLError on errors. - For ftp, file, and data urls and requests explicitly handled by legacy + For HTTP and HTTPS URLs, this function returns a http.client.HTTPResponse + object slightly modified. In addition to the three new methods above, the + msg attribute contains the same information as the reason attribute --- + the reason phrase returned by the server --- instead of the response + headers as it is specified in the documentation for HTTPResponse. + + For FTP, file, and data URLs and requests explicitly handled by legacy URLopener and FancyURLopener classes, this function returns a - urllib.response.addinfourl object which can work as a context manager and - also support the geturl(), info(), getcode() methods listed above. + urllib.response.addinfourl object. - Note that *None& may be returned if no handler handles the request (though + Note that None may be returned if no handler handles the request (though the default installed global OpenerDirector uses UnknownHandler to ensure this never happens). -- cgit v1.2.1 From 1d17079876f2e0b64a048eee2c1d11f1b4f7815e Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 24 Aug 2016 06:33:33 +0000 Subject: Issue #12319: Support for chunked encoding of HTTP request bodies When the body object is a file, its size is no longer determined with fstat(), since that can report the wrong result (e.g. reading from a pipe). Instead, determine the size using seek(), or fall back to chunked encoding for unseekable files. Also, change the logic for detecting text files to check for TextIOBase inheritance, rather than inspecting the ?mode? attribute, which may not exist (e.g. BytesIO and StringIO). The Content-Length for text files is no longer determined ahead of time, because the original logic could have been wrong depending on the codec and newline translation settings. Patch by Demian Brecht and Rolf Krahl, with a few tweaks by me. --- Lib/urllib/request.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) (limited to 'Lib/urllib/request.py') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index dc436bc73f..30bf6e051e 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -141,17 +141,9 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *, cafile=None, capath=None, cadefault=False, context=None): '''Open the URL url, which can be either a string or a Request object. - *data* must be a bytes object specifying additional data to be sent to the - server, or None if no such data is needed. data may also be an iterable - object and in that case Content-Length value must be specified in the - headers. Currently HTTP requests are the only ones that use data; the HTTP - request will be a POST instead of a GET when the data parameter is - provided. - - *data* should be a buffer in the standard application/x-www-form-urlencoded - format. The urllib.parse.urlencode() function takes a mapping or sequence - of 2-tuples and returns an ASCII text string in this format. It should be - encoded to bytes before being used as the data parameter. + *data* must be an object specifying additional data to be sent to + the server, or None if no such data is needed. See Request for + details. urllib.request module uses HTTP/1.1 and includes a "Connection:close" header in its HTTP requests. @@ -1235,6 +1227,11 @@ class AbstractHTTPHandler(BaseHandler): def set_http_debuglevel(self, level): self._debuglevel = level + def _get_content_length(self, request): + return http.client.HTTPConnection._get_content_length( + request.data, + request.get_method()) + def do_request_(self, request): host = request.host if not host: @@ -1243,24 +1240,22 @@ class AbstractHTTPHandler(BaseHandler): if request.data is not None: # POST data = request.data if isinstance(data, str): - msg = "POST data should be bytes or an iterable of bytes. " \ - "It cannot be of type str." + msg = "POST data should be bytes, an iterable of bytes, " \ + "or a file object. It cannot be of type str." raise TypeError(msg) if not request.has_header('Content-type'): request.add_unredirected_header( 'Content-type', 'application/x-www-form-urlencoded') - if not request.has_header('Content-length'): - try: - mv = memoryview(data) - except TypeError: - if isinstance(data, collections.Iterable): - raise ValueError("Content-Length should be specified " - "for iterable data of type %r %r" % (type(data), - data)) + if (not request.has_header('Content-length') + and not request.has_header('Transfer-encoding')): + content_length = self._get_content_length(request) + if content_length is not None: + request.add_unredirected_header( + 'Content-length', str(content_length)) else: request.add_unredirected_header( - 'Content-length', '%d' % (len(mv) * mv.itemsize)) + 'Transfer-encoding', 'chunked') sel_host = host if request.has_proxy(): @@ -1316,7 +1311,8 @@ class AbstractHTTPHandler(BaseHandler): try: try: - h.request(req.get_method(), req.selector, req.data, headers) + h.request(req.get_method(), req.selector, req.data, headers, + encode_chunked=req.has_header('Transfer-encoding')) except OSError as err: # timeout error raise URLError(err) r = h.getresponse() -- cgit v1.2.1 From a48be2f1f400d1d58196b72dc7c6b4bc1817da1a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Sep 2016 16:44:53 -0700 Subject: Merge --- Lib/urllib/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib/urllib/request.py') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 30bf6e051e..cc4f0bf089 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1686,7 +1686,7 @@ class URLopener: self.proxies = proxies self.key_file = x509.get('key_file') self.cert_file = x509.get('cert_file') - self.addheaders = [('User-Agent', self.version)] + self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')] self.__tempfiles = [] self.__unlink = os.unlink # See cleanup() self.tempcache = None -- cgit v1.2.1 From de56c23b9a770f4872dd5a89478b0b334cc7de86 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 10 Sep 2016 23:23:33 +0200 Subject: Issue #28022: Deprecate ssl-related arguments in favor of SSLContext. The deprecation include manual creation of SSLSocket and certfile/keyfile (or similar) in ftplib, httplib, imaplib, smtplib, poplib and urllib. ssl.wrap_socket() is not marked as deprecated yet. --- Lib/urllib/request.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Lib/urllib/request.py') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index cc4f0bf089..5f15b74f4d 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -198,6 +198,9 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ''' global _opener if cafile or capath or cadefault: + import warnings + warnings.warn("cafile, cpath and cadefault are deprecated, use a " + "custom context instead.", DeprecationWarning, 2) if context is not None: raise ValueError( "You can't pass both context and any of cafile, capath, and " -- cgit v1.2.1