[project @ fetchers: set Range header to 0-1MB, urllib2 fetcher only reads 1MB]

1 files changed, 12 insertions, 1 deletions

diff --git a/openid/fetchers.py b/openid/fetchers.py
index 354c907..50e0126 100644
--- a/openid/fetchers.py
+++ b/openid/fetchers.py
@@ -30,6 +30,7 @@ except ImportError:
     pycurl = None
 
 USER_AGENT = "python-openid/%s (%s)" % (openid.__version__, sys.platform)
+MAX_RESPONSE_KB = 1024
 
 def fetch(url, body=None, headers=None):
     """Invoke the fetch method on the default fetcher. Most users
@@ -197,6 +198,9 @@ class Urllib2Fetcher(HTTPFetcher):
         headers.setdefault(
             'User-Agent',
             "%s Python-urllib/%s" % (USER_AGENT, urllib2.__version__,))
+        headers.setdefault(
+            'Range',
+            '0-%s' % (1024*MAX_RESPONSE_KB,))
 
         req = urllib2.Request(url, data=body, headers=headers)
         try:
@@ -213,7 +217,7 @@ class Urllib2Fetcher(HTTPFetcher):
 
     def _makeResponse(self, urllib2_response):
         resp = HTTPResponse()
-        resp.body = urllib2_response.read()
+        resp.body = urllib2_response.read(MAX_RESPONSE_KB * 1024)
         resp.final_url = urllib2_response.geturl()
         resp.headers = dict(urllib2_response.info().items())
 
@@ -314,6 +318,7 @@ class CurlHTTPFetcher(HTTPFetcher):
                 c.setopt(pycurl.HEADERFUNCTION, response_header_data.write)
                 c.setopt(pycurl.TIMEOUT, off)
                 c.setopt(pycurl.URL, openid.urinorm.urinorm(url))
+                c.setopt(pycurl.RANGE, '0-%s'%(MAX_RESPONSE_KB*1024))
 
                 c.perform()
 
@@ -381,6 +386,12 @@ class HTTPLib2Fetcher(HTTPFetcher):
         else:
             method = 'GET'
 
+        if headers is None:
+            headers = {}
+        headers.setdefault(
+            'Range',
+            '0-%s' % (1024*MAX_RESPONSE_KB,))
+
         # httplib2 doesn't check to make sure that the URL's scheme is
         # 'http' so we do it here.
         if not (url.startswith('http://') or url.startswith('https://')):