Use Requests instead of urllib2 for remote HTTP requests

Requests is a much better library for this purpose.

5 files changed, 39 insertions, 31 deletions

diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py
index ab6e71fd..961e06b4 100644
--- a/morphlib/localrepocache_tests.py
+++ b/morphlib/localrepocache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,7 +15,6 @@
 
 
 import unittest
-import urllib2
 import os
 
 import cliapp
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index c5bf9a48..73f9cbed 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -16,16 +16,10 @@
 
 import cliapp
 import logging
-import shutil
 import urllib
-import urllib2
 import urlparse
 
-
-class HeadRequest(urllib2.Request):
-
-    def get_method(self):
-        return 'HEAD'
+import requests
 
 
 class GetError(cliapp.AppException):
@@ -54,12 +48,15 @@ class RemoteArtifactCache(object):
     def _has_file(self, filename):
         url = self._request_url(filename)
         logging.debug('RemoteArtifactCache._has_file: url=%s' % url)
-        request = HeadRequest(url)
-        try:
-            urllib2.urlopen(request)
-            return True
-        except (urllib2.HTTPError, urllib2.URLError):
+
+        response = requests.head(url)
+
+        if response.status_code == 404:
             return False
+        elif response.ok:
+            return True
+        else:
+            response.raise_for_status()
 
     def _request_url(self, filename):
         server_url = self.server_url
@@ -73,13 +70,17 @@ class RemoteArtifactCache(object):
         return self.server_url
 
     def _fetch_file(self, remote_filename, local_file):
+        chunk_size = 1024 * 100
+
         remote_url = self._request_url(remote_filename)
         logging.debug('RemoteArtifactCache._fetch_file: url=%s' % remote_url)
 
         try:
-            remote_file = urllib2.urlopen(remote_url)
-            shutil.copyfileobj(remote_file, local_file)
-        except (urllib2.HTTPError, urllib2.URLError) as e:
+            response = requests.get(remote_url, stream=True)
+            response.raise_for_status()
+            for chunk in response.iter_content(chunk_size):
+                local_file.write(chunk)
+        except requests.exceptions.HTTPError as e:
             logging.debug(str(e))
             raise GetError(self, remote_filename, e)
 
diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py
index f6d19ae7..fc9b6b27 100644
--- a/morphlib/remoteartifactcache_tests.py
+++ b/morphlib/remoteartifactcache_tests.py
@@ -68,7 +68,7 @@ def artifact_cache_server(valid_filenames):
             pass
 
         def artifacts(self, query, send_body=True):
-            '''Return a cached artifact, or 404.'''
+            '''Return a cached artifact, or a 404 (not found) error.'''
             filename = urlparse.parse_qs(query)['filename'][0]
 
             if filename in valid_filenames:
@@ -142,9 +142,9 @@ class RemoteArtifactCacheTests(unittest.TestCase):
                 rac.has_source_metadata(chunk, chunk.cache_key, 'meta'))
 
             lac = morphlib.testutils.FakeLocalArtifactCache()
-            self.assertRaises(
-                morphlib.remoteartifactcache.GetError, rac.get_artifacts,
-                chunk.artifacts.values(), lac, self.fake_status_cb)
+            with self.assertRaises(morphlib.remoteartifactcache.GetError):
+                rac.get_artifacts(
+                    chunk.artifacts.values(), lac, self.fake_status_cb)
 
     def test_escapes_pluses_in_request_urls(self):
         rac = morphlib.remoteartifactcache.RemoteArtifactCache(
diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py
index 004ba86e..71115f0d 100644
--- a/morphlib/remoterepocache.py
+++ b/morphlib/remoterepocache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,10 +17,11 @@
 import cliapp
 import json
 import logging
-import urllib2
 import urlparse
 import urllib
 
+import requests
+
 
 class ResolveRefError(cliapp.AppException):
 
@@ -63,9 +64,9 @@ class RemoteRepoCache(object):
         repo_url = self._resolver.pull_url(repo_name)
         try:
             return self._cat_file_for_repo_url(repo_url, ref, filename)
-        except urllib2.HTTPError as e:
+        except requests.exceptions.HTTPError as e:
             logging.error('Caught exception: %s' % str(e))
-            if e.code == 404:
+            if e.response.status_code == 404:
                 raise CatFileError(repo_name, ref, filename)
             raise # pragma: no cover
 
@@ -102,5 +103,10 @@ class RemoteRepoCache(object):
         if not server_url.endswith('/'):
             server_url += '/'
         url = urlparse.urljoin(server_url, '/1.0/%s' % path)
-        handle = urllib2.urlopen(url)
-        return handle.read()
+
+        response = requests.get(url)
+
+        if response.ok:
+            return response.text
+        else:
+            response.raise_for_status()
diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py
index ef81506f..d4f7d494 100644
--- a/morphlib/remoterepocache_tests.py
+++ b/morphlib/remoterepocache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,7 +16,8 @@
 
 import json
 import unittest
-import urllib2
+
+import requests
 
 import morphlib
 
@@ -30,8 +31,9 @@ class RemoteRepoCacheTests(unittest.TestCase):
         try:
             return self.files[repo_url][sha1][filename]
         except KeyError:
-            raise urllib2.HTTPError(url='', code=404, msg='Not found',
-                                    hdrs={}, fp=None)
+            response = requests.Response()
+            response.status_code = 404
+            raise requests.exceptions.HTTPError(response=response)
 
     def _ls_tree_for_repo_url(self, repo_url, sha1):
         return json.dumps({