Validate SSL certs accessed through urllib*

* Adds another module utility file which generalizes the
  access of urls via the urllib* libraries.
* Adds a new spec generator for common arguments.
* Makes the user-agent string configurable.

Fixes #6211

23 files changed, 600 insertions, 404 deletions

diff --git a/examples/ansible.cfg b/examples/ansible.cfg
index f543b2e4bc..396974bf01 100644
--- a/examples/ansible.cfg
+++ b/examples/ansible.cfg
@@ -103,6 +103,20 @@ filter_plugins     = /usr/share/ansible_plugins/filter_plugins
 # set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1
 #nocolor = 1
 
+# the CA certificate path used for validating SSL certs. This path 
+# should exist on the controlling node, not the target nodes
+# common locations:
+# RHEL/CentOS: /etc/pki/tls/certs/ca-bundle.crt
+# Fedora     : /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+# Ubuntu     : /usr/share/ca-certificates/cacert.org/cacert.org.crt
+#ca_file_path = 
+
+# the http user-agent string to use when fetching urls. Some web server
+# operators block the default urllib user agent as it is frequently used
+# by malicious attacks/scripts, so we set it to something unique to 
+# avoid issues.
+#http_user_agent = ansible-agent
+
 [paramiko_connection]
 
 # uncomment this line to cause the paramiko connection plugin to not record new host
diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py
index 431e6eb742..ed996f3bef 100644
--- a/lib/ansible/constants.py
+++ b/lib/ansible/constants.py
@@ -144,6 +144,10 @@ DEFAULT_VARS_PLUGIN_PATH       = get_config(p, DEFAULTS, 'vars_plugins',       '
 DEFAULT_FILTER_PLUGIN_PATH     = get_config(p, DEFAULTS, 'filter_plugins',     'ANSIBLE_FILTER_PLUGINS', '/usr/share/ansible_plugins/filter_plugins')
 DEFAULT_LOG_PATH               = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))
 
+# URL Arguments for generic module urllib2 use
+DEFAULT_HTTP_USER_AGENT        = get_config(p, DEFAULTS, 'http_user_agent',    'ANSIBLE_HTTP_USER_AGENT', 'ansible-agent')
+DEFAULT_CA_FILE_PATH           = shell_expand_path(get_config(p, DEFAULTS, 'ca_file_path',       'ANSIBLE_CA_FILE_PATH', ''))
+
 ANSIBLE_NOCOLOR                = get_config(p, DEFAULTS, 'nocolor', 'ANSIBLE_NOCOLOR', None, boolean=True)
 ANSIBLE_NOCOWS                 = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None, boolean=True)
 DISPLAY_SKIPPED_HOSTS          = get_config(p, DEFAULTS, 'display_skipped_hosts', 'DISPLAY_SKIPPED_HOSTS', True, boolean=True)
diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py
index c2be621d4b..fd0b2edfc3 100644
--- a/lib/ansible/module_utils/basic.py
+++ b/lib/ansible/module_utils/basic.py
@@ -59,6 +59,7 @@ import grp
 import pwd
 import platform
 import errno
+import tempfile
 
 try:
     import json
@@ -114,6 +115,7 @@ FILE_COMMON_ARGUMENTS=dict(
     remote_src = dict(), # used by assemble
 )
 
+
 def get_platform():
     ''' what's the platform?  example: Linux is a platform. '''
     return platform.system()
@@ -188,7 +190,7 @@ class AnsibleModule(object):
         os.environ['LANG'] = MODULE_LANG
         (self.params, self.args) = self._load_params()
 
-        self._legal_inputs = [ 'CHECKMODE', 'NO_LOG' ]
+        self._legal_inputs = ['CHECKMODE', 'NO_LOG']
         
         self.aliases = self._handle_aliases()
 
@@ -571,8 +573,9 @@ class AnsibleModule(object):
 
     def _check_invalid_arguments(self):
         for (k,v) in self.params.iteritems():
-            if k in ('CHECKMODE', 'NO_LOG'):
-                continue
+            # these should be in legal inputs already
+            #if k in ('CHECKMODE', 'NO_LOG'):
+            #    continue
             if k not in self._legal_inputs:
                 self.fail_json(msg="unsupported parameter for module: %s" % k)
 
@@ -1068,4 +1071,3 @@ class AnsibleModule(object):
                 break
         return '%.2f %s' % (float(size)/ limit, suffix)
 
-
diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py
index 9156df766b..58291c2d5d 100644
--- a/lib/ansible/module_utils/ec2.py
+++ b/lib/ansible/module_utils/ec2.py
@@ -1,3 +1,31 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 try:
     from distutils.version import LooseVersion
     HAS_LOOSE_VERSION = True
diff --git a/lib/ansible/module_utils/known_hosts.py b/lib/ansible/module_utils/known_hosts.py
index 000db9d1e6..36f5b87fff 100644
--- a/lib/ansible/module_utils/known_hosts.py
+++ b/lib/ansible/module_utils/known_hosts.py
@@ -1,3 +1,31 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 def add_git_host_key(module, url, accept_hostkey=True):
 
     """ idempotently add a git url hostkey """
diff --git a/lib/ansible/module_utils/rax.py b/lib/ansible/module_utils/rax.py
index 84e5686d24..98623c7d38 100644
--- a/lib/ansible/module_utils/rax.py
+++ b/lib/ansible/module_utils/rax.py
@@ -1,5 +1,32 @@
-import os
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import os
 
 def rax_argument_spec():
     return dict(
diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py
new file mode 100644
index 0000000000..f251c6b407
--- /dev/null
+++ b/lib/ansible/module_utils/urls.py
@@ -0,0 +1,262 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+try:
+    import urllib
+    HAS_URLLIB = True
+except:
+    HAS_URLLIB = False
+
+try:
+    import urllib2
+    HAS_URLLIB2 = True
+except:
+    HAS_URLLIB2 = False
+
+try:
+    import urlparse
+    HAS_URLPARSE = True
+except:
+    HAS_URLPARSE = False
+
+try:
+    import ssl
+    HAS_SSL=True
+except:
+    HAS_SSL=False
+
+
+class RequestWithMethod(urllib2.Request):
+    '''
+    Workaround for using DELETE/PUT/etc with urllib2
+    Originally contained in library/net_infrastructure/dnsmadeeasy
+    '''
+
+    def __init__(self, url, method, data=None, headers={}):
+        self._method = method
+        urllib2.Request.__init__(self, url, data, headers)
+
+    def get_method(self):
+        if self._method:
+            return self._method
+        else:
+            return urllib2.Request.get_method(self)
+
+
+class SSLValidationHandler(urllib2.BaseHandler):
+    '''
+    A custom handler class for SSL validation.
+
+    Based on:
+    http://stackoverflow.com/questions/1087227/validate-ssl-certificates-with-python
+    http://techknack.net/python-urllib2-handlers/
+    '''
+
+    def __init__(self, module, hostname, port, ca_cert=None):
+        self.module = module
+        self.hostname = hostname
+        self.port = port
+        self.ca_cert = ca_cert
+
+    def get_ca_cert(self):
+        # tries to find a valid CA cert in one of the
+        # standard locations for the current distribution
+
+        if self.ca_cert and os.path.exists(self.ca_cert):
+            # the user provided a custom CA cert (ie. one they
+            # uploaded themselves), so use it
+            return self.ca_cert
+
+        ca_cert = None
+        platform = get_platform()
+        distribution = get_distribution()
+        if platform == 'Linux':
+            if distribution in ('Fedora',):
+                ca_cert = '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'
+            elif distribution in ('RHEL','CentOS','ScientificLinux'):
+                ca_cert = '/etc/pki/tls/certs/ca-bundle.crt'
+            elif distribution in ('Ubuntu','Debian'):
+                ca_cert = '/usr/share/ca-certificates/cacert.org/cacert.org.crt'
+        elif platform == 'FreeBSD':
+            ca_cert = '/usr/local/share/certs/ca-root.crt'
+        elif platform == 'OpenBSD':
+            ca_cert = '/etc/ssl/cert.pem'
+        elif platform == 'NetBSD':
+            ca_cert = '/etc/openssl/certs/ca-cert.pem'
+        elif platform == 'SunOS':
+            # FIXME?
+            pass
+        elif platform == 'AIX':
+            # FIXME?
+            pass
+
+        if ca_cert and os.path.exists(ca_cert):
+            return ca_cert
+        elif os.path.exists('/etc/ansible/ca-cert.pem'):
+            # fall back to a user-deployed cert in a standard
+            # location if the OS platform one is not available
+            return '/etc/ansible/ca-cert.pem'
+        else:
+            # CA cert isn't available, no validation
+            return None
+
+    def http_request(self, req):
+        try:
+            server_cert = ssl.get_server_certificate((self.hostname, self.port), ca_certs=self.get_ca_cert())
+        except ssl.SSLError:
+            self.module.fail_json(msg='failed to validate the SSL certificate for %s:%s. You can use validate_certs=no, however this is unsafe and not recommended' % (self.hostname, self.port))
+        return req
+
+    https_request = http_request
+
+
+def url_argument_spec():
+    '''
+    Creates an argument spec that can be used with any module
+    that will be requesting content via urllib/urllib2
+    '''
+    return dict(
+        url = dict(),
+        force = dict(default='no', aliases=['thirsty'], type='bool'),
+        http_agent = dict(default='ansible-httpget'),
+        use_proxy = dict(default='yes', type='bool'),
+        validate_certs = dict(default='yes', type='bool'),
+    )
+
+
+def fetch_url(module, url, data=None, headers=None, method=None, 
+              use_proxy=False, validate_certs=True, force=False, last_mod_time=None, timeout=10):
+    '''
+    Fetches a file from an HTTP/FTP server using urllib2
+    '''
+
+    if not HAS_URLLIB:
+        module.fail_json(msg='urllib is not installed')
+    if not HAS_URLLIB2:
+        module.fail_json(msg='urllib2 is not installed')
+    elif not HAS_URLPARSE:
+        module.fail_json(msg='urlparse is not installed')
+
+    r = None
+    handlers = []
+    info = dict(url=url)
+
+    parsed = urlparse.urlparse(url)
+    if parsed[0] == 'https':
+        if not HAS_SSL and validate_certs:
+            module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended')
+        elif validate_certs:
+            # do the cert validation
+            netloc = parsed[1]
+            if '@' in netloc:
+                netloc = netloc.split('@', 1)[1]
+            if ':' in netloc:
+                hostname, port = netloc.split(':', 1)
+            else:
+                hostname = netloc
+                port = 443
+            # create the SSL validation handler and
+            # add it to the list of handlers
+            ssl_handler = SSLValidationHandler(module, hostname, port)
+            handlers.append(ssl_handler)
+
+    if '@' in parsed[1]:
+        credentials, netloc = parsed[1].split('@', 1)
+        if ':' in credentials:
+            username, password = credentials.split(':', 1)
+        else:
+            username = credentials
+            password = ''
+        parsed = list(parsed)
+        parsed[1] = netloc
+
+        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+        # this creates a password manager
+        passman.add_password(None, netloc, username, password)
+        # because we have put None at the start it will always
+        # use this username/password combination for  urls
+        # for which `theurl` is a super-url
+
+        authhandler = urllib2.HTTPBasicAuthHandler(passman)
+        # create the AuthHandler
+        handlers.append(authhandler)
+
+        #reconstruct url without credentials
+        url = urlparse.urlunparse(parsed)
+
+    if not use_proxy:
+        proxyhandler = urllib2.ProxyHandler({})
+        handlers.append(proxyhandler)
+
+    opener = urllib2.build_opener(*handlers)
+    urllib2.install_opener(opener)
+
+    if method:
+        if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'):
+            module.fail_json(msg='invalid HTTP request method; %s' % method.upper())
+        request = RequestWithMethod(url, method.upper(), data)
+    else:
+        request = urllib2.Request(url, data)
+
+    # add the custom agent header, to help prevent issues 
+    # with sites that block the default urllib agent string 
+    request.add_header('User-agent', module.params.get('http_agent'))
+
+    # if we're ok with getting a 304, set the timestamp in the 
+    # header, otherwise make sure we don't get a cached copy
+    if last_mod_time and not force:
+        tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
+        request.add_header('If-Modified-Since', tstamp)
+    else:
+        request.add_header('cache-control', 'no-cache')
+
+    # user defined headers now, which may override things we've set above
+    if headers:
+        if not isinstance(headers, dict):
+            module.fail_json("headers provided to fetch_url() must be a dict")
+        for header in headers:
+            request.add_header(header, headers[header])
+
+    try:
+        if sys.version_info < (2,6,0):
+            # urlopen in python prior to 2.6.0 did not
+            # have a timeout parameter
+            r = urllib2.urlopen(request, None)
+        else:
+            r = urllib2.urlopen(request, None, timeout)
+        info.update(r.info())
+        info['url'] = r.geturl()  # The URL goes in too, because of redirects.
+        info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
+    except urllib2.HTTPError, e:
+        info.update(dict(msg=str(e), status=e.code))
+    except urllib2.URLError, e:
+        code = int(getattr(e, 'code', -1))
+        info.update(dict(msg="Request failed: %s" % str(e), status=code))
+
+    return r, info
+
diff --git a/library/cloud/ec2_facts b/library/cloud/ec2_facts
index 1c17fa5b71..c6a6670a58 100644
--- a/library/cloud/ec2_facts
+++ b/library/cloud/ec2_facts
@@ -41,7 +41,6 @@ EXAMPLES = '''
   when: ansible_ec2_instance_type == "t1.micro"
 '''
    
-import urllib2
 import socket
 import re
 
@@ -62,7 +61,8 @@ class Ec2Metadata(object):
                    'us-west-1',
                    'us-west-2')
 
-    def __init__(self, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None):
+    def __init__(self, module, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None):
+        self.module   = module
         self.uri_meta = ec2_metadata_uri or self.ec2_metadata_uri
         self.uri_user = ec2_userdata_uri or self.ec2_userdata_uri
         self.uri_ssh  =  ec2_sshdata_uri or self.ec2_sshdata_uri
@@ -70,12 +70,9 @@ class Ec2Metadata(object):
         self._prefix   = 'ansible_ec2_%s'
 
     def _fetch(self, url):
-        try:
-            return urllib2.urlopen(url).read()
-        except urllib2.HTTPError:
-            return
-        except urllib2.URLError:
-            return
+        self.module.fail_json(msg="url is %s" % url)
+        (response, info) = fetch_url(self.module, url, force=True)
+        return response.read()
 
     def _mangle_fields(self, fields, uri, filter_patterns=['public-keys-0']):
         new_fields = {}
@@ -150,17 +147,20 @@ class Ec2Metadata(object):
         return data
 
 def main():
-
-    ec2_facts = Ec2Metadata().run()
-    ec2_facts_result = dict(changed=False, ansible_facts=ec2_facts)
+    argument_spec = url_argument_spec()
 
     module = AnsibleModule(
-        argument_spec = dict(),
+        argument_spec = argument_spec,
         supports_check_mode = True,
     )
+
+    ec2_facts = Ec2Metadata(module).run()
+    ec2_facts_result = dict(changed=False, ansible_facts=ec2_facts)
+
     module.exit_json(**ec2_facts_result)
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 
 main()
diff --git a/library/database/riak b/library/database/riak
index 53faba6e98..e0a7552f0a 100644
--- a/library/database/riak
+++ b/library/database/riak
@@ -138,24 +138,13 @@ def main():
     while True:
         if time.time() > timeout:
             module.fail_json(msg='Timeout, could not fetch Riak stats.')
-        try:
-            if sys.version_info<(2,6,0):
-                stats_raw = urllib2.urlopen(
-                    'http://%s/stats' % (http_conn), None).read()
-            else:
-                stats_raw = urllib2.urlopen(
-                    'http://%s/stats' % (http_conn), None, 5).read()
+        (response, info) = fetch_url(module, 'http://%s/stats' % (http_conn), force=True, timeout=5)
+        if info['status'] == 200:
+            stats_raw = response.read()
             break
-        except urllib2.HTTPError, e:
-            time.sleep(5)
-        except urllib2.URLError, e:
-            time.sleep(5)
-        except socket.timeout:
-            time.sleep(5)
-        except Exception, e:
-            module.fail_json(msg='Could not fetch Riak stats: %s' % e)
+        time.sleep(5)
 
-# here we attempt to load those stats,
+    # here we attempt to load those stats,
     try:
         stats = json.loads(stats_raw)
     except:
diff --git a/library/monitoring/airbrake_deployment b/library/monitoring/airbrake_deployment
index 8a4a834be7..6a83459906 100644
--- a/library/monitoring/airbrake_deployment
+++ b/library/monitoring/airbrake_deployment
@@ -52,6 +52,13 @@ options:
       - Optional URL to submit the notification to. Use to send notifications to Airbrake-compliant tools like Errbit.
     required: false
     default: https://airbrake.io/deploys
+  validate_certs:
+    description:
+      - If C(no), SSL certificates for the target url will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
 
 # informational: requirements for nodes
 requirements: [ urllib, urllib2 ]
@@ -64,29 +71,12 @@ EXAMPLES = '''
                        revision=4.2
 '''
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
 # ===========================================
 # Module execution.
 #
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -95,6 +85,7 @@ def main():
             repo=dict(required=False),
             revision=dict(required=False),
             url=dict(required=False, default='https://api.airbrake.io/deploys.txt')
+            validate_certs=dict(default='yes', type='bool'),
         ),
         supports_check_mode=True
     )
@@ -123,18 +114,16 @@ def main():
         module.exit_json(changed=True)
 
     # Send the data to airbrake
-    try:
-        req = urllib2.Request(url, urllib.urlencode(params))
-        result=urllib2.urlopen(req)
-    except Exception, e:
-        module.fail_json(msg="unable to update airbrake via %s?%s : %s" % (url, urllib.urlencode(params), e))
+    data = urllib.urlencode(params)
+    response, info = fetch_url(module, url, data=data, validate_certs=module.params['validate_certs'])
+    if info['status'] == 200:
+        module.exit_json(changed=True)
     else:
-        if result.code == 200:
-            module.exit_json(changed=True)
-        else:
-            module.fail_json(msg="HTTP result code: %d connecting to %s" % (result.code, url))
+        module.fail_json(msg="HTTP result code: %d connecting to %s" % (info['status'], url))
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
 
diff --git a/library/monitoring/boundary_meter b/library/monitoring/boundary_meter
index 202dfd03ae..3c9f90a4ce 100644
--- a/library/monitoring/boundary_meter
+++ b/library/monitoring/boundary_meter
@@ -24,7 +24,6 @@ along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
 import datetime
-import urllib2
 import base64
 import os
 
@@ -74,12 +73,6 @@ EXAMPLES='''
 
 '''
 
-try:
-    import urllib2
-    HAS_URLLIB2 = True
-except ImportError:
-    HAS_URLLIB2 = False
-
 api_host = "api.boundary.com"
 config_directory = "/etc/bprobe"
 
@@ -101,7 +94,7 @@ def build_url(name, apiid, action, meter_id=None, cert_type=None):
     elif action == "delete":
         return  "https://%s/%s/meters/%s" % (api_host, apiid, meter_id)
 
-def http_request(name, apiid, apikey, action, meter_id=None, cert_type=None):
+def http_request(module, name, apiid, apikey, action, data=None, meter_id=None, cert_type=None):
     
     if meter_id is None:
         url = build_url(name, apiid, action)
@@ -111,11 +104,11 @@ def http_request(name, apiid, apikey, action, meter_id=None, cert_type=None):
         else:
             url = build_url(name, apiid, action, meter_id, cert_type)
 
-    auth = auth_encode(apikey)
-    request = urllib2.Request(url)
-    request.add_header("Authorization", "Basic %s" % (auth)) 
-    request.add_header("Content-Type", "application/json")
-    return request
+    headers = dict()
+    headers["Authorization"] = "Basic %s" % auth_encode(apikey)
+    headers["Content-Type"] = "application/json"
+
+    return fetch_url(module, url, data=data, headers=headers)
 
 def create_meter(module, name, apiid, apikey):
 
@@ -126,14 +119,10 @@ def create_meter(module, name, apiid, apikey):
         module.exit_json(status="Meter " + name + " already exists",changed=False)
     else:
         # If it doesn't exist, create it
-        request = http_request(name, apiid, apikey, action="create")
-        # A create request seems to need a json body with the name of the meter in it
         body = '{"name":"' + name + '"}'
-        request.add_data(body)
+        response, info = http_request(module, name, apiid, apikey, data=body, action="create")
 
-        try: 
-            result = urllib2.urlopen(request)
-        except urllib2.URLError, e:
+        if info['status'] != 200:
             module.fail_json(msg="Failed to connect to api host to create meter")
 
         # If the config directory doesn't exist, create it
@@ -160,15 +149,13 @@ def create_meter(module, name, apiid, apikey):
 
 def search_meter(module, name, apiid, apikey):
 
-    request = http_request(name, apiid, apikey, action="search")
+    response, info = http_request(module, name, apiid, apikey, action="search")
 
-    try: 
-        result = urllib2.urlopen(request)
-    except urllib2.URLError, e:
+    if info['status'] != 200:
         module.fail_json("Failed to connect to api host to search for meter")
 
     # Return meters
-    return json.loads(result.read())
+    return json.loads(response.read())
 
 def get_meter_id(module, name, apiid, apikey):
     # In order to delete the meter we need its id
@@ -186,16 +173,9 @@ def delete_meter(module, name, apiid, apikey):
     if meter_id is None:
         return 1, "Meter does not exist, so can't delete it"
     else:
-        action = "delete"
-        request = http_request(name, apiid, apikey, action, meter_id)
-        # See http://stackoverflow.com/questions/4511598/how-to-make-http-delete-method-using-urllib2
-        # urllib2 only does GET or POST I believe, but here we need delete
-        request.get_method = lambda: 'DELETE'
-
-        try: 
-            result = urllib2.urlopen(request)
-        except urllib2.URLError, e:
-            module.fail_json("Failed to connect to api host to delete meter")
+        response, info = http_request(module, name, apiid, apikey, action, meter_id)
+        if info['status'] != 200:
+            module.fail_json("Failed to delete meter")
 
         # Each new meter gets a new key.pem and ca.pem file, so they should be deleted
         types = ['cert', 'key']
@@ -214,17 +194,14 @@ def download_request(module, name, apiid, apikey, cert_type):
 
     if meter_id is not None:
         action = "certificates"
-        request = http_request(name, apiid, apikey, action, meter_id, cert_type)
- 
-        try:
-            result = urllib2.urlopen(request)
-        except urllib2.URLError, e:
+        response, info = http_request(module, name, apiid, apikey, action, meter_id, cert_type)
+        if info['status'] != 200:
             module.fail_json("Failed to connect to api host to download certificate")
 
         if result:
             try:
                 cert_file_path = '%s/%s.pem' % (config_directory,cert_type)
-                body = result.read()
+                body = response.read()
                 cert_file = open(cert_file_path, 'w')
                 cert_file.write(body)
                 cert_file.close
@@ -238,9 +215,6 @@ def download_request(module, name, apiid, apikey, cert_type):
 
 def main():
 
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
         state=dict(required=True, choices=['present', 'absent']),
@@ -268,5 +242,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()
 
diff --git a/library/monitoring/datadog_event b/library/monitoring/datadog_event
index 629e86e98a..878aee6d34 100644
--- a/library/monitoring/datadog_event
+++ b/library/monitoring/datadog_event
@@ -67,7 +67,6 @@ datadog_event: title="Testing from ansible" text="Test!"
 '''
 
 import socket
-from urllib2 import urlopen, Request, URLError
 
 def main():
     module = AnsibleModule(
@@ -97,8 +96,7 @@ def main():
     post_event(module)
 
 def post_event(module):
-    uri = "https://app.datadoghq.com/api/v1/events?api_key=" + \
-          module.params['api_key']
+    uri = "https://app.datadoghq.com/api/v1/events?api_key=%s" % module.params['api_key']
 
     body = dict(
         title=module.params['title'],
@@ -117,22 +115,20 @@ def post_event(module):
 
     json_body = module.jsonify(body)
     headers = {"Content-Type": "application/json"}
-    request = Request(uri, json_body, headers, unverifiable=True)
 
-    try:
-        response = urlopen(request)
+    (response, info) = fetch_url(module, uri, data=json_body, headers=headers)
+    if info['status'] == 200:
         response_body = response.read()
         response_json = module.from_json(response_body)
         if response_json['status'] == 'ok':
             module.exit_json(changed=True)
         else:
             module.fail_json(msg=response)
-
-    except URLError, e:
-        module.fail_json(msg="URL error: %s." % e)
-    except socket.error, e:
-        module.fail_json(msg="Socket error: %s to %s" % (e, uri))
+    else:
+        module.fail_json(**info)
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
diff --git a/library/monitoring/newrelic_deployment b/library/monitoring/newrelic_deployment
index de64651969..08132722e1 100644
--- a/library/monitoring/newrelic_deployment
+++ b/library/monitoring/newrelic_deployment
@@ -75,29 +75,12 @@ EXAMPLES = '''
                        revision=1.0
 '''
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
 # ===========================================
 # Module execution.
 #
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -134,29 +117,20 @@ def main():
         module.exit_json(changed=True)
 
     # Send the data to NewRelic
-    try:
-        req = urllib2.Request("https://rpm.newrelic.com/deployments.xml", urllib.urlencode(params))
-        req.add_header('x-api-key',module.params["token"])
-        result=urllib2.urlopen(req)
-        # urlopen behaves differently in python 2.4 and 2.6 so we handle
-        # both cases here.  In python 2.4 it throws an exception if the
-        # return code is anything other than a 200.  In python 2.6 it
-        # doesn't throw an exception for any 2xx return codes.  In both
-        # cases we expect newrelic should return a 201 on success. So 
-        # to handle both cases, both the except & else cases below are
-        # effectively identical.
-    except Exception, e:
-        if e.code == 201:
-            module.exit_json(changed=True)
-        else:
-            module.fail_json(msg="unable to update newrelic: %s" % e)
+    url = "https://rpm.newrelic.com/deployments.xml"
+    data = urllib.urlencode(params)
+    headers = {
+        'x-api-key': module.params["token"],
+    }
+    response, info = fetch_url(module, url, data=data, headers=headers)
+    if info['status'] in (200, 201):
+        module.exit_json(changed=True)
     else:
-        if result.code == 201:
-            module.exit_json(changed=True)
-        else:
-            module.fail_json(msg="result code: %d" % result.code)
+        module.fail_json(msg="unable to update newrelic: %s" % info['msg'])
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
 
diff --git a/library/monitoring/pagerduty b/library/monitoring/pagerduty
index bfd0573f4d..9a7f21d077 100644
--- a/library/monitoring/pagerduty
+++ b/library/monitoring/pagerduty
@@ -87,24 +87,23 @@ EXAMPLES='''
 
 import json
 import datetime
-import urllib2
 import base64
 
 
-def ongoing(name, user, passwd):
+def ongoing(module, name, user, passwd):
 
     url = "https://" + name + ".pagerduty.com/api/v1/maintenance_windows/ongoing"
     auth = base64.encodestring('%s:%s' % (user, passwd)).replace('\n', '')
+    headers = {"Authorization": "Basic %s" % auth}
 
-    req = urllib2.Request(url)
-    req.add_header("Authorization", "Basic %s" % auth)
-    res = urllib2.urlopen(req)
-    out = res.read()
+    response, info = fetch_url(module, url, headers=headers)
+    if info['status'] != 200:
+        module.fail_json(msg="failed to lookup the ongoing window: %s" % info['msg'])
 
-    return False, out
+    return False, response.read()
 
 
-def create(name, user, passwd, service, hours, desc):
+def create(module, name, user, passwd, service, hours, desc):
 
     now = datetime.datetime.utcnow()
     later = now + datetime.timedelta(hours=int(hours))
@@ -113,15 +112,17 @@ def create(name, user, passwd, service, hours, desc):
 
     url = "https://" + name + ".pagerduty.com/api/v1/maintenance_windows"
     auth = base64.encodestring('%s:%s' % (user, passwd)).replace('\n', '')
+    headers = {
+        'Authorization': 'Basic %s' % auth,
+        'Content-Type' : 'application/json',
+    }
     data = json.dumps({'maintenance_window': {'start_time': start, 'end_time': end, 'description': desc, 'service_ids': [service]}})
 
-    req = urllib2.Request(url, data)
-    req.add_header("Authorization", "Basic %s" % auth)
-    req.add_header('Content-Type', 'application/json')
-    res = urllib2.urlopen(req)
-    out = res.read()
+    response, info = fetch_url(module, url, data=data, headers=headers, method='POST')
+    if info['status'] != 200:
+        module.fail_json(msg="failed to create the window: %s" % info['msg'])
 
-    return False, out
+    return False, response.read()
 
 
 def main():
@@ -149,10 +150,10 @@ def main():
     if state == "running" or state == "started":
         if not service:
             module.fail_json(msg="service not specified")
-        (rc, out) = create(name, user, passwd, service, hours, desc)
+        (rc, out) = create(module, name, user, passwd, service, hours, desc)
 
     if state == "ongoing":
-        (rc, out) = ongoing(name, user, passwd)
+        (rc, out) = ongoing(module, name, user, passwd)
 
     if rc != 0:
         module.fail_json(msg="failed", result=out)
@@ -161,4 +162,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
diff --git a/library/net_infrastructure/dnsmadeeasy b/library/net_infrastructure/dnsmadeeasy
index d4af13e884..9e2c14480e 100644
--- a/library/net_infrastructure/dnsmadeeasy
+++ b/library/net_infrastructure/dnsmadeeasy
@@ -106,8 +106,6 @@ EXAMPLES = '''
 
 IMPORT_ERROR = None
 try:
-    import urllib
-    import urllib2
     import json
     from time import strftime, gmtime
     import hashlib
@@ -115,22 +113,6 @@ try:
 except ImportError, e:
     IMPORT_ERROR = str(e)
 
-
-class RequestWithMethod(urllib2.Request):
-
-    """Workaround for using DELETE/PUT/etc with urllib2"""
-
-    def __init__(self, url, method, data=None, headers={}):
-        self._method = method
-        urllib2.Request.__init__(self, url, data, headers)
-
-    def get_method(self):
-        if self._method:
-            return self._method
-        else:
-            return urllib2.Request.get_method(self)
-
-
 class DME2:
 
     def __init__(self, apikey, secret, domain, module):
@@ -169,16 +151,10 @@ class DME2:
         url = self.baseurl + resource
         if data and not isinstance(data, basestring):
             data = urllib.urlencode(data)
-        request = RequestWithMethod(url, method, data, self._headers())
 
-        try:
-            response = urllib2.urlopen(request)
-        except urllib2.HTTPError, e:
-            self.module.fail_json(
-                msg="%s returned %s, with body: %s" % (url, e.code, e.read()))
-        except Exception, e:
-            self.module.fail_json(
-                msg="Failed contacting: %s : Exception %s" % (url, e.message()))
+        response, info = fetch_url(self.module, url, data=data, method=method)
+        if info['status'] not in (200, 201, 204):
+            self.module.fail_json(msg="%s returned %s, with body: %s" % (url, info['status'], info['msg']))
 
         try:
             return json.load(response)
@@ -338,4 +314,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
diff --git a/library/net_infrastructure/netscaler b/library/net_infrastructure/netscaler
index 1aa370895d..4756d90abd 100644
--- a/library/net_infrastructure/netscaler
+++ b/library/net_infrastructure/netscaler
@@ -73,6 +73,14 @@ options:
     default: server
     choices: ["server", "service"]
     aliases: []
+  validate_certs:
+    description:
+      - If C(no), SSL certificates for the target url will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
+
 requirements: [ "urllib", "urllib2" ]
 author: Nandor Sivok
 '''
@@ -90,8 +98,6 @@ ansible host -m netscaler -a "nsc_host=nsc.example.com user=apiuser password=api
 
 
 import json
-import urllib
-import urllib2
 import base64
 import socket
 
@@ -100,23 +106,25 @@ class netscaler(object):
 
     _nitro_base_url = '/nitro/v1/'
 
+    def __init__(self, module):
+        self.module = module
+
     def http_request(self, api_endpoint, data_json={}):
         request_url = self._nsc_protocol + '://' + self._nsc_host + self._nitro_base_url + api_endpoint
-        data_json = urllib.urlencode(data_json)
 
-        if len(data_json):
-            req = urllib2.Request(request_url, data_json)
-            req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        else:
-            req = urllib2.Request(request_url)
+        data_json = urllib.urlencode(data_json)
+        if not len(data_json):
+            data_json = None
 
-        base64string = base64.encodestring('%s:%s' % (self._nsc_user, self._nsc_pass)).replace('\n', '').strip()
-        req.add_header('Authorization', "Basic %s" % base64string)
+        auth = base64.encodestring('%s:%s' % (self._nsc_user, self._nsc_pass)).replace('\n', '').strip()
+        headers = {
+            'Authorization': 'Basic %s' % auth,
+            'Content-Type' : 'application/x-www-form-urlencoded',
+        }
 
-        resp = urllib2.urlopen(req)
-        resp = json.load(resp)
+        response, info = fetch_url(self.module, request_url, data=data_json, validate_certs=self.module.params['validate_certs'])
 
-        return resp
+        return json.load(response.read())
 
     def prepare_request(self, action):
         resp = self.http_request(
@@ -134,7 +142,7 @@ class netscaler(object):
 
 
 def core(module):
-    n = netscaler()
+    n = netscaler(module)
     n._nsc_host = module.params.get('nsc_host')
     n._nsc_user = module.params.get('user')
     n._nsc_pass = module.params.get('password')
@@ -158,7 +166,8 @@ def main():
             password = dict(required=True),
             action = dict(default='enable', choices=['enable','disable']),
             name = dict(default=socket.gethostname()),
-            type = dict(default='server', choices=['service', 'server'])
+            type = dict(default='server', choices=['service', 'server']),
+            validate_certs=dict(default='yes', type='bool'),
         )
     )
 
@@ -177,4 +186,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()
diff --git a/library/network/get_url b/library/network/get_url
index 9704b8dbad..c249c44049 100644
--- a/library/network/get_url
+++ b/library/network/get_url
@@ -83,6 +83,13 @@ options:
     required: false
     default: 'yes'
     choices: ['yes', 'no']
+  validate_certs:
+    description:
+      - If C(no), SSL certificates will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
   others:
     description:
       - all arguments accepted by the M(file) module also work here
@@ -108,19 +115,6 @@ try:
 except ImportError:
     HAS_HASHLIB=False
 
-try:
-    import urllib2
-    HAS_URLLIB2 = True
-except ImportError:
-    HAS_URLLIB2 = False
-
-try:
-    import urlparse
-    import socket
-    HAS_URLPARSE = True
-except ImportError:
-    HAS_URLPARSE=False
-
 # ==============================================================
 # url handling
 
@@ -130,80 +124,14 @@ def url_filename(url):
         return 'index.html'
     return fn
 
-def url_do_get(module, url, dest, use_proxy, last_mod_time, force):
-    """
-    Get url and return request and info
-    Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp
-    """
-
-    USERAGENT = 'ansible-httpget'
-    info = dict(url=url, dest=dest)
-    r = None
-    handlers = []
-
-    parsed = urlparse.urlparse(url)
-
-    if '@' in parsed[1]:
-        credentials, netloc = parsed[1].split('@', 1)
-        if ':' in credentials:
-            username, password = credentials.split(':', 1)
-        else:
-            username = credentials
-            password = ''
-        parsed = list(parsed)
-        parsed[1] = netloc
-
-        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-        # this creates a password manager
-        passman.add_password(None, netloc, username, password)
-        # because we have put None at the start it will always
-        # use this username/password combination for  urls
-        # for which `theurl` is a super-url
-
-        authhandler = urllib2.HTTPBasicAuthHandler(passman)
-        # create the AuthHandler
-        handlers.append(authhandler)
-
-        #reconstruct url without credentials
-        url = urlparse.urlunparse(parsed)
-
-    if not use_proxy:
-        proxyhandler = urllib2.ProxyHandler({})
-        handlers.append(proxyhandler)
-
-    opener = urllib2.build_opener(*handlers)
-    urllib2.install_opener(opener)
-    request = urllib2.Request(url)
-    request.add_header('User-agent', USERAGENT)
-
-    if last_mod_time and not force:
-        tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
-        request.add_header('If-Modified-Since', tstamp)
-    else:
-        request.add_header('cache-control', 'no-cache')
-
-    try:
-        r = urllib2.urlopen(request)
-        info.update(r.info())
-        info['url'] = r.geturl()  # The URL goes in too, because of redirects.
-        info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
-    except urllib2.HTTPError, e:
-        # Must not fail_json() here so caller can handle HTTP 304 unmodified
-        info.update(dict(msg=str(e), status=e.code))
-    except urllib2.URLError, e:
-        code = getattr(e, 'code', -1)
-        module.fail_json(msg="Request failed: %s" % str(e), status_code=code)
-
-    return r, info
-
-def url_get(module, url, dest, use_proxy, last_mod_time, force):
+def url_get(module, url, dest, use_proxy, last_mod_time, force, validate_certs):
     """
     Download data from the url and store in a temporary file.
 
     Return (tempfile, info about the request)
     """
 
-    req, info = url_do_get(module, url, dest, use_proxy, last_mod_time, force)
+    rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, validate_certs=validate_certs)
 
     if info['status'] == 304:
         module.exit_json(url=url, dest=dest, changed=False, msg=info.get('msg', ''))
@@ -215,12 +143,12 @@ def url_get(module, url, dest, use_proxy, last_mod_time, force):
     fd, tempname = tempfile.mkstemp()
     f = os.fdopen(fd, 'wb')
     try:
-        shutil.copyfileobj(req, f)
+        shutil.copyfileobj(rsp, f)
     except Exception, err:
         os.remove(tempname)
         module.fail_json(msg="failed to create temporary content file: %s" % str(err))
     f.close()
-    req.close()
+    rsp.close()
     return tempname, info
 
 def extract_filename_from_headers(headers):
@@ -247,21 +175,15 @@ def extract_filename_from_headers(headers):
 
 def main():
 
-    # does this really happen on non-ancient python?
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-    if not HAS_URLPARSE:
-        module.fail_json(msg="urlparse is not installed")
+    argument_spec = url_argument_spec()
+    argument_spec.update(
+        dest = dict(required=True),
+        sha256sum = dict(default=''),
+    )
 
     module = AnsibleModule(
         # not checking because of daisy chain to file module
-        argument_spec = dict(
-            url = dict(required=True),
-            dest = dict(required=True),
-            force = dict(default='no', aliases=['thirsty'], type='bool'),
-            sha256sum = dict(default=''),
-            use_proxy = dict(default='yes', type='bool')
-        ),
+        argument_spec = argument_spec,
         add_file_common_args=True
     )
 
@@ -270,6 +192,7 @@ def main():
     force = module.params['force']
     sha256sum = module.params['sha256sum']
     use_proxy = module.params['use_proxy']
+    validate_certs = module.params['validate_certs']
 
     dest_is_dir = os.path.isdir(dest)
     last_mod_time = None
@@ -284,7 +207,7 @@ def main():
         last_mod_time = datetime.datetime.utcfromtimestamp(mtime)
 
     # download to tmpsrc
-    tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force)
+    tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, validate_certs)
 
     # Now the request has completed, we can finally generate the final
     # destination file name from the info dict.
@@ -366,4 +289,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()
diff --git a/library/notification/flowdock b/library/notification/flowdock
index a5be40d1f1..32817d756d 100644
--- a/library/notification/flowdock
+++ b/library/notification/flowdock
@@ -96,31 +96,12 @@ EXAMPLES = '''
             tags=tag1,tag2,tag3
 '''
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
-
-
 # ===========================================
 # Module execution.
 #
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -187,14 +168,16 @@ def main():
         module.exit_json(changed=False)
 
     # Send the data to Flowdock
-    try:
-        response = urllib2.urlopen(url, urllib.urlencode(params))
-    except Exception, e:
-        module.fail_json(msg="unable to send msg: %s" % e)
+    data = urllib.urlencode(params)
+    response, info = fetch_url(module, url, data=data)
+    if info['status'] != 200:
+        module.fail_json(msg="unable to send msg: %s" % info['msg'])
 
-    module.exit_json(changed=False, msg=module.params["msg"])
+    module.exit_json(changed=True, msg=module.params["msg"])
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
 
diff --git a/library/notification/grove b/library/notification/grove
index b759f025e2..1e2132cfb7 100644
--- a/library/notification/grove
+++ b/library/notification/grove
@@ -41,8 +41,6 @@ EXAMPLES = '''
     message=deployed {{ target }}
 '''
 
-import urllib
-
 BASE_URL = 'https://grove.io/api/notice/%s/'
 
 # ==============================================================
@@ -57,7 +55,10 @@ def do_notify_grove(module, channel_token, service, message, url=None, icon_url=
     if icon_url is not None:
         my_data['icon_url'] = icon_url
 
-    urllib.urlopen(my_url, urllib.urlencode(my_data))
+    data = urllib.urlencode(my_data)
+    response, info = fetch_url(module, my_url, data=data)
+    if info['status'] != 200:
+        module.fail_json(msg="failed to send notification: %s" % info['msg'])
 
 # ==============================================================
 # main
diff --git a/library/notification/hipchat b/library/notification/hipchat
index eec2b8c361..c4b36d64ce 100644
--- a/library/notification/hipchat
+++ b/library/notification/hipchat
@@ -60,22 +60,10 @@ EXAMPLES = '''
 # HipChat module specific support methods.
 #
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
 MSG_URI = "https://api.hipchat.com/v1/rooms/message?"
 
 
-def send_msg(token, room, msg_from, msg, msg_format='text',
+def send_msg(module, token, room, msg_from, msg, msg_format='text',
              color='yellow', notify=False):
     '''sending message to hipchat'''
 
@@ -92,8 +80,12 @@ def send_msg(token, room, msg_from, msg, msg_format='text',
         params['notify'] = 0
 
     url = MSG_URI + "auth_token=%s" % (token)
-    response = urllib2.urlopen(url, urllib.urlencode(params))
-    return response.read()
+    data = urllib.urlencode(params)
+    response, info = fetch_url(module, url, data=data)
+    if info['status'] == 200:
+        return response.read()
+    else:
+        module.fail_json(msg="failed to send message, return status=%s" % str(info['status']))
 
 
 # ===========================================
@@ -102,11 +94,6 @@ def send_msg(token, room, msg_from, msg, msg_format='text',
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -130,15 +117,15 @@ def main():
     notify = module.params["notify"]
 
     try:
-        send_msg(token, room, msg_from, msg, msg_format,
-                 color, notify)
+        send_msg(module, token, room, msg_from, msg, msg_format, color, notify)
     except Exception, e:
         module.fail_json(msg="unable to sent msg: %s" % e)
 
     changed = True
-    module.exit_json(changed=changed, room=room, msg_from=msg_from,
-                     msg=msg)
+    module.exit_json(changed=changed, room=room, msg_from=msg_from, msg=msg)
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
diff --git a/library/packaging/apt_key b/library/packaging/apt_key
index eee8633702..ff05bb93d1 100644
--- a/library/packaging/apt_key
+++ b/library/packaging/apt_key
@@ -64,6 +64,14 @@ options:
         default: present
         description:
             - used to specify if key is being added or revoked
+    validate_certs:
+        description:
+            - If C(no), SSL certificates for the target url will not be validated. This should only be used
+              on personally controlled sites using self-signed certificates.
+        required: false
+        default: 'yes'
+        choices: ['yes', 'no']
+
 '''
 
 EXAMPLES = '''
@@ -88,7 +96,6 @@ EXAMPLES = '''
 
 
 # FIXME: standardize into module_common
-from urllib2 import urlopen, URLError
 from traceback import format_exc
 from re import compile as re_compile
 # FIXME: standardize into module_common
@@ -133,11 +140,8 @@ def download_key(module, url):
     if url is None:
         module.fail_json(msg="needed a URL but was not specified")
     try:
-        connection = urlopen(url)
-        if connection is None:
-            module.fail_json("error connecting to download key from url")
-        data = connection.read()
-        return data
+        rsp, info = fetch_url(module, url, validate_certs=module.params['validate_certs'])
+        return rsp.read()
     except Exception:
         module.fail_json(msg="error getting key id from url", traceback=format_exc())
 
@@ -175,7 +179,8 @@ def main():
             file=dict(required=False),
             key=dict(required=False),
             keyring=dict(required=False),
-            state=dict(required=False, choices=['present', 'absent'], default='present')
+            state=dict(required=False, choices=['present', 'absent'], default='present'),
+            validate_certs=dict(default='yes', type='bool'),
         ),
         supports_check_mode=True
     )
@@ -240,4 +245,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()
diff --git a/library/packaging/rpm_key b/library/packaging/rpm_key
index 8253247734..9d85f30ac8 100644
--- a/library/packaging/rpm_key
+++ b/library/packaging/rpm_key
@@ -42,6 +42,14 @@ options:
       choices: [present, absent]
       description:
           - Wheather the key will be imported or removed from the rpm db.
+    validate_certs:
+      description:
+          - If C(no) and the C(key) is a url starting with https, SSL certificates will not be validated. This should only be used
+            on personally controlled sites using self-signed certificates.
+      required: false
+      default: 'yes'
+      choices: ['yes', 'no']
+
 '''
 
 EXAMPLES = '''
@@ -57,7 +65,6 @@ EXAMPLES = '''
 import syslog
 import os.path
 import re
-import urllib2
 import tempfile
 
 # Attempt to download at most 8192 bytes.
@@ -116,8 +123,8 @@ class RpmKey:
     def fetch_key(self, url, maxbytes=MAXBYTES):
         """Downloads a key from url, returns a valid path to a gpg key"""
         try:
-            fd = urllib2.urlopen(url)
-            key = fd.read(maxbytes)
+            rsp, info = fetch_url(self.module, url, validate_certs=self.module.params['validate_certs'])
+            key = rsp.read(maxbytes)
             if not is_pubkey(key):
                 self.module.fail_json(msg="Not a public key: %s" % url)
             tmpfd, tmpname = tempfile.mkstemp()
@@ -187,7 +194,8 @@ def main():
     module = AnsibleModule(
             argument_spec = dict(
                 state=dict(default='present', choices=['present', 'absent'], type='str'),
-                key=dict(required=True, type='str')
+                key=dict(required=True, type='str'),
+                validate_certs=dict(default='yes', type='bool'),
                 ),
             supports_check_mode=True
             )
@@ -198,4 +206,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()
diff --git a/library/source_control/github_hooks b/library/source_control/github_hooks
index 55eb8d3c8d..c5c5b648c7 100644
--- a/library/source_control/github_hooks
+++ b/library/source_control/github_hooks
@@ -19,7 +19,6 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
-import urllib2
 import base64
 
 DOCUMENTATION = '''
@@ -51,6 +50,14 @@ options:
       - This tells the githooks module what you want it to do.
     required: true
     choices: [ "create", "cleanall" ]
+  validate_certs:
+    description:
+      - If C(no), SSL certificates for the target repo will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
+
 author: Phillip Gentry, CX Inc
 '''
 
@@ -62,16 +69,19 @@ EXAMPLES = '''
 - local_action: github_hooks action=cleanall user={{ gituser }} oauthkey={{ oauthkey }} repo={{ repo }}
 '''
 
-def list(hookurl, oauthkey, repo, user):
+def list(module, hookurl, oauthkey, repo, user):
     url = "%s/hooks" % repo
     auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '')
-    req = urllib2.Request(url)
-    req.add_header("Authorization", "Basic %s" % auth)
-    res = urllib2.urlopen(req)
-    out = res.read()
-    return False, out
-
-def clean504(hookurl, oauthkey, repo, user):
+    headers = {
+        'Authorization': 'Basic %s' % auth,
+    }
+    response, info = fetch_url(module, url, headers=headers, validate_certs=module.params['validate_certs'])
+    if info['status'] != 200:
+        return False, ''
+    else:
+        return False, response.read()
+
+def clean504(module, hookurl, oauthkey, repo, user):
     current_hooks = list(hookurl, oauthkey, repo, user)[1]
     decoded = json.loads(current_hooks)
 
@@ -79,11 +89,11 @@ def clean504(hookurl, oauthkey, repo, user):
         if hook['last_response']['code'] == 504:
             # print "Last response was an ERROR for hook:"
             # print hook['id']
-            delete(hookurl, oauthkey, repo, user, hook['id'])
+            delete(module, hookurl, oauthkey, repo, user, hook['id'])
             
     return 0, current_hooks
 
-def cleanall(hookurl, oauthkey, repo, user):
+def cleanall(module, hookurl, oauthkey, repo, user):
     current_hooks = list(hookurl, oauthkey, repo, user)[1]
     decoded = json.loads(current_hooks)
 
@@ -91,11 +101,11 @@ def cleanall(hookurl, oauthkey, repo, user):
         if hook['last_response']['code'] != 200:
             # print "Last response was an ERROR for hook:"
             # print hook['id']
-            delete(hookurl, oauthkey, repo, user, hook['id'])
+            delete(module, hookurl, oauthkey, repo, user, hook['id'])
             
     return 0, current_hooks
 
-def create(hookurl, oauthkey, repo, user):
+def create(module, hookurl, oauthkey, repo, user):
     url = "%s/hooks" % repo
     values = {
         "active": True,
@@ -107,29 +117,23 @@ def create(hookurl, oauthkey, repo, user):
         }
     data = json.dumps(values)
     auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '')
-    out='[]'
-    try :
-        req = urllib2.Request(url)
-        req.add_data(data)
-        req.add_header("Authorization", "Basic %s" % auth)
-        res = urllib2.urlopen(req)
-        out = res.read()
-        return 0, out
-    except urllib2.HTTPError, e :
-        if e.code == 422 :
-            return 0, out
-
-def delete(hookurl, oauthkey, repo, user, hookid):
+    headers = {
+        'Authorization': 'Basic %s' % auth,
+    }
+    response, info = fetch_url(module, url, data=data, headers=headers, validate_certs=module.params['validate_certs'])
+    if info['status'] != 200:
+        return 0, '[]'
+    else:
+        return 0, response.read()
+
+def delete(module, hookurl, oauthkey, repo, user, hookid):
     url = "%s/hooks/%s" % (repo, hookid)
     auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '')
-    req = urllib2.Request(url)
-    req.get_method = lambda: 'DELETE'
-    req.add_header("Authorization", "Basic %s" % auth)
-    # req.add_header('Content-Type', 'application/xml') 
-    # req.add_header('Accept', 'application/xml')
-    res = urllib2.urlopen(req)
-    out = res.read()
-    return out
+    headers = {
+        'Authorization': 'Basic %s' % auth,
+    }
+    response, info = fetch_url(module, url, data=data, headers=headers, method='DELETE', validate_certs=module.params['validate_certs'])
+    return response.read()
 
 def main():
     module = AnsibleModule(
@@ -139,6 +143,7 @@ def main():
         oauthkey=dict(required=True),
         repo=dict(required=True),
         user=dict(required=True),
+        validate_certs=dict(default='yes', type='bool'),
         )
     )
 
@@ -149,16 +154,16 @@ def main():
     user = module.params['user']
 
     if action == "list":
-        (rc, out) = list(hookurl, oauthkey, repo, user)
+        (rc, out) = list(module, hookurl, oauthkey, repo, user)
 
     if action == "clean504":
-        (rc, out) = clean504(hookurl, oauthkey, repo, user)
+        (rc, out) = clean504(module, hookurl, oauthkey, repo, user)
 
     if action == "cleanall":
-        (rc, out) = cleanall(hookurl, oauthkey, repo, user)
+        (rc, out) = cleanall(module, hookurl, oauthkey, repo, user)
 
     if action == "create":
-        (rc, out) = create(hookurl, oauthkey, repo, user)
+        (rc, out) = create(module, hookurl, oauthkey, repo, user)
 
     if rc != 0:
         module.fail_json(msg="failed", result=out)
@@ -168,4 +173,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()