Misc module cleanup (#1418)

- move datasource helpers to dedicated directory
- drop unnecessary executable bit on shebangless python files

35 files changed, 40 insertions, 44 deletions

diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py
index 0e1db118..1a017608 100644..100755
--- a/cloudinit/cmd/clean.py
+++ b/cloudinit/cmd/clean.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # Copyright (C) 2017 Canonical Ltd.
 #
 # This file is part of cloud-init. See LICENSE file for license information.
@@ -120,5 +122,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/cloud_id.py b/cloudinit/cmd/cloud_id.py
index b9c30fb4..34160f8c 100755
--- a/cloudinit/cmd/cloud_id.py
+++ b/cloudinit/cmd/cloud_id.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Commandline utility to list the canonical cloud-id for an instance."""
@@ -111,5 +113,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
index ead5f7a9..ead5f7a9 100644..100755
--- a/cloudinit/cmd/devel/__init__.py
+++ b/cloudinit/cmd/devel/__init__.py
diff --git a/cloudinit/cmd/devel/hotplug_hook.py b/cloudinit/cmd/devel/hotplug_hook.py
index 7f663d24..af45ed22 100644..100755
--- a/cloudinit/cmd/devel/hotplug_hook.py
+++ b/cloudinit/cmd/devel/hotplug_hook.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # This file is part of cloud-init. See LICENSE file for license information.
 """Handle reconfiguration on hotplug events"""
 import abc
diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py
index d54b809a..fbe8c500 100644..100755
--- a/cloudinit/cmd/devel/logs.py
+++ b/cloudinit/cmd/devel/logs.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # Copyright (C) 2017 Canonical Ltd.
 #
 # This file is part of cloud-init. See LICENSE file for license information.
@@ -189,5 +191,3 @@ def main():
 
 if __name__ == "__main__":
     sys.exit(main())
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py
index c7671a93..b1b61949 100755
--- a/cloudinit/cmd/devel/make_mime.py
+++ b/cloudinit/cmd/devel/make_mime.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Generate multi-part mime messages for user-data """
@@ -138,6 +140,3 @@ def main():
 
 if __name__ == "__main__":
     sys.exit(main())
-
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
index 647fe07b..e3f58e90 100755
--- a/cloudinit/cmd/devel/net_convert.py
+++ b/cloudinit/cmd/devel/net_convert.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Debug network config format conversions."""
@@ -179,6 +180,3 @@ def handle_args(name, args):
 if __name__ == "__main__":
     args = get_parser().parse_args()
     handle_args(NAME, args)
-
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
index 2f9a22a8..62b432d2 100755
--- a/cloudinit/cmd/devel/render.py
+++ b/cloudinit/cmd/devel/render.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Debug jinja template rendering of user-data."""
@@ -110,6 +112,3 @@ def main():
 
 if __name__ == "__main__":
     sys.exit(main())
-
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 101ac1a4..9642dc8e 100644..100755
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -1,4 +1,5 @@
-#
+#!/usr/bin/env python3
+
 # Copyright (C) 2012 Canonical Ltd.
 # Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
 # Copyright (C) 2012 Yahoo! Inc.
@@ -1099,5 +1100,3 @@ if __name__ == "__main__":
     return_value = main(sys.argv)
     if return_value:
         sys.exit(return_value)
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py
index 46f17699..b9347200 100644..100755
--- a/cloudinit/cmd/query.py
+++ b/cloudinit/cmd/query.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Query standardized instance metadata provided to machine, returning a JSON
@@ -317,5 +319,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/status.py b/cloudinit/cmd/status.py
index 5176549d..f3b4f161 100644..100755
--- a/cloudinit/cmd/status.py
+++ b/cloudinit/cmd/status.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # Copyright (C) 2017 Canonical Ltd.
 #
 # This file is part of cloud-init. See LICENSE file for license information.
@@ -180,5 +182,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py
index fbc20410..fbc20410 100755..100644
--- a/cloudinit/config/cc_byobu.py
+++ b/cloudinit/config/cc_byobu.py
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index 939e773b..939e773b 100755..100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index ac5640b7..ac5640b7 100755..100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index db5c1454..db5c1454 100755..100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py
index 6a15895d..6a15895d 100755..100644
--- a/cloudinit/config/cc_ssh_import_id.py
+++ b/cloudinit/config/cc_ssh_import_id.py
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index b97ee3ab..b97ee3ab 100755..100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py
index e0a4d068..e0a4d068 100755..100644
--- a/cloudinit/distros/ug_util.py
+++ b/cloudinit/distros/ug_util.py
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index eab86d9f..eab86d9f 100755..100644
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py
index e163e168..e163e168 100755..100644
--- a/cloudinit/reporting/handlers.py
+++ b/cloudinit/reporting/handlers.py
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index fd846dd6..fd846dd6 100755..100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index de71c3e9..7d702137 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -10,7 +10,7 @@ from base64 import b64decode
 from cloudinit import dmi
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit.cs_utils import SERIAL_PORT, Cepko
+from cloudinit.sources.helpers.cloudsigma import SERIAL_PORT, Cepko
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index a742a5e6..32cc078b 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -17,12 +17,12 @@ import time
 from socket import gaierror, getaddrinfo, inet_ntoa
 from struct import pack
 
-from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources, subp
 from cloudinit import url_helper as uhelp
 from cloudinit import util
 from cloudinit.net import dhcp
+from cloudinit.sources.helpers import ec2
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 961c5090..9c525d3d 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -13,13 +13,13 @@ import os
 import time
 
 from cloudinit import dmi
-from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import net, sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util, warnings
 from cloudinit.event import EventScope, EventType
 from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
+from cloudinit.sources.helpers import ec2
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/sources/DataSourceExoscale.py b/cloudinit/sources/DataSourceExoscale.py
index cc5136d7..23478e9e 100644
--- a/cloudinit/sources/DataSourceExoscale.py
+++ b/cloudinit/sources/DataSourceExoscale.py
@@ -3,11 +3,10 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit import dmi
-from cloudinit import ec2_utils as ec2
-from cloudinit import helpers
+from cloudinit import dmi, helpers
 from cloudinit import log as logging
 from cloudinit import sources, url_helper, util
+from cloudinit.sources.helpers import ec2
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 4bb8b8db..4bb8b8db 100755..100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
diff --git a/cloudinit/cs_utils.py b/cloudinit/sources/helpers/cloudsigma.py
index 6db7e117..6db7e117 100644
--- a/cloudinit/cs_utils.py
+++ b/cloudinit/sources/helpers/cloudsigma.py
diff --git a/cloudinit/ec2_utils.py b/cloudinit/sources/helpers/ec2.py
index d4019557..d4019557 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/sources/helpers/ec2.py
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index a42543e4..1f48dac6 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -12,10 +12,10 @@ import copy
 import functools
 import os
 
-from cloudinit import ec2_utils
 from cloudinit import log as logging
 from cloudinit import net, sources, subp, url_helper, util
 from cloudinit.sources import BrokenMetadata
+from cloudinit.sources.helpers import ec2
 
 # See https://docs.openstack.org/user-guide/cli-config-drive.html
 
@@ -515,7 +515,7 @@ class MetadataReader(BaseReader):
         return url_helper.combine_url(base, *add_ons)
 
     def _read_ec2_metadata(self):
-        return ec2_utils.get_instance_metadata(
+        return ec2.get_instance_metadata(
             ssl_details=self.ssl_details,
             timeout=self.timeout,
             retries=self.retries,
diff --git a/setup.py b/setup.py
index a7ddf604..a7ddf604 100755..100644
--- a/setup.py
+++ b/setup.py
diff --git a/tests/integration_tests/datasources/test_ec2.py b/tests/integration_tests/datasources/test_ec2_ipv6.py
index 8cde4dc9..8cde4dc9 100644
--- a/tests/integration_tests/datasources/test_ec2.py
+++ b/tests/integration_tests/datasources/test_ec2_ipv6.py
diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/sources/helpers/test_cloudsigma.py
index 109e0208..3c687388 100644
--- a/tests/unittests/test_cs_util.py
+++ b/tests/unittests/sources/helpers/test_cloudsigma.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit.cs_utils import Cepko
+from cloudinit.sources.helpers.cloudsigma import Cepko
 from tests.unittests import helpers as test_helpers
 
 SERVER_CONTEXT = {
diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/sources/helpers/test_ec2.py
index f447d295..77e7c7b6 100644
--- a/tests/unittests/test_ec2_util.py
+++ b/tests/unittests/sources/helpers/test_ec2.py
@@ -2,8 +2,8 @@
 
 import httpretty as hp
 
-from cloudinit import ec2_utils as eu
 from cloudinit import url_helper as uh
+from cloudinit.sources.helpers import ec2
 from tests.unittests import helpers
 
 
@@ -17,7 +17,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             body="stuff",
             status=200,
         )
-        userdata = eu.get_instance_userdata(self.VERSION)
+        userdata = ec2.get_instance_userdata(self.VERSION)
         self.assertEqual("stuff", userdata.decode("utf-8"))
 
     def test_userdata_fetch_fail_not_found(self):
@@ -26,7 +26,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             "http://169.254.169.254/%s/user-data" % (self.VERSION),
             status=404,
         )
-        userdata = eu.get_instance_userdata(self.VERSION, retries=0)
+        userdata = ec2.get_instance_userdata(self.VERSION, retries=0)
         self.assertEqual("", userdata)
 
     def test_userdata_fetch_fail_server_dead(self):
@@ -35,7 +35,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             "http://169.254.169.254/%s/user-data" % (self.VERSION),
             status=500,
         )
-        userdata = eu.get_instance_userdata(self.VERSION, retries=0)
+        userdata = ec2.get_instance_userdata(self.VERSION, retries=0)
         self.assertEqual("", userdata)
 
     def test_userdata_fetch_fail_server_not_found(self):
@@ -44,7 +44,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             "http://169.254.169.254/%s/user-data" % (self.VERSION),
             status=404,
         )
-        userdata = eu.get_instance_userdata(self.VERSION)
+        userdata = ec2.get_instance_userdata(self.VERSION)
         self.assertEqual("", userdata)
 
     def test_metadata_fetch_no_keys(self):
@@ -73,7 +73,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             status=200,
             body="1",
         )
-        md = eu.get_instance_metadata(self.VERSION, retries=0)
+        md = ec2.get_instance_metadata(self.VERSION, retries=0)
         self.assertEqual(md["hostname"], "ec2.fake.host.name.com")
         self.assertEqual(md["instance-id"], "123")
         self.assertEqual(md["ami-launch-index"], "1")
@@ -110,7 +110,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             status=200,
             body="ssh-rsa AAAA.....wZEf my-public-key",
         )
-        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
         self.assertEqual(md["hostname"], "ec2.fake.host.name.com")
         self.assertEqual(md["instance-id"], "123")
         self.assertEqual(1, len(md["public-keys"]))
@@ -153,7 +153,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             status=200,
             body="ssh-rsa AAAA.....wZEf my-other-key",
         )
-        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
         self.assertEqual(md["hostname"], "ec2.fake.host.name.com")
         self.assertEqual(md["instance-id"], "123")
         self.assertEqual(2, len(md["public-keys"]))
@@ -198,7 +198,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             status=200,
             body="sdc",
         )
-        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
         self.assertEqual(md["hostname"], "ec2.fake.host.name.com")
         self.assertEqual(md["instance-id"], "123")
         bdm = md["block-device-mapping"]
@@ -266,7 +266,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
             status=200,
             body="2016-10-28T00:00:34Z",
         )
-        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
         self.assertEqual(md["instance-id"], "i-0123451689abcdef0")
         iam = md["iam"]
         self.assertEqual(1, len(iam))
diff --git a/tests/unittests/sources/test_cloudsigma.py b/tests/unittests/sources/test_cloudsigma.py
index a2f26245..8cd58c96 100644
--- a/tests/unittests/sources/test_cloudsigma.py
+++ b/tests/unittests/sources/test_cloudsigma.py
@@ -3,8 +3,8 @@
 import copy
 
 from cloudinit import distros, helpers, sources
-from cloudinit.cs_utils import Cepko
 from cloudinit.sources import DataSourceCloudSigma
+from cloudinit.sources.helpers.cloudsigma import Cepko
 from tests.unittests import helpers as test_helpers
 
 SERVER_CONTEXT = {
diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py
index 800ef379..e5648007 100644
--- a/tests/unittests/sources/test_ec2.py
+++ b/tests/unittests/sources/test_ec2.py
@@ -39,7 +39,7 @@ DYNAMIC_METADATA = {
 
 # collected from api version 2016-09-02/ with
 # python3 -c 'import json
-# from cloudinit.ec2_utils import get_instance_metadata as gm
+# from cloudinit.sources.helpers.ec2 import get_instance_metadata as gm
 # print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
 # Note that the MAC addresses have been modified to sort in the opposite order
 # to the device-number attribute, to test LP: #1876312
@@ -124,7 +124,7 @@ DEFAULT_METADATA = {
 
 # collected from api version 2018-09-24/ with
 # python3 -c 'import json
-# from cloudinit.ec2_utils import get_instance_metadata as gm
+# from cloudinit.sources.helpers.ec2 import get_instance_metadata as gm
 # print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
 
 NIC1_MD_IPV4_IPV6_MULTI_IP = {