From c3c3dc693c14175e110b5fe125d4d5f98ace9700 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Aug 2016 09:06:15 -0600 Subject: README: Mention move of revision control to git. cloud-init development has moved its revision control to git. It is available at https://code.launchpad.net/cloud-init Clone with git clone https://git.launchpad.net/cloud-init or git clone git+ssh://git.launchpad.net/cloud-init For more information see https://git.launchpad.net/cloud-init/tree/HACKING.rst --- .bzrignore | 4 - ChangeLog | 776 ------- HACKING.rst | 48 - LICENSE | 674 ------ MANIFEST.in | 8 - Makefile | 82 - README | 11 + TODO.rst | 43 - cloudinit/__init__.py | 21 - cloudinit/cloud.py | 109 - cloudinit/cmd/__init__.py | 21 - cloudinit/cmd/main.py | 685 ------ cloudinit/config/__init__.py | 58 - cloudinit/config/cc_apt_configure.py | 319 --- cloudinit/config/cc_apt_pipelining.py | 57 - cloudinit/config/cc_bootcmd.py | 54 - cloudinit/config/cc_byobu.py | 80 - cloudinit/config/cc_ca_certs.py | 104 - cloudinit/config/cc_chef.py | 342 --- cloudinit/config/cc_debug.py | 109 - cloudinit/config/cc_disable_ec2_metadata.py | 36 - cloudinit/config/cc_disk_setup.py | 863 -------- cloudinit/config/cc_emit_upstart.py | 69 - cloudinit/config/cc_fan.py | 101 - cloudinit/config/cc_final_message.py | 73 - cloudinit/config/cc_foo.py | 52 - cloudinit/config/cc_growpart.py | 300 --- cloudinit/config/cc_grub_dpkg.py | 73 - cloudinit/config/cc_keys_to_console.py | 62 - cloudinit/config/cc_landscape.py | 99 - cloudinit/config/cc_locale.py | 37 - cloudinit/config/cc_lxd.py | 177 -- cloudinit/config/cc_mcollective.py | 106 - cloudinit/config/cc_migrator.py | 85 - cloudinit/config/cc_mounts.py | 405 ---- .../config/cc_package_update_upgrade_install.py | 99 - cloudinit/config/cc_phone_home.py | 122 -- cloudinit/config/cc_power_state_change.py | 223 -- cloudinit/config/cc_puppet.py | 118 - cloudinit/config/cc_resizefs.py | 185 -- cloudinit/config/cc_resolv_conf.py | 116 - cloudinit/config/cc_rh_subscription.py | 408 ---- cloudinit/config/cc_rightscale_userdata.py | 102 - cloudinit/config/cc_rsyslog.py | 366 ---- cloudinit/config/cc_runcmd.py | 38 - cloudinit/config/cc_salt_minion.py | 59 - cloudinit/config/cc_scripts_per_boot.py | 41 - cloudinit/config/cc_scripts_per_instance.py | 41 - cloudinit/config/cc_scripts_per_once.py | 41 - cloudinit/config/cc_scripts_user.py | 42 - cloudinit/config/cc_scripts_vendor.py | 43 - cloudinit/config/cc_seed_random.py | 94 - cloudinit/config/cc_set_hostname.py | 37 - cloudinit/config/cc_set_passwords.py | 167 -- cloudinit/config/cc_snappy.py | 304 --- cloudinit/config/cc_ssh.py | 142 -- cloudinit/config/cc_ssh_authkey_fingerprints.py | 105 - cloudinit/config/cc_ssh_import_id.py | 99 - cloudinit/config/cc_timezone.py | 39 - cloudinit/config/cc_ubuntu_init_switch.py | 162 -- cloudinit/config/cc_update_etc_hosts.py | 60 - cloudinit/config/cc_update_hostname.py | 43 - cloudinit/config/cc_users_groups.py | 34 - cloudinit/config/cc_write_files.py | 105 - cloudinit/config/cc_yum_add_repo.py | 107 - cloudinit/cs_utils.py | 106 - cloudinit/distros/__init__.py | 980 --------- cloudinit/distros/arch.py | 201 -- cloudinit/distros/debian.py | 236 -- cloudinit/distros/fedora.py | 31 - cloudinit/distros/freebsd.py | 417 ---- cloudinit/distros/gentoo.py | 160 -- cloudinit/distros/net_util.py | 182 -- cloudinit/distros/parsers/__init__.py | 28 - cloudinit/distros/parsers/hostname.py | 88 - cloudinit/distros/parsers/hosts.py | 92 - cloudinit/distros/parsers/resolv_conf.py | 169 -- cloudinit/distros/parsers/sys_conf.py | 113 - cloudinit/distros/rhel.py | 230 -- cloudinit/distros/rhel_util.py | 89 - cloudinit/distros/sles.py | 179 -- cloudinit/distros/ubuntu.py | 31 - cloudinit/ec2_utils.py | 201 -- cloudinit/filters/__init__.py | 21 - cloudinit/filters/launch_index.py | 75 - cloudinit/gpg.py | 74 - cloudinit/handlers/__init__.py | 274 --- cloudinit/handlers/boot_hook.py | 70 - cloudinit/handlers/cloud_config.py | 163 -- cloudinit/handlers/shell_script.py | 55 - cloudinit/handlers/upstart_job.py | 119 -- cloudinit/helpers.py | 460 ---- cloudinit/importer.py | 58 - cloudinit/log.py | 155 -- cloudinit/mergers/__init__.py | 166 -- cloudinit/mergers/m_dict.py | 88 - cloudinit/mergers/m_list.py | 89 - cloudinit/mergers/m_str.py | 46 - cloudinit/net/__init__.py | 371 ---- cloudinit/net/cmdline.py | 203 -- cloudinit/net/eni.py | 504 ----- cloudinit/net/network_state.py | 454 ---- cloudinit/net/renderer.py | 48 - cloudinit/net/sysconfig.py | 400 ---- cloudinit/net/udev.py | 54 - cloudinit/netinfo.py | 249 --- cloudinit/patcher.py | 58 - cloudinit/registry.py | 37 - cloudinit/reporting/__init__.py | 42 - cloudinit/reporting/events.py | 248 --- cloudinit/reporting/handlers.py | 91 - cloudinit/safeyaml.py | 32 - cloudinit/serial.py | 50 - cloudinit/settings.py | 68 - cloudinit/signal_handler.py | 71 - cloudinit/sources/DataSourceAltCloud.py | 292 --- cloudinit/sources/DataSourceAzure.py | 651 ------ cloudinit/sources/DataSourceBigstep.py | 57 - cloudinit/sources/DataSourceCloudSigma.py | 132 -- cloudinit/sources/DataSourceCloudStack.py | 253 --- cloudinit/sources/DataSourceConfigDrive.py | 278 --- cloudinit/sources/DataSourceDigitalOcean.py | 110 - cloudinit/sources/DataSourceEc2.py | 211 -- cloudinit/sources/DataSourceGCE.py | 167 -- cloudinit/sources/DataSourceMAAS.py | 353 --- cloudinit/sources/DataSourceNoCloud.py | 323 --- cloudinit/sources/DataSourceNone.py | 57 - cloudinit/sources/DataSourceOVF.py | 429 ---- cloudinit/sources/DataSourceOpenNebula.py | 429 ---- cloudinit/sources/DataSourceOpenStack.py | 168 -- cloudinit/sources/DataSourceSmartOS.py | 781 ------- cloudinit/sources/__init__.py | 371 ---- cloudinit/sources/helpers/__init__.py | 13 - cloudinit/sources/helpers/azure.py | 279 --- cloudinit/sources/helpers/openstack.py | 648 ------ cloudinit/sources/helpers/vmware/__init__.py | 13 - cloudinit/sources/helpers/vmware/imc/__init__.py | 13 - cloudinit/sources/helpers/vmware/imc/boot_proto.py | 25 - cloudinit/sources/helpers/vmware/imc/config.py | 95 - .../sources/helpers/vmware/imc/config_file.py | 129 -- .../sources/helpers/vmware/imc/config_namespace.py | 25 - cloudinit/sources/helpers/vmware/imc/config_nic.py | 247 --- .../sources/helpers/vmware/imc/config_source.py | 23 - .../sources/helpers/vmware/imc/guestcust_error.py | 24 - .../sources/helpers/vmware/imc/guestcust_event.py | 27 - .../sources/helpers/vmware/imc/guestcust_state.py | 25 - .../sources/helpers/vmware/imc/guestcust_util.py | 128 -- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 45 - cloudinit/sources/helpers/vmware/imc/nic.py | 147 -- cloudinit/sources/helpers/vmware/imc/nic_base.py | 154 -- cloudinit/ssh_util.py | 314 --- cloudinit/stages.py | 890 -------- cloudinit/templater.py | 155 -- cloudinit/type_utils.py | 52 - cloudinit/url_helper.py | 509 ----- cloudinit/user_data.py | 356 ---- cloudinit/util.py | 2246 -------------------- cloudinit/version.py | 27 - config/cloud.cfg | 115 - config/cloud.cfg-freebsd | 88 - config/cloud.cfg.d/05_logging.cfg | 66 - config/cloud.cfg.d/README | 3 - doc/README | 4 - doc/examples/cloud-config-add-apt-repos.txt | 34 - doc/examples/cloud-config-archive-launch-index.txt | 30 - doc/examples/cloud-config-archive.txt | 16 - doc/examples/cloud-config-boot-cmds.txt | 15 - doc/examples/cloud-config-ca-certs.txt | 31 - doc/examples/cloud-config-chef-oneiric.txt | 90 - doc/examples/cloud-config-chef.txt | 95 - doc/examples/cloud-config-datasources.txt | 73 - doc/examples/cloud-config-disk-setup.txt | 251 --- doc/examples/cloud-config-final-message.txt | 7 - doc/examples/cloud-config-gluster.txt | 18 - doc/examples/cloud-config-growpart.txt | 31 - doc/examples/cloud-config-install-packages.txt | 15 - doc/examples/cloud-config-landscape.txt | 22 - doc/examples/cloud-config-launch-index.txt | 23 - doc/examples/cloud-config-lxd.txt | 55 - doc/examples/cloud-config-mcollective.txt | 49 - doc/examples/cloud-config-mount-points.txt | 46 - doc/examples/cloud-config-phone-home.txt | 14 - doc/examples/cloud-config-power-state.txt | 40 - doc/examples/cloud-config-puppet.txt | 51 - doc/examples/cloud-config-reporting.txt | 17 - doc/examples/cloud-config-resolv-conf.txt | 20 - doc/examples/cloud-config-rh_subscription.txt | 49 - doc/examples/cloud-config-rsyslog.txt | 46 - doc/examples/cloud-config-run-cmds.txt | 22 - doc/examples/cloud-config-salt-minion.txt | 53 - doc/examples/cloud-config-seed-random.txt | 32 - doc/examples/cloud-config-ssh-keys.txt | 46 - doc/examples/cloud-config-update-apt.txt | 7 - doc/examples/cloud-config-update-packages.txt | 8 - doc/examples/cloud-config-user-groups.txt | 109 - doc/examples/cloud-config-vendor-data.txt | 16 - doc/examples/cloud-config-write-files.txt | 33 - doc/examples/cloud-config-yum-repo.txt | 20 - doc/examples/cloud-config.txt | 752 ------- doc/examples/include-once.txt | 7 - doc/examples/include.txt | 5 - doc/examples/kernel-cmdline.txt | 18 - doc/examples/part-handler-v2.txt | 38 - doc/examples/part-handler.txt | 23 - doc/examples/plain-ignored.txt | 2 - doc/examples/seed/README | 22 - doc/examples/seed/meta-data | 30 - doc/examples/seed/user-data | 3 - doc/examples/upstart-cloud-config.txt | 12 - doc/examples/upstart-rclocal.txt | 12 - doc/examples/user-script.txt | 8 - doc/merging.rst | 194 -- doc/rtd/conf.py | 77 - doc/rtd/index.rst | 31 - doc/rtd/static/logo.png | Bin 12751 -> 0 bytes doc/rtd/static/logo.svg | 89 - doc/rtd/topics/availability.rst | 20 - doc/rtd/topics/capabilities.rst | 24 - doc/rtd/topics/datasources.rst | 200 -- doc/rtd/topics/dir_layout.rst | 81 - doc/rtd/topics/examples.rst | 133 -- doc/rtd/topics/format.rst | 159 -- doc/rtd/topics/hacking.rst | 1 - doc/rtd/topics/merging.rst | 5 - doc/rtd/topics/modules.rst | 342 --- doc/rtd/topics/moreinfo.rst | 12 - doc/sources/altcloud/README.rst | 87 - doc/sources/azure/README.rst | 134 -- doc/sources/cloudsigma/README.rst | 38 - doc/sources/cloudstack/README.rst | 29 - doc/sources/configdrive/README.rst | 123 -- doc/sources/digitalocean/README.rst | 21 - doc/sources/kernel-cmdline.txt | 48 - doc/sources/nocloud/README.rst | 71 - doc/sources/opennebula/README.rst | 142 -- doc/sources/openstack/README.rst | 24 - doc/sources/ovf/README | 83 - doc/sources/ovf/example/ovf-env.xml | 46 - doc/sources/ovf/example/ubuntu-server.ovf | 130 -- doc/sources/ovf/make-iso | 156 -- doc/sources/ovf/ovf-env.xml.tmpl | 28 - doc/sources/ovf/ovfdemo.pem | 27 - doc/sources/ovf/user-data | 7 - doc/sources/smartos/README.rst | 149 -- doc/status.txt | 53 - doc/userdata.txt | 79 - doc/var-lib-cloud.txt | 63 - doc/vendordata.txt | 53 - packages/bddeb | 267 --- packages/brpm | 280 --- packages/debian/changelog.in | 6 - packages/debian/cloud-init.postinst | 16 - packages/debian/cloud-init.preinst | 20 - packages/debian/compat | 1 - packages/debian/control.in | 29 - packages/debian/copyright | 29 - packages/debian/dirs | 6 - packages/debian/rules.in | 23 - packages/debian/watch | 2 - packages/redhat/cloud-init.spec.in | 204 -- packages/suse/cloud-init.spec.in | 163 -- requirements.txt | 40 - setup.py | 217 -- systemd/cloud-config.service | 16 - systemd/cloud-config.target | 11 - systemd/cloud-final.service | 17 - systemd/cloud-init-generator | 130 -- systemd/cloud-init-local.service | 22 - systemd/cloud-init.service | 18 - systemd/cloud-init.target | 6 - sysvinit/debian/cloud-config | 64 - sysvinit/debian/cloud-final | 66 - sysvinit/debian/cloud-init | 64 - sysvinit/debian/cloud-init-local | 63 - sysvinit/freebsd/cloudconfig | 35 - sysvinit/freebsd/cloudfinal | 35 - sysvinit/freebsd/cloudinit | 35 - sysvinit/freebsd/cloudinitlocal | 35 - sysvinit/gentoo/cloud-config | 13 - sysvinit/gentoo/cloud-final | 11 - sysvinit/gentoo/cloud-init | 12 - sysvinit/gentoo/cloud-init-local | 13 - sysvinit/redhat/cloud-config | 121 -- sysvinit/redhat/cloud-final | 121 -- sysvinit/redhat/cloud-init | 121 -- sysvinit/redhat/cloud-init-local | 124 -- templates/chef_client.rb.tmpl | 58 - templates/hosts.debian.tmpl | 26 - templates/hosts.freebsd.tmpl | 24 - templates/hosts.redhat.tmpl | 24 - templates/hosts.suse.tmpl | 26 - templates/resolv.conf.tmpl | 30 - templates/sources.list.debian.tmpl | 32 - templates/sources.list.ubuntu.tmpl | 113 - test-requirements.txt | 18 - tests/__init__.py | 0 tests/configs/sample1.yaml | 52 - tests/data/filter_cloud_multipart.yaml | 30 - tests/data/filter_cloud_multipart_1.email | 11 - tests/data/filter_cloud_multipart_2.email | 39 - tests/data/filter_cloud_multipart_header.email | 11 - tests/data/merge_sources/expected1.yaml | 1 - tests/data/merge_sources/expected10.yaml | 7 - tests/data/merge_sources/expected11.yaml | 5 - tests/data/merge_sources/expected12.yaml | 5 - tests/data/merge_sources/expected2.yaml | 3 - tests/data/merge_sources/expected3.yaml | 1 - tests/data/merge_sources/expected4.yaml | 2 - tests/data/merge_sources/expected5.yaml | 7 - tests/data/merge_sources/expected6.yaml | 9 - tests/data/merge_sources/expected7.yaml | 38 - tests/data/merge_sources/expected8.yaml | 7 - tests/data/merge_sources/expected9.yaml | 5 - tests/data/merge_sources/source1-1.yaml | 3 - tests/data/merge_sources/source1-2.yaml | 5 - tests/data/merge_sources/source10-1.yaml | 6 - tests/data/merge_sources/source10-2.yaml | 6 - tests/data/merge_sources/source11-1.yaml | 5 - tests/data/merge_sources/source11-2.yaml | 3 - tests/data/merge_sources/source11-3.yaml | 3 - tests/data/merge_sources/source12-1.yaml | 8 - tests/data/merge_sources/source12-2.yaml | 5 - tests/data/merge_sources/source2-1.yaml | 6 - tests/data/merge_sources/source2-2.yaml | 5 - tests/data/merge_sources/source3-1.yaml | 4 - tests/data/merge_sources/source3-2.yaml | 4 - tests/data/merge_sources/source4-1.yaml | 3 - tests/data/merge_sources/source4-2.yaml | 6 - tests/data/merge_sources/source5-1.yaml | 6 - tests/data/merge_sources/source5-2.yaml | 8 - tests/data/merge_sources/source6-1.yaml | 5 - tests/data/merge_sources/source6-2.yaml | 8 - tests/data/merge_sources/source7-1.yaml | 27 - tests/data/merge_sources/source7-2.yaml | 17 - tests/data/merge_sources/source8-1.yaml | 7 - tests/data/merge_sources/source8-2.yaml | 6 - tests/data/merge_sources/source9-1.yaml | 5 - tests/data/merge_sources/source9-2.yaml | 6 - tests/data/mountinfo_precise_ext4.txt | 24 - tests/data/mountinfo_raring_btrfs.txt | 13 - .../roots/simple_ubuntu/etc/networks/interfaces | 3 - tests/data/user_data.1.txt | 15 - tests/data/vmware/cust-dhcp-2nic.cfg | 34 - tests/data/vmware/cust-static-2nic.cfg | 39 - tests/unittests/__init__.py | 0 tests/unittests/helpers.py | 291 --- tests/unittests/test__init__.py | 211 -- tests/unittests/test_builtin_handlers.py | 73 - tests/unittests/test_cli.py | 34 - tests/unittests/test_cs_util.py | 63 - tests/unittests/test_data.py | 576 ----- tests/unittests/test_datasource/__init__.py | 0 tests/unittests/test_datasource/test_altcloud.py | 452 ---- tests/unittests/test_datasource/test_azure.py | 640 ------ .../unittests/test_datasource/test_azure_helper.py | 412 ---- tests/unittests/test_datasource/test_cloudsigma.py | 99 - tests/unittests/test_datasource/test_cloudstack.py | 78 - .../unittests/test_datasource/test_configdrive.py | 597 ------ .../unittests/test_datasource/test_digitalocean.py | 127 -- tests/unittests/test_datasource/test_gce.py | 166 -- tests/unittests/test_datasource/test_maas.py | 163 -- tests/unittests/test_datasource/test_nocloud.py | 178 -- tests/unittests/test_datasource/test_opennebula.py | 300 --- tests/unittests/test_datasource/test_openstack.py | 347 --- tests/unittests/test_datasource/test_smartos.py | 543 ----- tests/unittests/test_distros/__init__.py | 0 tests/unittests/test_distros/test_generic.py | 233 -- tests/unittests/test_distros/test_hostname.py | 38 - tests/unittests/test_distros/test_hosts.py | 41 - tests/unittests/test_distros/test_netconfig.py | 381 ---- tests/unittests/test_distros/test_resolv.py | 67 - tests/unittests/test_distros/test_sysconfig.py | 82 - .../test_distros/test_user_data_normalize.py | 297 --- tests/unittests/test_ec2_util.py | 139 -- tests/unittests/test_filters/__init__.py | 0 tests/unittests/test_filters/test_launch_index.py | 132 -- tests/unittests/test_handler/__init__.py | 0 .../test_handler/test_handler_apt_configure.py | 109 - .../test_handler_apt_configure_sources_list.py | 180 -- .../test_handler/test_handler_apt_source.py | 516 ----- .../test_handler/test_handler_ca_certs.py | 271 --- tests/unittests/test_handler/test_handler_chef.py | 192 -- tests/unittests/test_handler/test_handler_debug.py | 81 - .../test_handler/test_handler_disk_setup.py | 30 - .../test_handler/test_handler_growpart.py | 220 -- .../unittests/test_handler/test_handler_locale.py | 67 - tests/unittests/test_handler/test_handler_lxd.py | 134 -- .../test_handler/test_handler_mcollective.py | 148 -- .../unittests/test_handler/test_handler_mounts.py | 133 -- .../test_handler/test_handler_power_state.py | 127 -- .../unittests/test_handler/test_handler_rsyslog.py | 174 -- .../test_handler/test_handler_seed_random.py | 227 -- .../test_handler/test_handler_set_hostname.py | 72 - .../unittests/test_handler/test_handler_snappy.py | 306 --- .../test_handler/test_handler_timezone.py | 76 - .../test_handler/test_handler_write_files.py | 112 - .../test_handler/test_handler_yum_add_repo.py | 68 - tests/unittests/test_helpers.py | 33 - tests/unittests/test_merging.py | 257 --- tests/unittests/test_net.py | 689 ------ tests/unittests/test_pathprefix2dict.py | 44 - tests/unittests/test_registry.py | 28 - tests/unittests/test_reporting.py | 371 ---- tests/unittests/test_rh_subscription.py | 226 -- tests/unittests/test_runs/__init__.py | 0 tests/unittests/test_runs/test_merge_run.py | 54 - tests/unittests/test_runs/test_simple_run.py | 81 - tests/unittests/test_sshutil.py | 171 -- tests/unittests/test_templating.py | 119 -- tests/unittests/test_util.py | 489 ----- tests/unittests/test_vmware_config_file.py | 103 - tools/21-cloudinit.conf | 6 - tools/Z99-cloud-locale-test.sh | 98 - tools/build-on-freebsd | 66 - tools/ccfg-merge-debug | 90 - tools/cloud-init-per | 60 - tools/hacking.py | 170 -- tools/make-dist-tarball | 21 - tools/make-mime.py | 60 - tools/make-tarball | 39 - tools/mock-meta.py | 454 ---- tools/motd-hook | 35 - tools/read-dependencies | 29 - tools/read-version | 26 - tools/run-pep8 | 21 - tools/run-pyflakes | 18 - tools/run-pyflakes3 | 2 - tools/tox-venv | 42 - tools/uncloud-init | 141 -- tools/validate-yaml.py | 25 - tools/write-ssh-key-fingerprints | 38 - tox.ini | 30 - udev/66-azure-ephemeral.rules | 18 - upstart/cloud-config.conf | 9 - upstart/cloud-final.conf | 10 - upstart/cloud-init-blocknet.conf | 83 - upstart/cloud-init-container.conf | 57 - upstart/cloud-init-local.conf | 16 - upstart/cloud-init-nonet.conf | 66 - upstart/cloud-init.conf | 9 - upstart/cloud-log-shutdown.conf | 19 - 441 files changed, 11 insertions(+), 53183 deletions(-) delete mode 100644 .bzrignore delete mode 100644 ChangeLog delete mode 100644 HACKING.rst delete mode 100644 LICENSE delete mode 100644 MANIFEST.in delete mode 100644 Makefile create mode 100644 README delete mode 100644 TODO.rst delete mode 100644 cloudinit/__init__.py delete mode 100644 cloudinit/cloud.py delete mode 100644 cloudinit/cmd/__init__.py delete mode 100644 cloudinit/cmd/main.py delete mode 100644 cloudinit/config/__init__.py delete mode 100644 cloudinit/config/cc_apt_configure.py delete mode 100644 cloudinit/config/cc_apt_pipelining.py delete mode 100644 cloudinit/config/cc_bootcmd.py delete mode 100644 cloudinit/config/cc_byobu.py delete mode 100644 cloudinit/config/cc_ca_certs.py delete mode 100644 cloudinit/config/cc_chef.py delete mode 100644 cloudinit/config/cc_debug.py delete mode 100644 cloudinit/config/cc_disable_ec2_metadata.py delete mode 100644 cloudinit/config/cc_disk_setup.py delete mode 100644 cloudinit/config/cc_emit_upstart.py delete mode 100644 cloudinit/config/cc_fan.py delete mode 100644 cloudinit/config/cc_final_message.py delete mode 100644 cloudinit/config/cc_foo.py delete mode 100644 cloudinit/config/cc_growpart.py delete mode 100644 cloudinit/config/cc_grub_dpkg.py delete mode 100644 cloudinit/config/cc_keys_to_console.py delete mode 100644 cloudinit/config/cc_landscape.py delete mode 100644 cloudinit/config/cc_locale.py delete mode 100644 cloudinit/config/cc_lxd.py delete mode 100644 cloudinit/config/cc_mcollective.py delete mode 100644 cloudinit/config/cc_migrator.py delete mode 100644 cloudinit/config/cc_mounts.py delete mode 100644 cloudinit/config/cc_package_update_upgrade_install.py delete mode 100644 cloudinit/config/cc_phone_home.py delete mode 100644 cloudinit/config/cc_power_state_change.py delete mode 100644 cloudinit/config/cc_puppet.py delete mode 100644 cloudinit/config/cc_resizefs.py delete mode 100644 cloudinit/config/cc_resolv_conf.py delete mode 100644 cloudinit/config/cc_rh_subscription.py delete mode 100644 cloudinit/config/cc_rightscale_userdata.py delete mode 100644 cloudinit/config/cc_rsyslog.py delete mode 100644 cloudinit/config/cc_runcmd.py delete mode 100644 cloudinit/config/cc_salt_minion.py delete mode 100644 cloudinit/config/cc_scripts_per_boot.py delete mode 100644 cloudinit/config/cc_scripts_per_instance.py delete mode 100644 cloudinit/config/cc_scripts_per_once.py delete mode 100644 cloudinit/config/cc_scripts_user.py delete mode 100644 cloudinit/config/cc_scripts_vendor.py delete mode 100644 cloudinit/config/cc_seed_random.py delete mode 100644 cloudinit/config/cc_set_hostname.py delete mode 100644 cloudinit/config/cc_set_passwords.py delete mode 100644 cloudinit/config/cc_snappy.py delete mode 100644 cloudinit/config/cc_ssh.py delete mode 100644 cloudinit/config/cc_ssh_authkey_fingerprints.py delete mode 100644 cloudinit/config/cc_ssh_import_id.py delete mode 100644 cloudinit/config/cc_timezone.py delete mode 100644 cloudinit/config/cc_ubuntu_init_switch.py delete mode 100644 cloudinit/config/cc_update_etc_hosts.py delete mode 100644 cloudinit/config/cc_update_hostname.py delete mode 100644 cloudinit/config/cc_users_groups.py delete mode 100644 cloudinit/config/cc_write_files.py delete mode 100644 cloudinit/config/cc_yum_add_repo.py delete mode 100644 cloudinit/cs_utils.py delete mode 100644 cloudinit/distros/__init__.py delete mode 100644 cloudinit/distros/arch.py delete mode 100644 cloudinit/distros/debian.py delete mode 100644 cloudinit/distros/fedora.py delete mode 100644 cloudinit/distros/freebsd.py delete mode 100644 cloudinit/distros/gentoo.py delete mode 100644 cloudinit/distros/net_util.py delete mode 100644 cloudinit/distros/parsers/__init__.py delete mode 100644 cloudinit/distros/parsers/hostname.py delete mode 100644 cloudinit/distros/parsers/hosts.py delete mode 100644 cloudinit/distros/parsers/resolv_conf.py delete mode 100644 cloudinit/distros/parsers/sys_conf.py delete mode 100644 cloudinit/distros/rhel.py delete mode 100644 cloudinit/distros/rhel_util.py delete mode 100644 cloudinit/distros/sles.py delete mode 100644 cloudinit/distros/ubuntu.py delete mode 100644 cloudinit/ec2_utils.py delete mode 100644 cloudinit/filters/__init__.py delete mode 100644 cloudinit/filters/launch_index.py delete mode 100644 cloudinit/gpg.py delete mode 100644 cloudinit/handlers/__init__.py delete mode 100644 cloudinit/handlers/boot_hook.py delete mode 100644 cloudinit/handlers/cloud_config.py delete mode 100644 cloudinit/handlers/shell_script.py delete mode 100644 cloudinit/handlers/upstart_job.py delete mode 100644 cloudinit/helpers.py delete mode 100644 cloudinit/importer.py delete mode 100644 cloudinit/log.py delete mode 100644 cloudinit/mergers/__init__.py delete mode 100644 cloudinit/mergers/m_dict.py delete mode 100644 cloudinit/mergers/m_list.py delete mode 100644 cloudinit/mergers/m_str.py delete mode 100644 cloudinit/net/__init__.py delete mode 100644 cloudinit/net/cmdline.py delete mode 100644 cloudinit/net/eni.py delete mode 100644 cloudinit/net/network_state.py delete mode 100644 cloudinit/net/renderer.py delete mode 100644 cloudinit/net/sysconfig.py delete mode 100644 cloudinit/net/udev.py delete mode 100644 cloudinit/netinfo.py delete mode 100644 cloudinit/patcher.py delete mode 100644 cloudinit/registry.py delete mode 100644 cloudinit/reporting/__init__.py delete mode 100644 cloudinit/reporting/events.py delete mode 100644 cloudinit/reporting/handlers.py delete mode 100644 cloudinit/safeyaml.py delete mode 100644 cloudinit/serial.py delete mode 100644 cloudinit/settings.py delete mode 100644 cloudinit/signal_handler.py delete mode 100644 cloudinit/sources/DataSourceAltCloud.py delete mode 100644 cloudinit/sources/DataSourceAzure.py delete mode 100644 cloudinit/sources/DataSourceBigstep.py delete mode 100644 cloudinit/sources/DataSourceCloudSigma.py delete mode 100644 cloudinit/sources/DataSourceCloudStack.py delete mode 100644 cloudinit/sources/DataSourceConfigDrive.py delete mode 100644 cloudinit/sources/DataSourceDigitalOcean.py delete mode 100644 cloudinit/sources/DataSourceEc2.py delete mode 100644 cloudinit/sources/DataSourceGCE.py delete mode 100644 cloudinit/sources/DataSourceMAAS.py delete mode 100644 cloudinit/sources/DataSourceNoCloud.py delete mode 100644 cloudinit/sources/DataSourceNone.py delete mode 100644 cloudinit/sources/DataSourceOVF.py delete mode 100644 cloudinit/sources/DataSourceOpenNebula.py delete mode 100644 cloudinit/sources/DataSourceOpenStack.py delete mode 100644 cloudinit/sources/DataSourceSmartOS.py delete mode 100644 cloudinit/sources/__init__.py delete mode 100644 cloudinit/sources/helpers/__init__.py delete mode 100644 cloudinit/sources/helpers/azure.py delete mode 100644 cloudinit/sources/helpers/openstack.py delete mode 100644 cloudinit/sources/helpers/vmware/__init__.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/__init__.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/boot_proto.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/config.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/config_file.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/config_namespace.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/config_nic.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/config_source.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_error.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_event.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_state.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_util.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/ipv4_mode.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/nic.py delete mode 100644 cloudinit/sources/helpers/vmware/imc/nic_base.py delete mode 100644 cloudinit/ssh_util.py delete mode 100644 cloudinit/stages.py delete mode 100644 cloudinit/templater.py delete mode 100644 cloudinit/type_utils.py delete mode 100644 cloudinit/url_helper.py delete mode 100644 cloudinit/user_data.py delete mode 100644 cloudinit/util.py delete mode 100644 cloudinit/version.py delete mode 100644 config/cloud.cfg delete mode 100644 config/cloud.cfg-freebsd delete mode 100644 config/cloud.cfg.d/05_logging.cfg delete mode 100644 config/cloud.cfg.d/README delete mode 100644 doc/README delete mode 100644 doc/examples/cloud-config-add-apt-repos.txt delete mode 100644 doc/examples/cloud-config-archive-launch-index.txt delete mode 100644 doc/examples/cloud-config-archive.txt delete mode 100644 doc/examples/cloud-config-boot-cmds.txt delete mode 100644 doc/examples/cloud-config-ca-certs.txt delete mode 100644 doc/examples/cloud-config-chef-oneiric.txt delete mode 100644 doc/examples/cloud-config-chef.txt delete mode 100644 doc/examples/cloud-config-datasources.txt delete mode 100644 doc/examples/cloud-config-disk-setup.txt delete mode 100644 doc/examples/cloud-config-final-message.txt delete mode 100644 doc/examples/cloud-config-gluster.txt delete mode 100644 doc/examples/cloud-config-growpart.txt delete mode 100644 doc/examples/cloud-config-install-packages.txt delete mode 100644 doc/examples/cloud-config-landscape.txt delete mode 100644 doc/examples/cloud-config-launch-index.txt delete mode 100644 doc/examples/cloud-config-lxd.txt delete mode 100644 doc/examples/cloud-config-mcollective.txt delete mode 100644 doc/examples/cloud-config-mount-points.txt delete mode 100644 doc/examples/cloud-config-phone-home.txt delete mode 100644 doc/examples/cloud-config-power-state.txt delete mode 100644 doc/examples/cloud-config-puppet.txt delete mode 100644 doc/examples/cloud-config-reporting.txt delete mode 100644 doc/examples/cloud-config-resolv-conf.txt delete mode 100644 doc/examples/cloud-config-rh_subscription.txt delete mode 100644 doc/examples/cloud-config-rsyslog.txt delete mode 100644 doc/examples/cloud-config-run-cmds.txt delete mode 100644 doc/examples/cloud-config-salt-minion.txt delete mode 100644 doc/examples/cloud-config-seed-random.txt delete mode 100644 doc/examples/cloud-config-ssh-keys.txt delete mode 100644 doc/examples/cloud-config-update-apt.txt delete mode 100644 doc/examples/cloud-config-update-packages.txt delete mode 100644 doc/examples/cloud-config-user-groups.txt delete mode 100644 doc/examples/cloud-config-vendor-data.txt delete mode 100644 doc/examples/cloud-config-write-files.txt delete mode 100644 doc/examples/cloud-config-yum-repo.txt delete mode 100644 doc/examples/cloud-config.txt delete mode 100644 doc/examples/include-once.txt delete mode 100644 doc/examples/include.txt delete mode 100644 doc/examples/kernel-cmdline.txt delete mode 100644 doc/examples/part-handler-v2.txt delete mode 100644 doc/examples/part-handler.txt delete mode 100644 doc/examples/plain-ignored.txt delete mode 100644 doc/examples/seed/README delete mode 100644 doc/examples/seed/meta-data delete mode 100644 doc/examples/seed/user-data delete mode 100644 doc/examples/upstart-cloud-config.txt delete mode 100644 doc/examples/upstart-rclocal.txt delete mode 100644 doc/examples/user-script.txt delete mode 100644 doc/merging.rst delete mode 100644 doc/rtd/conf.py delete mode 100644 doc/rtd/index.rst delete mode 100644 doc/rtd/static/logo.png delete mode 100644 doc/rtd/static/logo.svg delete mode 100644 doc/rtd/topics/availability.rst delete mode 100644 doc/rtd/topics/capabilities.rst delete mode 100644 doc/rtd/topics/datasources.rst delete mode 100644 doc/rtd/topics/dir_layout.rst delete mode 100644 doc/rtd/topics/examples.rst delete mode 100644 doc/rtd/topics/format.rst delete mode 100644 doc/rtd/topics/hacking.rst delete mode 100644 doc/rtd/topics/merging.rst delete mode 100644 doc/rtd/topics/modules.rst delete mode 100644 doc/rtd/topics/moreinfo.rst delete mode 100644 doc/sources/altcloud/README.rst delete mode 100644 doc/sources/azure/README.rst delete mode 100644 doc/sources/cloudsigma/README.rst delete mode 100644 doc/sources/cloudstack/README.rst delete mode 100644 doc/sources/configdrive/README.rst delete mode 100644 doc/sources/digitalocean/README.rst delete mode 100644 doc/sources/kernel-cmdline.txt delete mode 100644 doc/sources/nocloud/README.rst delete mode 100644 doc/sources/opennebula/README.rst delete mode 100644 doc/sources/openstack/README.rst delete mode 100644 doc/sources/ovf/README delete mode 100644 doc/sources/ovf/example/ovf-env.xml delete mode 100644 doc/sources/ovf/example/ubuntu-server.ovf delete mode 100755 doc/sources/ovf/make-iso delete mode 100644 doc/sources/ovf/ovf-env.xml.tmpl delete mode 100644 doc/sources/ovf/ovfdemo.pem delete mode 100644 doc/sources/ovf/user-data delete mode 100644 doc/sources/smartos/README.rst delete mode 100644 doc/status.txt delete mode 100644 doc/userdata.txt delete mode 100644 doc/var-lib-cloud.txt delete mode 100644 doc/vendordata.txt delete mode 100755 packages/bddeb delete mode 100755 packages/brpm delete mode 100644 packages/debian/changelog.in delete mode 100644 packages/debian/cloud-init.postinst delete mode 100644 packages/debian/cloud-init.preinst delete mode 100644 packages/debian/compat delete mode 100644 packages/debian/control.in delete mode 100644 packages/debian/copyright delete mode 100644 packages/debian/dirs delete mode 100755 packages/debian/rules.in delete mode 100644 packages/debian/watch delete mode 100644 packages/redhat/cloud-init.spec.in delete mode 100644 packages/suse/cloud-init.spec.in delete mode 100644 requirements.txt delete mode 100755 setup.py delete mode 100644 systemd/cloud-config.service delete mode 100644 systemd/cloud-config.target delete mode 100644 systemd/cloud-final.service delete mode 100755 systemd/cloud-init-generator delete mode 100644 systemd/cloud-init-local.service delete mode 100644 systemd/cloud-init.service delete mode 100644 systemd/cloud-init.target delete mode 100644 sysvinit/debian/cloud-config delete mode 100644 sysvinit/debian/cloud-final delete mode 100755 sysvinit/debian/cloud-init delete mode 100644 sysvinit/debian/cloud-init-local delete mode 100755 sysvinit/freebsd/cloudconfig delete mode 100755 sysvinit/freebsd/cloudfinal delete mode 100755 sysvinit/freebsd/cloudinit delete mode 100755 sysvinit/freebsd/cloudinitlocal delete mode 100644 sysvinit/gentoo/cloud-config delete mode 100644 sysvinit/gentoo/cloud-final delete mode 100644 sysvinit/gentoo/cloud-init delete mode 100644 sysvinit/gentoo/cloud-init-local delete mode 100755 sysvinit/redhat/cloud-config delete mode 100755 sysvinit/redhat/cloud-final delete mode 100755 sysvinit/redhat/cloud-init delete mode 100755 sysvinit/redhat/cloud-init-local delete mode 100644 templates/chef_client.rb.tmpl delete mode 100644 templates/hosts.debian.tmpl delete mode 100644 templates/hosts.freebsd.tmpl delete mode 100644 templates/hosts.redhat.tmpl delete mode 100644 templates/hosts.suse.tmpl delete mode 100644 templates/resolv.conf.tmpl delete mode 100644 templates/sources.list.debian.tmpl delete mode 100644 templates/sources.list.ubuntu.tmpl delete mode 100644 test-requirements.txt delete mode 100644 tests/__init__.py delete mode 100644 tests/configs/sample1.yaml delete mode 100644 tests/data/filter_cloud_multipart.yaml delete mode 100644 tests/data/filter_cloud_multipart_1.email delete mode 100644 tests/data/filter_cloud_multipart_2.email delete mode 100644 tests/data/filter_cloud_multipart_header.email delete mode 100644 tests/data/merge_sources/expected1.yaml delete mode 100644 tests/data/merge_sources/expected10.yaml delete mode 100644 tests/data/merge_sources/expected11.yaml delete mode 100644 tests/data/merge_sources/expected12.yaml delete mode 100644 tests/data/merge_sources/expected2.yaml delete mode 100644 tests/data/merge_sources/expected3.yaml delete mode 100644 tests/data/merge_sources/expected4.yaml delete mode 100644 tests/data/merge_sources/expected5.yaml delete mode 100644 tests/data/merge_sources/expected6.yaml delete mode 100644 tests/data/merge_sources/expected7.yaml delete mode 100644 tests/data/merge_sources/expected8.yaml delete mode 100644 tests/data/merge_sources/expected9.yaml delete mode 100644 tests/data/merge_sources/source1-1.yaml delete mode 100644 tests/data/merge_sources/source1-2.yaml delete mode 100644 tests/data/merge_sources/source10-1.yaml delete mode 100644 tests/data/merge_sources/source10-2.yaml delete mode 100644 tests/data/merge_sources/source11-1.yaml delete mode 100644 tests/data/merge_sources/source11-2.yaml delete mode 100644 tests/data/merge_sources/source11-3.yaml delete mode 100644 tests/data/merge_sources/source12-1.yaml delete mode 100644 tests/data/merge_sources/source12-2.yaml delete mode 100644 tests/data/merge_sources/source2-1.yaml delete mode 100644 tests/data/merge_sources/source2-2.yaml delete mode 100644 tests/data/merge_sources/source3-1.yaml delete mode 100644 tests/data/merge_sources/source3-2.yaml delete mode 100644 tests/data/merge_sources/source4-1.yaml delete mode 100644 tests/data/merge_sources/source4-2.yaml delete mode 100644 tests/data/merge_sources/source5-1.yaml delete mode 100644 tests/data/merge_sources/source5-2.yaml delete mode 100644 tests/data/merge_sources/source6-1.yaml delete mode 100644 tests/data/merge_sources/source6-2.yaml delete mode 100644 tests/data/merge_sources/source7-1.yaml delete mode 100644 tests/data/merge_sources/source7-2.yaml delete mode 100644 tests/data/merge_sources/source8-1.yaml delete mode 100644 tests/data/merge_sources/source8-2.yaml delete mode 100644 tests/data/merge_sources/source9-1.yaml delete mode 100644 tests/data/merge_sources/source9-2.yaml delete mode 100644 tests/data/mountinfo_precise_ext4.txt delete mode 100644 tests/data/mountinfo_raring_btrfs.txt delete mode 100644 tests/data/roots/simple_ubuntu/etc/networks/interfaces delete mode 100644 tests/data/user_data.1.txt delete mode 100644 tests/data/vmware/cust-dhcp-2nic.cfg delete mode 100644 tests/data/vmware/cust-static-2nic.cfg delete mode 100644 tests/unittests/__init__.py delete mode 100644 tests/unittests/helpers.py delete mode 100644 tests/unittests/test__init__.py delete mode 100644 tests/unittests/test_builtin_handlers.py delete mode 100644 tests/unittests/test_cli.py delete mode 100644 tests/unittests/test_cs_util.py delete mode 100644 tests/unittests/test_data.py delete mode 100644 tests/unittests/test_datasource/__init__.py delete mode 100644 tests/unittests/test_datasource/test_altcloud.py delete mode 100644 tests/unittests/test_datasource/test_azure.py delete mode 100644 tests/unittests/test_datasource/test_azure_helper.py delete mode 100644 tests/unittests/test_datasource/test_cloudsigma.py delete mode 100644 tests/unittests/test_datasource/test_cloudstack.py delete mode 100644 tests/unittests/test_datasource/test_configdrive.py delete mode 100644 tests/unittests/test_datasource/test_digitalocean.py delete mode 100644 tests/unittests/test_datasource/test_gce.py delete mode 100644 tests/unittests/test_datasource/test_maas.py delete mode 100644 tests/unittests/test_datasource/test_nocloud.py delete mode 100644 tests/unittests/test_datasource/test_opennebula.py delete mode 100644 tests/unittests/test_datasource/test_openstack.py delete mode 100644 tests/unittests/test_datasource/test_smartos.py delete mode 100644 tests/unittests/test_distros/__init__.py delete mode 100644 tests/unittests/test_distros/test_generic.py delete mode 100644 tests/unittests/test_distros/test_hostname.py delete mode 100644 tests/unittests/test_distros/test_hosts.py delete mode 100644 tests/unittests/test_distros/test_netconfig.py delete mode 100644 tests/unittests/test_distros/test_resolv.py delete mode 100644 tests/unittests/test_distros/test_sysconfig.py delete mode 100644 tests/unittests/test_distros/test_user_data_normalize.py delete mode 100644 tests/unittests/test_ec2_util.py delete mode 100644 tests/unittests/test_filters/__init__.py delete mode 100644 tests/unittests/test_filters/test_launch_index.py delete mode 100644 tests/unittests/test_handler/__init__.py delete mode 100644 tests/unittests/test_handler/test_handler_apt_configure.py delete mode 100644 tests/unittests/test_handler/test_handler_apt_configure_sources_list.py delete mode 100644 tests/unittests/test_handler/test_handler_apt_source.py delete mode 100644 tests/unittests/test_handler/test_handler_ca_certs.py delete mode 100644 tests/unittests/test_handler/test_handler_chef.py delete mode 100644 tests/unittests/test_handler/test_handler_debug.py delete mode 100644 tests/unittests/test_handler/test_handler_disk_setup.py delete mode 100644 tests/unittests/test_handler/test_handler_growpart.py delete mode 100644 tests/unittests/test_handler/test_handler_locale.py delete mode 100644 tests/unittests/test_handler/test_handler_lxd.py delete mode 100644 tests/unittests/test_handler/test_handler_mcollective.py delete mode 100644 tests/unittests/test_handler/test_handler_mounts.py delete mode 100644 tests/unittests/test_handler/test_handler_power_state.py delete mode 100644 tests/unittests/test_handler/test_handler_rsyslog.py delete mode 100644 tests/unittests/test_handler/test_handler_seed_random.py delete mode 100644 tests/unittests/test_handler/test_handler_set_hostname.py delete mode 100644 tests/unittests/test_handler/test_handler_snappy.py delete mode 100644 tests/unittests/test_handler/test_handler_timezone.py delete mode 100644 tests/unittests/test_handler/test_handler_write_files.py delete mode 100644 tests/unittests/test_handler/test_handler_yum_add_repo.py delete mode 100644 tests/unittests/test_helpers.py delete mode 100644 tests/unittests/test_merging.py delete mode 100644 tests/unittests/test_net.py delete mode 100644 tests/unittests/test_pathprefix2dict.py delete mode 100644 tests/unittests/test_registry.py delete mode 100644 tests/unittests/test_reporting.py delete mode 100644 tests/unittests/test_rh_subscription.py delete mode 100644 tests/unittests/test_runs/__init__.py delete mode 100644 tests/unittests/test_runs/test_merge_run.py delete mode 100644 tests/unittests/test_runs/test_simple_run.py delete mode 100644 tests/unittests/test_sshutil.py delete mode 100644 tests/unittests/test_templating.py delete mode 100644 tests/unittests/test_util.py delete mode 100644 tests/unittests/test_vmware_config_file.py delete mode 100644 tools/21-cloudinit.conf delete mode 100755 tools/Z99-cloud-locale-test.sh delete mode 100755 tools/build-on-freebsd delete mode 100755 tools/ccfg-merge-debug delete mode 100755 tools/cloud-init-per delete mode 100755 tools/hacking.py delete mode 100755 tools/make-dist-tarball delete mode 100755 tools/make-mime.py delete mode 100755 tools/make-tarball delete mode 100755 tools/mock-meta.py delete mode 100755 tools/motd-hook delete mode 100755 tools/read-dependencies delete mode 100755 tools/read-version delete mode 100755 tools/run-pep8 delete mode 100755 tools/run-pyflakes delete mode 100755 tools/run-pyflakes3 delete mode 100755 tools/tox-venv delete mode 100755 tools/uncloud-init delete mode 100755 tools/validate-yaml.py delete mode 100755 tools/write-ssh-key-fingerprints delete mode 100644 tox.ini delete mode 100644 udev/66-azure-ephemeral.rules delete mode 100644 upstart/cloud-config.conf delete mode 100644 upstart/cloud-final.conf delete mode 100644 upstart/cloud-init-blocknet.conf delete mode 100644 upstart/cloud-init-container.conf delete mode 100644 upstart/cloud-init-local.conf delete mode 100644 upstart/cloud-init-nonet.conf delete mode 100644 upstart/cloud-init.conf delete mode 100644 upstart/cloud-log-shutdown.conf diff --git a/.bzrignore b/.bzrignore deleted file mode 100644 index 926e4581..00000000 --- a/.bzrignore +++ /dev/null @@ -1,4 +0,0 @@ -.tox -dist -cloud_init.egg-info -__pycache__ diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index bae982e3..00000000 --- a/ChangeLog +++ /dev/null @@ -1,776 +0,0 @@ -0.7.7: - - open 0.7.7 - - Digital Ocean: add datasource for Digital Ocean. [Neal Shrader] - - expose uses_systemd as a distro function (fix rhel7) - - fix broken 'output' config (LP: #1387340) - - begin adding cloud config module docs to config modules (LP: #1383510) - - retain trailing eol from template files (sources.list) when - rendered with jinja (LP: #1355343) - - Only use datafiles and initsys addon outside virtualenvs - - Fix the digital ocean test case on python 2.6 - - Increase the usefulness, robustness, configurability of the chef module - so that it is more useful, more documented and better for users - - Fix how '=' signs are not handled that well in ssh_utils (LP: #1391303) - - Be more tolerant of ssh keys passed into 'ssh_authorized_keys'; allowing - for list, tuple, set, dict, string types and warning on other unexpected - types - - Update to use newer/better OMNIBUS_URL for chef module - - GCE: Allow base64 encoded user-data (LP: #1404311) [Wayne Witzell III] - - GCE: use short hostname rather than fqdn (LP: #1383794) [Ben Howard] - - systemd: make init stage run before login prompts shown [Steve Langasek] - - hostname: on first boot apply hostname to be same as is written for - persistent hostname. (LP: #1246485) - - remove usage of dmidecode on linux in favor of /sys interface [Ben Howard] - - python3 support [Barry Warsaw, Daniel Watkins, Josh Harlow] (LP: #1247132) - - support managing gpt partitions in disk config [Daniel Watkins] - - Azure: utilze gpt support for ephemeral formating [Daniel Watkins] - - CloudStack: support fetching password from virtual router [Daniel Watkins] - (LP: #1422388) - - readurl, read_file_or_url returns bytes, user must convert as necessary - - SmartOS: use v2 metadata service (LP: #1436417) [Daniel Watkins] - - NoCloud: fix local datasource claiming found without explicit dsmode - - Snappy: add support for installing snappy packages and configuring. - - systemd: use network-online instead of network.target (LP: #1440180) - [Steve Langasek] - - Add functionality to fixate the uid of a newly added user. - - Don't overwrite the hostname if the user has changed it after we set it. - - GCE datasource does not handle instance ssh keys (LP: 1403617) - - sysvinit: make cloud-init-local run before network (LP: #1275098) - [Surojit Pathak] - - Azure: do not re-set hostname if user has changed it (LP: #1375252) - - Fix exception when running with no arguments on Python 3. [Daniel Watkins] - - Centos: detect/expect use of systemd on centos 7. [Brian Rak] - - Azure: remove dependency on walinux-agent [Daniel Watkins] - - EC2: know about eu-central-1 availability-zone (LP: #1456684) - - Azure: remove password from on-disk ovf-env.xml (LP: #1443311) [Ben Howard] - - Doc: include information on user-data in OpenStack [Daniel Watkins] - - Systemd: check for systemd using sd_booted symantics (LP: #1461201) - [Lars Kellogg-Stedman] - - Add an rh_subscription module to handle registration of Red Hat instances. - [Brent Baude] - - cc_apt_configure: fix importing keys under python3 (LP: #1463373) - - cc_growpart: fix specification of 'devices' list (LP: #1465436) - - CloudStack: fix password setting on cloudstack > 4.5.1 (LP: #1464253) - - GCE: fix determination of availability zone (LP: #1470880) - - ssh: generate ed25519 host keys (LP: #1461242) - - distro mirrors: provide datasource to mirror selection code to support - GCE regional mirrors. (LP: #1470890) - - add udev rules that identify ephemeral device on Azure (LP: #1411582) - - _read_dmi_syspath: fix bad log message causing unintended exception - - rsyslog: add additional configuration mode (LP: #1478103) - - status_wrapper in main: fix use of print_exc when handling exception - - reporting: add reporting module for web hook or logging of events. - - NoCloud: fix consumption of vendordata (LP: #1493453) - - power_state_change: support 'condition' to disable or enable poweroff - - ubuntu fan: support for config and installing of ubuntu fan (LP: #1504604) - - Azure: support extracting SSH key values from ovf-env.xml (LP: #1506244) - - AltCloud: fix call to udevadm settle (LP: #1507526) - - Ubuntu templates: modify sources.list template to provide same sources - as install from server or desktop ISO. (LP: #1177432) - - cc_mounts: use 'nofail' if system uses systemd. (LP: #1514485) - - Azure: get instance id from dmi instead of SharedConfig (LP: #1506187) - - systemd/power_state: fix power_state to work even if cloud-final - exited non-zero (LP: #1449318) - - SmartOS: Add support for Joyent LX-Brand Zones (LP: #1540965) - [Robert C Jennings] - - systemd: support using systemd-detect-virt to detect container - (LP: #1539016) [Martin Pitt] - - docs: fix lock_passwd documentation [Robert C Jennings] - - Azure: Handle escaped quotes in WALinuxAgentShim.find_endpoint. - (LP: #1488891) [Dan Watkins] - - lxd: add support for setting up lxd using 'lxd init' (LP: #1522879) - - Add Image Customization Parser for VMware vSphere Hypervisor - Support. [Sankar Tanguturi] - - timezone: use a symlink rather than copy for /etc/localtime - unless it is already a file (LP: #1543025). - - Enable password changing via a hashed string [Alex Sirbu] - - Added BigStep datasource [Alex Sirbu] - - No longer run pollinate in seed_random (LP: #1554152) - - groups: add defalt user to 'lxd' group. Create groups listed - for a user if they do not exist. (LP: #1539317) - - dmi data: fix failure of reading dmi data for unset dmi values - - doc: mention label for nocloud datasource must be 'cidata' [Peter Hurley] - - ssh_pwauth: fix module to support 'unchanged' and match behavior - described in documentation [Chris Cosby] - - quickly check to see if the previous instance id is still valid to - avoid dependency on network metadata service on every boot (LP: #1553815) - - support network configuration in cloud-init --local with support - device naming via systemd.link. - - FreeBSD: add support for installing packages, setting password and - timezone. Change default user to 'freebsd'. [Ben Arblaster] - - locale: list unsupported environment settings in warning (LP: #1558069) - - disk_setup: correctly send --force to mkfs on block devices (LP: #1548772) - - chef: fix chef install from gems (LP: #1553345) - - systemd: do not specify After of obsolete syslog.target (LP: #1536964) - - centos: Ensure that resolve conf object is written as a str (LP: #1479988) - - chef: straighten out validation_cert and validation_key (LP: #1568940) - - phone_home: allow usage of fqdn (LP: #1566824) [Ollie Armstrong] - - cloudstack: Only use DHCPv4 lease files as a datasource (LP: #1576273) - [Wido den Hollander] - - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) - [Robert Jennings] - - Ec2: do not retry requests for user-data path on 404. - - settings on the kernel command line (cc:) override all local settings - rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) - - Improve merging documentation [Daniel Watkins] - - apt sources: support inserting key/key-id only, custom sources.list, - long gpg key fingerprints with spaces, and dictionary format (LP: #1574113) - - SmartOS: datasource improvements and support for metadata service - providing networking information. - - Datasources: centrally handle 'dsmode' and no longer require datasources - to "pass" if modules_init should be executed with network access. - - ConfigDrive: improved support for networking information from - a network_data.json or older interfaces formated network_config. - - Change missing Cheetah log warning to debug [Andrew Jorgensen] - - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] - - support network rendering to sysconfig (for centos and RHEL) - - write_files: if no permissions are given, just use default without warn. - - user_data: fix error when user-data is not utf-8 decodable (LP: #1532072) - - fix mcollective module with python3 (LP: #1597699) [Sergii Golovatiuk] - -0.7.6: - - open 0.7.6 - - Enable vendordata on CloudSigma datasource (LP: #1303986) - - Poll on /dev/ttyS1 in CloudSigma datasource only if dmidecode says - we're running on cloudsigma (LP: #1316475) [Kiril Vladimiroff] - - SmartOS test: do not require existance of /dev/ttyS1. [LP: #1316597] - - doc: fix user-groups doc to reference plural ssh-authorized-keys - (LP: #1327065) [Joern Heissler] - - fix 'make test' in python 2.6 - - support jinja2 as a templating engine. Drop the hard requirement on - cheetah. This helps in python3 effort. (LP: #1219223) - - change install path for systemd files to /lib/systemd/system - [Dimitri John Ledkov] - - change trunk debian packaging to use pybuild and drop cdbs. - [Dimitri John Ledkov] - - SeLinuxGuard: remove invalid check that looked for stat.st_mode in os.lstat. - - do not write comments in /etc/timezone (LP: #1341710) - - ubuntu: provide 'ubuntu-init-switch' module to aid in systemd testing. - - status/result json: remove 'end' entry which was always null - - systemd: make cloud-init block ssh service startup to guarantee keys - are generated. [Jordan Evans] (LP: #1333920) - - default settings: fix typo resulting in OpenStack and GCE not working - unless config explicitly provided (LP: #1329583) [Garrett Holmstrom]) - - fix rendering resolv.conf if no 'options' are provided (LP: #1328953) - - docs: fix disk-setup to reference 'table_type' [Rail Aliiev] (LP: #1313114) - - ssh_authkey_fingerprints: fix bug that prevented disabling the module. - (LP: #1340903) [Patrick Lucas] - - no longer use pylint as a checker, fix pep8 [Jay Faulkner]. - - Openstack: do not load some urls twice. - - FreeBsd: fix initscripts and add working config file [Harm Weites] - - Datasource: fix broken logic to provide hostname if datasource does not - provide one - - Improved and less verbose logging. - - resizefs: first check that device is writable. - - configdrive: fix reading of vendor data to be like metadata service reader. - [Jay Faulkner] - - resizefs: fix broken background resizing [Jay Faulkner] (LP: #1338614) - - cc_grub_dpkg: fix EC2 hvm instances to avoid prompt on grub update. - (LP: #1336855) - - FreeBsd: support config drive datasource [Joseph bajin] - - cc_mounts: support creating a swap file - - DigitalOcean & GCE: fix get_hostname consistency -0.7.5: - - open 0.7.5 - - Add a debug log message around import failures - - add a 'debug' module for easily printing out some information about - datasource and cloud-init [Shraddha Pandhe] - - support running apt with 'eatmydata' via configuration token - apt_get_wrapper (LP: #1236531). - - convert paths provided in config-drive 'files' to string before writing - (LP: #1260072). - - Azure: minor changes in logging output. ensure filenames are strings (not - unicode). - - config/cloud.cfg.d/05_logging.cfg: provide a default 'output' setting, to - redirect cloud-init stderr and stdout /var/log/cloud-init-output.log. - - drop support for resizing partitions with parted entirely (LP: #1212492). - This was broken as it was anyway. - - add support for vendordata in SmartOS and NoCloud datasources. - - drop dependency on boto for crawling ec2 metadata service. - - add 'Requires' on sudo (for OpenNebula datasource) in rpm specs, and - 'Recommends' in the debian/control.in [Vlastimil Holer] - - if mount_info reports /dev/root is a device path for /, then convert - that to a device via help of kernel cmdline. - - configdrive: consider partitions as possible datasources if they have - theh correct filesystem label. [Paul Querna] - - initial freebsd support [Harm Weites] - - fix in is_ipv4 to accept IP addresses with a '0' in them. - - Azure: fix issue when stale data in /var/lib/waagent (LP: #1269626) - - skip config_modules that declare themselves only verified on a set of - distros. Add them to 'unverified_modules' list to run anyway. - - Add CloudSigma datasource [Kiril Vladimiroff] - - Add initial support for Gentoo and Arch distributions [Nate House] - - Add GCE datasource [Vaidas Jablonskis] - - Add native Openstack datasource which reads openstack metadata - rather than relying on EC2 data in openstack metadata service. - - SmartOS, AltCloud: disable running on arm systems due to bug - (LP: #1243287, #1285686) [Oleg Strikov] - - Allow running a command to seed random, default is 'pollinate -q' - (LP: #1286316) [Dustin Kirkland] - - Write status to /run/cloud-init/status.json for consumption by - other programs (LP: #1284439) - - Azure: if a reboot causes ephemeral storage to be re-provisioned - Then we need to re-format it. (LP: #1292648) - - OpenNebula: support base64 encoded user-data - [Enol Fernandez, Peter Kotcauer] -0.7.4: - - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a - partitioned block device with target filesystem on ephemeral0.1. - (LP: #1236594) - - fix DataSourceAzure incompatibility with 2.6 (LP: #1232175) - - fix power_state_change config module so that example works. Improve - its documentation and add reference to 'timeout' - - support apt-add-archive with 'cloud-archive:' format. (LP: #1244355) - - Change SmartOS verb for availability zone (LP: #1249124) - - documentation fix for boothooks to use 'cloud-init-per' - - fix resizefs module by supporting kernels that do not have - /proc/PID/mountinfo. (LP: #1248625) [Tim Daly Jr.] - - fix 'make rpm' by removing 0.6.4 entry from ChangeLog (LP: #1241834) -0.7.3: - - fix omnibus chef installer (LP: #1182265) [Chris Wing] - - small fix for OVF datasource for iso transport on non-iso9660 filesystem - - determine if upstart version is suitable for - 'initctl reload-configuration' (LP: #1124384). If so, then invoke it. - supports setting up instance-store disk with partition table and filesystem. - - add Azure datasource. - - add support for SuSE / SLES [Juerg Haefliger] - - add a trailing carriage return to chpasswd input, which reportedly - caused a problem on rhel5 if missing. - - support individual MIME segments to be gzip compressed (LP: #1203203) - - always finalize handlers even if processing failed (LP: #1203368) - - support merging into cloud-config via jsonp. (LP: #1200476) - - add datasource 'SmartOS' for Joyent Cloud. Adds a dependency on serial. - - add 'log_time' helper to util for timing how long things take - which also reads from uptime. uptime is useful as clock may change during - boot due to ntp. - - prefer growpart resizer to 'parted resizepart' (LP: #1212492) - - support random data seed from config drive or azure, and a module - 'seed_random' to read that and write it to /dev/urandom. - - add OpenNebula Datasource [Vlastimil Holer] - - add 'cc_disk_setup' config module for paritioning disks and creating - filesystems. Useful if attached disks are not formatted (LP: #1218506) - - Fix usage of libselinux-python when selinux is disabled. [Garrett Holmstrom] - - multi_log: only write to /dev/console if it exists [Garrett Holmstrom] - - config/cloud.cfg: add 'sudo' to list groups for the default user - (LP: #1228228) - - documentation fix for use of 'mkpasswd' [Eric Nordlund] - - respect /etc/growroot-disabled file (LP: #1234331) -0.7.2: - - add a debian watch file - - add 'sudo' entry to ubuntu's default user (LP: #1080717) - - fix resizefs module when 'noblock' was provided (LP: #1080985) - - make sure there is no blank line before cloud-init entry in - there are no blank lines in /etc/ca-certificates.conf (LP: #1077020) - - fix sudoers writing when entry is a string (LP: #1079002) - - tools/write-ssh-key-fingerprints: use '-s' rather than '--stderr' - option (LP: #1083715) - - make install of puppet configurable (LP: #1090205) [Craig Tracey] - - support omnibus installer for chef [Anatoliy Dobrosynets] - - fix bug where cloud-config in user-data could not modify system_info - settings (LP: #1090482) - - fix CloudStack DataSource to use Virtual Router as described by - CloudStack documentation if it is available by searching through dhclient - lease files. If it is not available, then fall back to the default - gateway. (LP: #1089989) - - fix redaction of password field in log (LP: #1096417) - - fix to cloud-config user setup. Previously, lock_passwd was broken and - all accounts would be locked unless 'system' was given (LP: #1096423). - - Allow 'sr0' (or sr[0-9]) to be specified without /dev/ as a source for - mounts. [Vlastimil Holer] - - allow config-drive-data to come from a CD device by more correctly - filtering out partitions. (LP: #1100545) - - setup docs to be available on read-the-docs - https://cloudinit.readthedocs.org/en/latest/ (LP: #1093039) - - add HACKING file for information on contributing - - handle the legacy 'user:' configuration better, making it affect the - configured OS default user (LP: #1100920) - - Adding a resolv.conf configuration module (LP: #1100434). Currently only - working on redhat systems (no support for resolvconf) - - support grouping linux distros into "os_families". This allows a module - to operate on the family (redhat or debian) rather than the distro (ubuntu, - debian, fedora, rhel) (LP: #1100029) - - fix /etc/hosts writing when templates are used (LP: #1100036) - - add package versioning logic to package installation - functionality (LP: #1108047) - - fix documentation for write_files to correctly list 'permissions' - rather than 'perms' (LP: #1111205) - - cloud-init-container.conf: ensure /run/network before running ifquery - - DataSourceNoCloud: allow user-data and meta-data to be specified - in config (LP: #1115833). - - improve debian support in sysvinit scripts, package build scripts, and - split sources.list template to be distro specific. - - support for resizing btrfs root filesystems [Blair Zajac] - - fix issue when writing ssh keys to .ssh/authorized_keys (LP: #1136343) - - upstart: cloud-init-nonet.conf trap the TERM signal, so that dmesg or other - output does not get a 'killed by TERM signal' message. - - support resizing partitions via growpart or parted (LP: #1136936) - - allow specifying apt-get command in distro config ('apt_get_command') - - support different and user-suppliable merging algorithms for cloud-config - (LP: #1023179) - - use python-requests rather than urllib2. By using recent versions of - python-requests, we get https support (LP: #1067888). - - make apt-get invoke 'dist-upgrade' rather than 'upgrade' for - package_upgrade. (LP: #1164147) - - improvements for systemd with Fedora 18 - - workaround 2.6 kernel issue that stopped blkid from showing /dev/sr0 - - add new, backwards compatible merging syntax so merging of cloud-config - can be more useful. - -0.7.1: - - sysvinit: fix missing dependency in cloud-init job for RHEL 5.6 - - config-drive: map hostname to local-hostname (LP: #1061964) - - landscape: install landscape-client package if not installed. - only take action if cloud-config is present (LP: #1066115) - - cc_landscape: restart landscape after install or config (LP: #1070345) - - multipart/archive. do not fail on unknown headers in multipart - mime or cloud-archive config (LP: #1065116). - - tools/Z99-cloud-locale-test.sh: avoid warning when user's shell is - zsh (LP: #1073077) - - fix stack trace when unknown user-data input had unicode (LP: #1075756) - - split 'apt-update-upgrade' config module into 'apt-configure' and - 'package-update-upgrade-install'. The 'package-update-upgrade-install' - will be a cross distro module. - - Cleanups: - - Remove usage of paths.join, as all code should run through util helpers - - Fix pylint complaining about tests folder 'helpers.py' not being found - - Add a pylintrc file that is used instead options hidden in 'run_pylint' - - fix bug where cloud-config from user-data could not affect system_info - settings [revno 703] (LP: #1076811) - - for write fqdn to system config for rh/fedora [revno 704] - - add yaml/cloud config examples checking tool [revno 706] - - Fix the merging of group configuration when that group configuration is a - dict => members. [revno 707] - - add yum_add_repo configuration module for adding additional yum repos - - fix public key importing with config-drive-v2 datasource (LP: #1077700) - - handle renaming and fixing up of marker names (LP: 1075980) [revno 710] - this relieves that burden from the distro/packaging. - - group config: fix how group members weren't being translated correctly - when the group: [member, member...] format was used (LP: #1077245) - - sysconfig: fix how the /etc/sysconfig/network should be using the fully - qualified domain name instead of the partially qualified domain name - which is used in the ubuntu/debian case (LP: #1076759) - - fix how string escaping was not working when the string was a unicode - string which was causing the warning message not to be written - out (LP: #1075756) - - for boto > 0.6.0 there was a lazy load of the metadata added, when - cloud-init runs the usage of this lazy loading is hidden and since that lazy - loading will be performed on future attribute access we must traverse the - lazy loaded dictionary and force it to full expand so that if cloud-init - blocks the ec2 metadata port the lazy loaded dictionary will continue - working properly instead of trying to make additional url calls which will - fail (LP: #1068801) - - use a set of helper/parsing classes to perform system configuration - for easier test. (/etc/sysconfig, /etc/hostname, resolv.conf, /etc/hosts) - - add power_state_change config module for shutting down stystem after - cloud-init finishes. (LP: #1064665) -0.7.0: - - add a 'exception_cb' argument to 'wait_for_url'. If provided, this - method will be called back with the exception received and the message. - - utilize the 'exception_cb' above to modify the oauth timestamp in - DataSourceMAAS requests if a 401 or 403 is received. (LP: #978127) - - catch signals and exit rather than stack tracing - - if logging fails, enable a fallback logger by patching the logging module - - do not 'start networking' in cloud-init-nonet, but add - cloud-init-container job that runs only if in container and emits - net-device-added (LP: #1031065) - - search only top level dns for 'instance-data' in - DataSourceEc2 (LP: #1040200) - - add support for config-drive-v2 (LP:#1037567) - - support creating users, including the default user. - [Ben Howard] (LP: #1028503) - - add apt_reboot_if_required to reboot if an upgrade or package installation - forced the need for one (LP: #1038108) - - allow distro mirror selection to include availability-zone (LP: #1037727) - - allow arch specific mirror selection (select ports.ubuntu.com on arm) - LP: #1028501 - - allow specification of security mirrors (LP: #1006963) - - add the 'None' datasource (LP: #906669), which will allow jobs - to run even if there is no "real" datasource found. - - write ssh authorized keys to console, ssh_authkey_fingerprints - config module [Joshua Harlow] (LP: #1010582) - - Added RHEVm and vSphere support as source AltCloud [Joseph VLcek] - - add write-files module (LP: #1012854) - - Add setuptools + cheetah to debian package build dependencies (LP: #1022101) - - Adjust the sysvinit local script to provide 'cloud-init-local' and have - the cloud-config script depend on that as well. - - Add the 'bzr' name to all packages built - - Reduce logging levels for certain non-critical cases to DEBUG instead of the - previous level of WARNING - - unified binary that activates the various stages - - Now using argparse + subcommands to specify the various CLI options - - a stage module that clearly separates the stages of the different - components (also described how they are used and in what order in the - new unified binary) - - user_data is now a module that just does user data processing while the - actual activation and 'handling' of the processed user data is done via - a separate set of files (and modules) with the main 'init' stage being the - controller of this - - creation of boot_hook, cloud_config, shell_script, upstart_job version 2 - modules (with classes that perform there functionality) instead of those - having functionality that is attached to the cloudinit object (which - reduces reuse and limits future functionality, and makes testing harder) - - removal of global config that defined paths, shared config, now this is - via objects making unit testing testing and global side-effects a non issue - - creation of a 'helpers.py' - - this contains an abstraction for the 'lock' like objects that the various - module/handler running stages use to avoid re-running a given - module/handler for a given frequency. this makes it separated from - the actual usage of that object (thus helpful for testing and clear lines - usage and how the actual job is accomplished) - - a common 'runner' class is the main entrypoint using these locks to - run function objects passed in (along with there arguments) and there - frequency - - add in a 'paths' object that provides access to the previously global - and/or config based paths (thus providing a single entrypoint object/type - that provides path information) - - this also adds in the ability to change the path when constructing - that path 'object' and adding in additional config that can be used to - alter the root paths of 'joins' (useful for testing or possibly useful - in chroots?) - - config options now avaiable that can alter the 'write_root' and the - 'read_root' when backing code uses the paths join() function - - add a config parser subclass that will automatically add unknown sections - and return default values (instead of throwing exceptions for these cases) - - a new config merging class that will be the central object that knows - how to do the common configuration merging from the various configuration - sources. The order is the following: - - cli config files override environment config files - which override instance configs which override datasource - configs which override base configuration which overrides - default configuration. - - remove the passing around of the 'cloudinit' object as a 'cloud' variable - and instead pass around an 'interface' object that can be given to modules - and handlers as there cloud access layer while the backing of that - object can be varied (good for abstraction and testing) - - use a single set of functions to do importing of modules - - add a function in which will search for a given set of module names with - a given set of attributes and return those which are found - - refactor logging so that instead of using a single top level 'log' that - instead each component/module can use its own logger (if desired), this - should be backwards compatible with handlers and config modules that used - the passed in logger (its still passed in) - - ensure that all places where exception are caught and where applicable - that the util logexc() is called, so that no exceptions that may occur - are dropped without first being logged (where it makes sense for this - to happen) - - add a 'requires' file that lists cloud-init dependencies - - applying it in package creation (bdeb and brpm) as well as using it - in the modified setup.py to ensure dependencies are installed when - using that method of packaging - - add a 'version.py' that lists the active version (in code) so that code - inside cloud-init can report the version in messaging and other config files - - cleanup of subprocess usage so that all subprocess calls go through the - subp() utility method, which now has an exception type that will provide - detailed information on python 2.6 and 2.7 - - forced all code loading, moving, chmod, writing files and other system - level actions to go through standard set of util functions, this greatly - helps in debugging and determining exactly which system actions cloud-init - is performing - - adjust url fetching and url trying to go through a single function that - reads urls in the new 'url helper' file, this helps in tracing, debugging - and knowing which urls are being called and/or posted to from with-in - cloud-init code - - add in the sending of a 'User-Agent' header for all urls fetched that - do not provide there own header mapping, derive this user-agent from - the following template, 'Cloud-Init/{version}' where the version is the - cloud-init version number - - using prettytable for netinfo 'debug' printing since it provides a standard - and defined output that should be easier to parse than a custom format - - add a set of distro specific classes, that handle distro specific actions - that modules and or handler code can use as needed, this is organized into - a base abstract class with child classes that implement the shared - functionality. config determines exactly which subclass to load, so it can - be easily extended as needed. - - current functionality - - network interface config file writing - - hostname setting/updating - - locale/timezone/ setting - - updating of /etc/hosts (with templates or generically) - - package commands (ie installing, removing)/mirror finding - - interface up/down activating - - implemented a debian + ubuntu subclass - - implemented a redhat + fedora subclass - - adjust the root 'cloud.cfg' file to now have distrobution/path specific - configuration values in it. these special configs are merged as the normal - config is, but the system level config is not passed into modules/handlers - - modules/handlers must go through the path and distro object instead - - have the cloudstack datasource test the url before calling into boto to - avoid the long wait for boto to finish retrying and finally fail when - the gateway meta-data address is unavailable - - add a simple mock ec2 meta-data python based http server that can serve a - very simple set of ec2 meta-data back to callers - - useful for testing or for understanding what the ec2 meta-data - service can provide in terms of data or functionality - - for ssh key and authorized key file parsing add in classes and util - functions that maintain the state of individual lines, allowing for a - clearer separation of parsing and modification (useful for testing and - tracing) - - add a set of 'base' init.d scripts that can be used on systems that do - not have full upstart or systemd support (or support that does not match - the standard fedora/ubuntu implementation) - - currently these are being tested on RHEL 6.2 - - separate the datasources into there own subdirectory (instead of being - a top-level item), this matches how config 'modules' and user-data - 'handlers' are also in there own subdirectory (thus helping new developers - and others understand the code layout in a quicker manner) - - add the building of rpms based off a new cli tool and template 'spec' file - that will templatize and perform the necessary commands to create a source - and binary package to be used with a cloud-init install on a 'rpm' - supporting system - - uses the new standard set of requires and converts those pypi requirements - into a local set of package requirments (that are known to exist on RHEL - systems but should also exist on fedora systems) - - adjust the bdeb builder to be a python script (instead of a shell script) - and make its 'control' file a template that takes in the standard set of - pypi dependencies and uses a local mapping (known to work on ubuntu) to - create the packages set of dependencies (that should also work on - ubuntu-like systems) - - pythonify a large set of various pieces of code - - remove wrapping return statements with () when it has no effect - - upper case all constants used - - correctly 'case' class and method names (where applicable) - - use os.path.join (and similar commands) instead of custom path creation - - use 'is None' instead of the frowned upon '== None' which picks up a large - set of 'true' cases than is typically desired (ie for objects that have - there own equality) - - use context managers on locks, tempdir, chdir, file, selinux, umask, - unmounting commands so that these actions do not have to be closed and/or - cleaned up manually in finally blocks, which is typically not done and - will eventually be a bug in the future - - use the 'abc' module for abstract classes base where possible - - applied in the datasource root class, the distro root class, and the - user-data v2 root class - - when loading yaml, check that the 'root' type matches a predefined set of - valid types (typically just 'dict') and throw a type error if a mismatch - occurs, this seems to be a good idea to do when loading user config files - - when forking a long running task (ie resizing a filesytem) use a new util - function that will fork and then call a callback, instead of having to - implement all that code in a non-shared location (thus allowing it to be - used by others in the future) - - when writing out filenames, go through a util function that will attempt to - ensure that the given filename is 'filesystem' safe by replacing '/' with - '_' and removing characters which do not match a given whitelist of allowed - filename characters - - for the varying usages of the 'blkid' command make a function in the util - module that can be used as the single point of entry for interaction with - that command (and its results) instead of having X separate implementations - - place the rfc 8222 time formatting and uptime repeated pieces of code in the - util module as a set of function with the name 'time_rfc2822'/'uptime' - - separate the pylint+pep8 calling from one tool into two indivudal tools so - that they can be called independently, add make file sections that can be - used to call these independently - - remove the support for the old style config that was previously located in - '/etc/ec2-init/ec2-config.cfg', no longer supported! - - instead of using a altered config parser that added its own 'dummy' section - on in the 'mcollective' module, use configobj which handles the parsing of - config without sections better (and it also maintains comments instead of - removing them) - - use the new defaulting config parser (that will not raise errors on sections - that do not exist or return errors when values are fetched that do not - exist) in the 'puppet' module - - for config 'modules' add in the ability for the module to provide a list of - distro names which it is known to work with, if when ran and the distro - being used name does not match one of those in this list, a warning will be - written out saying that this module may not work correctly on this - distrobution - - for all dynamically imported modules ensure that they are fixed up before - they are used by ensuring that they have certain attributes, if they do not - have those attributes they will be set to a sensible set of defaults instead - - adjust all 'config' modules and handlers to use the adjusted util functions - and the new distro objects where applicable so that those pieces of code can - benefit from the unified and enhanced functionality being provided in that - util module - - fix a potential bug whereby when a #includeonce was encountered it would - enable checking of urls against a cache, if later a #include was encountered - it would continue checking against that cache, instead of refetching (which - would likely be the expected case) - - add a openstack/nova based pep8 extension utility ('hacking.py') that allows - for custom checks (along with the standard pep8 checks) to occur when - running 'make pep8' and its derivatives - - support relative path in AuthorizedKeysFile (LP: #970071). - - make apt-get update run with --quiet (suitable for logging) (LP: #1012613) - - cc_salt_minion: use package 'salt-minion' rather than 'salt' (LP: #996166) - - use yaml.safe_load rather than yaml.load (LP: #1015818) -0.6.3: - - add sample systemd config files [Garrett Holmstrom] - - add Fedora support [Garrent Holstrom] (LP: #883286) - - fix bug in netinfo.debug_info if no net devices available (LP: #883367) - - use python module hashlib rather than md5 to avoid deprecation warnings. - - support configuration of mirror based on dns name ubuntu-mirror in - local domain. - - support setting of Acquire::HTTP::Proxy via 'apt_proxy' - - DataSourceEc2: more resilliant to slow metadata service - - config change: 'retries' dropped, 'max_wait' added, timeout increased - - close stdin in all cloud-init programs that are launched at boot - (LP: #903993) - - revert management of /etc/hosts to 0.6.1 style (LP: #890501, LP: #871966) - - write full ssh keys to console for easy machine consumption (LP: #893400) - - put INSTANCE_ID environment variable in bootcmd scripts - - add 'cloud-init-per' script for easily running things with a given frequency - - replace cloud-init-run-module with cloud-init-per - - support configuration of landscape-client via cloud-config (LP: #857366) - - part-handlers now get base64 decoded content rather than 2xbase64 encoded - in the payload parameter. (LP: #874342) - - add test case framework [Mike Milner] (LP: #890851) - - fix pylint warnings [Juerg Haefliger] (LP: #914739) - - add support for adding and deleting CA Certificates [Mike Milner] - (LP: #915232) - - in ci-info lines, use '.' to indicate empty field for easier machine reading - - support empty lines in "#include" files (LP: #923043) - - support configuration of salt minions (Jeff Bauer) (LP: #927795) - - DataSourceOVF: only search for OVF data on ISO9660 filesystems (LP: #898373) - - DataSourceConfigDrive: support getting data from openstack config drive - (LP: #857378) - - DataSourceNoCloud: support seed from external disk of ISO or vfat - (LP: #857378) - - DataSourceNoCloud: support inserting /etc/network/interfaces - - DataSourceMaaS: add data source for Ubuntu Machines as a Service (MaaS) - (LP: #942061) - - DataSourceCloudStack: add support for CloudStack datasource [Cosmin Luta] - - add option 'apt_pipelining' to address issue with S3 mirrors - (LP: #948461) [Ben Howard] - - warn on non-multipart, non-handled user-data [Martin Packman] - - run resizefs in the background in order to not block boot (LP: #961226) - - Fix bug in Chef support where validation_key was present in config, but - 'validation_cert' was not (LP: #960547) - - Provide user friendly message when an invalid locale is set - [Ben Howard] (LP: #859814) - - Support reading cloud-config from kernel command line parameter and - populating local file with it, which can then provide data for DataSources - - improve chef examples for working configurations on 11.10 and 12.04 - [Lorin Hochstein] (LP: #960564) - -0.6.2: - - fix bug where update was not done unless update was explicitly set. - It would not be run if 'upgrade' or packages were set to be installed - - fix bug in part-handler code, that prevented working part-handlers - (LP: #739694) - - fix bug in resizefs cloud-config that would cause trace based on - failure of 'blkid /dev/root' (LP: #726938) - - convert dos formated files to unix for user-scripts, boothooks, - and upstart jobs (LP: #744965) - - fix bug in seeding of grub dpkg configuration (LP: #752361) due - to renamed devices in newer (natty) kernels (/dev/sda1 -> /dev/xvda1) - - make metadata urls configurable, to support eucalyptus in - STATIC or SYSTEM modes (LP: #761847) - - support disabling byobu in cloud-config - - run cc_ssh as a cloud-init module so it is guaranteed to run before - ssh starts (LP: #781101) - - make prefix for keys added to /root/.ssh/authorized_keys configurable - and add 'no-port-forwarding,no-agent-forwarding,no-X11-forwarding' - to the default (LP: #798505) - - make 'cloud-config ready' command configurable (LP: #785551) - - make fstab fields used to 'fill in' shorthand entries configurable - This means you do not have to have 'nobootwait' in the values - (LP: #785542) - - read /etc/ssh/sshd_config for AuthorizedKeysFile rather than - assuming ~/.ssh/authorized_keys (LP: #731849) - - fix cloud-init in ubuntu lxc containers (LP: #800824) - - sanitize hosts file for system's hostname to 127.0.1.1 (LP: #802637) - - add chef support (cloudinit/CloudConfig/cc_chef.py) (LP: ##798844) - - do not give trace on failure to resize in lxc container (LP: #800856) - - increase the timeout on url gets for "seedfrom" values (LP: #812646) - - do not write entries for ephemeral0 on t1.micro (LP: #744019) - - support 'include-once' so that expiring or one-time use urls can - be used for '#include' to provide sensitive data. - - support for passing public and private keys to mcollective via cloud-config - - support multiple staticly configured network devices, as long as - all of them come up early (LP: #810044) - - Changes to handling user data mean that: - * boothooks will now run more than once as they were intended (and as - bootcmd commands do) - * cloud-config and user-scripts will be updated from user data every boot - - Fix issue where 'isatty' would return true for apt-add-repository. - apt-add-repository would get stdin which was attached to a terminal - (/dev/console) and would thus hang when running during boot. (LP: 831505) - This was done by changing all users of util.subp to have None input unless - specified - - Add some debug info to the console when cloud-init runs. - This is useful if debugging, IP and route information is printed to the - console. - - change the mechanism for handling .ssh/authorized_keys, to update entries - rather than appending. This ensures that the authorized_keys that are - being inserted actually do something (LP: #434076, LP: #833499) - - log warning on failure to set hostname (LP: #832175) - - upstart/cloud-init-nonet.conf: wait for all network interfaces to be up - allow for the possibility of /var/run != /run. - - DataSourceNoCloud, DataSourceOVF : do not provide a default hostname. - This way the configured hostname of the system will be used if not provided - by metadata (LP: #838280) - - DataSourceOVF: change the default instance id to 'iid-dsovf' from 'nocloud' - - Improve the OVF documentation, and provide a simple command line - tool for creating a useful ISO file. - -0.6.1: - - fix bug in fixing permission on /var/log/cloud-init.log (LP: #704509) - - improve comment strings in rsyslog file tools/21-cloudinit.conf - - add previous-instance-id and previous-datasource files to datadir - - add 'datasource' file to instance dir - - add setting of passwords and enabling/disabling of PasswordAuthentication - for sshd. By default no changes are done to sshd. - - fix for puppet configuration options (LP: #709946) [Ryan Lane] - - fix pickling of DataSource, which broke seeding. - - turn resize_rootfs default to True - - avoid mounts in DataSourceOVF if 'read' on device fails - 'mount /dev/sr0' for an empty virtual cdrom device was taking 18 seconds - - add 'manual_cache_clean' option to select manual cleaning of - the /var/lib/cloud/instance/ link, for a data source that might - not be present on every boot - - make DataSourceEc2 retries and timeout configurable - - add helper routines for apt-get update and install - - add 'bootcmd' like 'runcmd' to cloud-config syntax for running things early - - move from '#opt_include' in config file format to conf_d. - ie, now files in /etc/cloud.cfg.d/ is read rather than reading - '#opt_include ' or '#include ' in cloud.cfg - - allow /etc/hosts to be written from hosts.tmpl. which allows - getting local-hostname into /etc/hosts (LP: #720440) - - better handle startup if there is no eth0 (LP: #714807) - - update rather than append in puppet config [Marc Cluet] - - add cloud-config for mcollective [Marc Cluet] -0.6.0: - - change permissions of /var/log/cloud-init.log to accomodate - syslog writing to it (LP: #704509) - - rework of /var/lib/cloud layout - - remove updates-check (LP: #653220) - - support resizing / on first boot (enabled by default) - - added support for running CloudConfig modules at cloud-init time - rather than cloud-config time, and the new 'cloud_init_modules' - entry in cloud.cfg to indicate which should run then. - The driving force behind this was to have the rsyslog module - able to run before rsyslog even runs so that a restart would - not be needed (rsyslog on ubuntu runs on 'filesystem') - - moved setting and updating of hostname to cloud_init_modules - this allows the user to easily disable these from running. - This also means: - - the semaphore name for 'set_hostname' and 'update_hostname' - changes to 'config_set_hostname' and 'config_update_hostname' - - added cloud-config option 'hostname' for setting hostname - - moved upstart/cloud-run-user-script.conf to upstart/cloud-final.conf - - cloud-final.conf now runs runs cloud-config modules similar - to cloud-config and cloud-init. - - LP: #653271 - - added writing of "boot-finished" to /var/lib/cloud/instance/boot-finished - this is the last thing done, indicating cloud-init is finished booting - - writes message to console with timestamp and uptime - - write ssh keys to console as one of the last things done - this is to ensure they don't get run off the 'get-console-ouptut' buffer - - user_scripts run via cloud-final and thus semaphore renamed from - user_scripts to config_user_scripts - - add support for redirecting output of cloud-init, cloud-config, cloud-final - via the config file, or user data config file - - add support for posting data about the instance to a url (phone_home) - - add minimal OVF transport (iso) support - - make DataSources that are attempted dynamic and configurable from - system config. changen "cloud_type: auto" as configuration for this - to 'datasource_list: [ "Ec2" ]'. Each of the items in that list - must be modules that can be loaded by "DataSource" - - add 'timezone' option to cloud-config (LP: #645458) - - Added an additional archive format, that can be used for multi-part - input to cloud-init. This may be more user friendly then mime-multipart - See example in doc/examples/cloud-config-archive.txt (LP: #641504) - - add support for reading Rightscale style user data (LP: #668400) - and acting on it in cloud-config (cc_rightscale_userdata.py) - - make the message on 'disable_root' more clear (LP: #672417) - - do not require public key if private is given in ssh cloud-config - (LP: #648905) -# vi: syntax=text textwidth=79 diff --git a/HACKING.rst b/HACKING.rst deleted file mode 100644 index 6bfe4b4d..00000000 --- a/HACKING.rst +++ /dev/null @@ -1,48 +0,0 @@ -===================== -Hacking on cloud-init -===================== - -To get changes into cloud-init, the process to follow is: - -* If you have not already, be sure to sign the CCA: - - - `Canonical Contributor Agreement`_ - -* Get your changes into a local bzr branch. - Initialize a repo, and checkout trunk (init repo is to share bzr info across multiple checkouts, its different than git): - - - ``bzr init-repo cloud-init`` - - ``bzr branch lp:cloud-init trunk.dist`` - - ``bzr branch trunk.dist my-topic-branch`` - -* Commit your changes (note, you can make multiple commits, fixes, more commits.): - - - ``bzr commit`` - -* Check pep8 and test, and address any issues: - - - ``make test pep8`` - -* Push to launchpad to a personal branch: - - - ``bzr push lp:~/cloud-init/`` - -* Propose that for a merge into lp:cloud-init via web browser. - - - Open the branch in `Launchpad`_ - - - It will typically be at ``https://code.launchpad.net///`` - - ie. https://code.launchpad.net/~smoser/cloud-init/mybranch - -* Click 'Propose for merging' -* Select 'lp:cloud-init' as the target branch - -Then, someone on cloud-init-dev (currently `Scott Moser`_ and `Joshua Harlow`_) will -review your changes and follow up in the merge request. - -Feel free to ping and/or join #cloud-init on freenode (irc) if you have any questions. - -.. _Launchpad: https://launchpad.net -.. _Canonical Contributor Agreement: http://www.canonical.com/contributors -.. _Scott Moser: https://launchpad.net/~smoser -.. _Joshua Harlow: https://launchpad.net/~harlowja diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 94a9ed02..00000000 --- a/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 90f6c7d5..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -include *.py MANIFEST.in ChangeLog -global-include *.txt *.rst *.ini *.in *.conf *.cfg *.sh -graft tools -prune build -prune dist -prune .tox -prune .bzr -exclude .bzrignore diff --git a/Makefile b/Makefile deleted file mode 100644 index 32c50aee..00000000 --- a/Makefile +++ /dev/null @@ -1,82 +0,0 @@ -CWD=$(shell pwd) -PYVER ?= 3 -noseopts ?= -v - -YAML_FILES=$(shell find cloudinit bin tests tools -name "*.yaml" -type f ) -YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) - -CHANGELOG_VERSION=$(shell $(CWD)/tools/read-version) -CODE_VERSION=$(shell python -c "from cloudinit import version; print version.version_string()") - -PIP_INSTALL := pip install - -ifeq ($(PYVER),3) - pyflakes = pyflakes3 - unittests = unittest3 - yaml = yaml -else -ifeq ($(PYVER),2) - pyflakes = pyflakes - unittests = unittest -else - pyflakes = pyflakes pyflakes3 - unittests = unittest unittest3 -endif -endif - -ifeq ($(distro),) - distro = redhat -endif - -all: check - -check: check_version pep8 $(pyflakes) test $(yaml) - -pep8: - @$(CWD)/tools/run-pep8 - -pyflakes: - @$(CWD)/tools/run-pyflakes - -pyflakes3: - @$(CWD)/tools/run-pyflakes3 - -unittest: clean_pyc - nosetests $(noseopts) tests/unittests - -unittest3: clean_pyc - nosetests3 $(noseopts) tests/unittests - -pip-requirements: - @echo "Installing cloud-init dependencies..." - $(PIP_INSTALL) -r "$@.txt" -q - -pip-test-requirements: - @echo "Installing cloud-init test dependencies..." - $(PIP_INSTALL) -r "$@.txt" -q - -test: $(unittests) - -check_version: - @if [ "$(CHANGELOG_VERSION)" != "$(CODE_VERSION)" ]; then \ - echo "Error: ChangeLog version $(CHANGELOG_VERSION)" \ - "not equal to code version $(CODE_VERSION)"; exit 2; \ - else true; fi - -clean_pyc: - @find . -type f -name "*.pyc" -delete - -clean: clean_pyc - rm -rf /var/log/cloud-init.log /var/lib/cloud/ - -yaml: - @$(CWD)/tools/validate-yaml.py $(YAML_FILES) - -rpm: - ./packages/brpm --distro $(distro) - -deb: - ./packages/bddeb - -.PHONY: test pyflakes pyflakes3 clean pep8 rpm deb yaml check_version -.PHONY: pip-test-requirements pip-requirements clean_pyc unittest unittest3 diff --git a/README b/README new file mode 100644 index 00000000..b12e6dff --- /dev/null +++ b/README @@ -0,0 +1,11 @@ +cloud-init development has moved its revision control to git. +It is available at + https://code.launchpad.net/cloud-init + +Clone with + git clone https://git.launchpad.net/cloud-init +or + git clone git+ssh://git.launchpad.net/cloud-init + +For more information see + https://git.launchpad.net/cloud-init/tree/HACKING.rst diff --git a/TODO.rst b/TODO.rst deleted file mode 100644 index 7d126864..00000000 --- a/TODO.rst +++ /dev/null @@ -1,43 +0,0 @@ -============================================== -Things that cloud-init may do (better) someday -============================================== - -- Consider making ``failsafe`` ``DataSource`` - - sets the user password, writing it to console - -- Consider a ``previous`` ``DataSource``, if no other data source is - found, fall back to the ``previous`` one that worked. -- Rewrite ``cloud-init-query`` (currently not implemented) -- Possibly have a ``DataSource`` expose explicit fields: - - - instance-id - - hostname - - mirror - - release - - ssh public keys - -- Remove the conversion of the ubuntu network interface format conversion - to a RH/fedora format and replace it with a top level format that uses - the netcf libraries format instead (which itself knows how to translate - into the specific formats). See for example `netcf`_ which seems to be - an active project that has this capability. -- Replace the ``apt*`` modules with variants that now use the distro classes - to perform distro independent packaging commands (wherever possible). -- Replace some the LOG.debug calls with a LOG.info where appropriate instead - of how right now there is really only 2 levels (``WARN`` and ``DEBUG``) -- Remove the ``cc_`` prefix for config modules, either have them fully - specified (ie ``cloudinit.config.resizefs``) or by default only look in - the ``cloudinit.config`` namespace for these modules (or have a combination - of the above), this avoids having to understand where your modules are - coming from (which can be altered by the current python inclusion path) -- Instead of just warning when a module is being ran on a ``unknown`` - distribution perhaps we should not run that module in that case? Or we might - want to start reworking those modules so they will run on all - distributions? Or if that is not the case, then maybe we want to allow - fully specified python paths for modules and start encouraging - packages of ``ubuntu`` modules, packages of ``rhel`` specific modules that - people can add instead of having them all under the cloud-init ``root`` - tree? This might encourage more development of other modules instead of - having to go edit the cloud-init code to accomplish this. - -.. _netcf: https://fedorahosted.org/netcf/ diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py deleted file mode 100644 index da124641..00000000 --- a/cloudinit/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py deleted file mode 100644 index 3e6be203..00000000 --- a/cloudinit/cloud.py +++ /dev/null @@ -1,109 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import copy -import os - -from cloudinit import log as logging -from cloudinit.reporting import events - -LOG = logging.getLogger(__name__) - -# This class is the high level wrapper that provides -# access to cloud-init objects without exposing the stage objects -# to handler and or module manipulation. It allows for cloud -# init to restrict what those types of user facing code may see -# and or adjust (which helps avoid code messing with each other) -# -# It also provides util functions that avoid having to know -# how to get a certain member from this submembers as well -# as providing a backwards compatible object that can be maintained -# while the stages/other objects can be worked on independently... - - -class Cloud(object): - def __init__(self, datasource, paths, cfg, distro, runners, reporter=None): - self.datasource = datasource - self.paths = paths - self.distro = distro - self._cfg = cfg - self._runners = runners - if reporter is None: - reporter = events.ReportEventStack( - name="unnamed-cloud-reporter", - description="unnamed-cloud-reporter", - reporting_enabled=False) - self.reporter = reporter - - # If a 'user' manipulates logging or logging services - # it is typically useful to cause the logging to be - # setup again. - def cycle_logging(self): - logging.resetLogging() - logging.setupLogging(self.cfg) - - @property - def cfg(self): - # Ensure that not indirectly modified - return copy.deepcopy(self._cfg) - - def run(self, name, functor, args, freq=None, clear_on_fail=False): - return self._runners.run(name, functor, args, freq, clear_on_fail) - - def get_template_filename(self, name): - fn = self.paths.template_tpl % (name) - if not os.path.isfile(fn): - LOG.warn("No template found at %s for template named %s", fn, name) - return None - return fn - - # The rest of thes are just useful proxies - def get_userdata(self, apply_filter=True): - return self.datasource.get_userdata(apply_filter) - - def get_instance_id(self): - return self.datasource.get_instance_id() - - @property - def launch_index(self): - return self.datasource.launch_index - - def get_public_ssh_keys(self): - return self.datasource.get_public_ssh_keys() - - def get_locale(self): - return self.datasource.get_locale() - - def get_hostname(self, fqdn=False): - return self.datasource.get_hostname(fqdn=fqdn) - - def device_name_to_device(self, name): - return self.datasource.device_name_to_device(name) - - def get_ipath_cur(self, name=None): - return self.paths.get_ipath_cur(name) - - def get_cpath(self, name=None): - return self.paths.get_cpath(name) - - def get_ipath(self, name=None): - return self.paths.get_ipath(name) diff --git a/cloudinit/cmd/__init__.py b/cloudinit/cmd/__init__.py deleted file mode 100644 index da124641..00000000 --- a/cloudinit/cmd/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py deleted file mode 100644 index 63621c1d..00000000 --- a/cloudinit/cmd/main.py +++ /dev/null @@ -1,685 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -import json -import os -import sys -import tempfile -import time -import traceback - -from cloudinit import patcher -patcher.patch() # noqa - -from cloudinit import log as logging -from cloudinit import netinfo -from cloudinit import signal_handler -from cloudinit import sources -from cloudinit import stages -from cloudinit import templater -from cloudinit import util -from cloudinit import version - -from cloudinit import reporting -from cloudinit.reporting import events - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, - CLOUD_CONFIG) - - -# Pretty little cheetah formatted welcome message template -WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " - "${timestamp}. Up ${uptime} seconds.") - -# Module section template -MOD_SECTION_TPL = "cloud_%s_modules" - -# Things u can query on -QUERY_DATA_TYPES = [ - 'data', - 'data_raw', - 'instance_id', -] - -# Frequency shortname to full name -# (so users don't have to remember the full name...) -FREQ_SHORT_NAMES = { - 'instance': PER_INSTANCE, - 'always': PER_ALWAYS, - 'once': PER_ONCE, -} - -LOG = logging.getLogger() - - -# Used for when a logger may not be active -# and we still want to print exceptions... -def print_exc(msg=''): - if msg: - sys.stderr.write("%s\n" % (msg)) - sys.stderr.write('-' * 60) - sys.stderr.write("\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.write('-' * 60) - sys.stderr.write("\n") - - -def welcome(action, msg=None): - if not msg: - msg = welcome_format(action) - util.multi_log("%s\n" % (msg), - console=False, stderr=True, log=LOG) - return msg - - -def welcome_format(action): - tpl_params = { - 'version': version.version_string(), - 'uptime': util.uptime(), - 'timestamp': util.time_rfc2822(), - 'action': action, - } - return templater.render_string(WELCOME_MSG_TPL, tpl_params) - - -def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... - fn_cfgs = [] - if args.files: - for fh in args.files: - # The realpath is more useful in logging - # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) - return fn_cfgs - - -def run_module_section(mods, action_name, section): - full_section_name = MOD_SECTION_TPL % (section) - (which_ran, failures) = mods.run_section(full_section_name) - total_attempted = len(which_ran) + len(failures) - if total_attempted == 0: - msg = ("No '%s' modules to run" - " under section '%s'") % (action_name, full_section_name) - sys.stderr.write("%s\n" % (msg)) - LOG.debug(msg) - return [] - else: - LOG.debug("Ran %s modules with %s failures", - len(which_ran), len(failures)) - return failures - - -def apply_reporting_cfg(cfg): - if cfg.get('reporting'): - reporting.update_configuration(cfg.get('reporting')) - - -def main_init(name, args): - deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - if args.local: - deps = [sources.DEP_FILESYSTEM] - - if not args.local: - # See doc/kernel-cmdline.txt - # - # This is used in maas datasource, in "ephemeral" (read-only root) - # environment where the instance netboots to iscsi ro root. - # and the entity that controls the pxe config has to configure - # the maas datasource. - # - # Could be used elsewhere, only works on network based (not local). - root_name = "%s.d" % (CLOUD_CONFIG) - target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") - util.read_write_cmdline_url(target_fn) - - # Cloud-init 'init' stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Setup logging/output redirections with resultant config (if any) - # 3. Initialize the cloud-init filesystem - # 4. Check if we can stop early by looking for various files - # 5. Fetch the datasource - # 6. Connect to the current instance location + update the cache - # 7. Consume the userdata (handlers get activated here) - # 8. Construct the modules object - # 9. Adjust any subsequent logging/output redirections using the modules - # objects config as it may be different from init object - # 10. Run the modules for the 'init' stage - # 11. Done! - if not args.local: - w_msg = welcome_format(name) - else: - w_msg = welcome_format("%s-local" % (name)) - init = stages.Init(ds_deps=deps, reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - outfmt = None - errfmt = None - try: - LOG.debug("Closing stdin") - util.close_stdin() - (outfmt, errfmt) = util.fixup_output(init.cfg, name) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - print_exc("Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(init.cfg) - apply_reporting_cfg(init.cfg) - - # Any log usage prior to setupLogging above did not have local user log - # config applied. We send the welcome message now, as stderr/out have - # been redirected and log now configured. - welcome(name, msg=w_msg) - - # Stage 3 - try: - init.initialize() - except Exception: - util.logexc(LOG, "Failed to initialize, likely bad things to come!") - # Stage 4 - path_helper = init.paths - mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK - - if mode == sources.DSMODE_NETWORK: - existing = "trust" - sys.stderr.write("%s\n" % (netinfo.debug_info())) - LOG.debug(("Checking to see if files that we need already" - " exist from a previous run that would allow us" - " to stop early.")) - # no-net is written by upstart cloud-init-nonet when network failed - # to come up - stop_files = [ - os.path.join(path_helper.get_cpath("data"), "no-net"), - ] - existing_files = [] - for fn in stop_files: - if os.path.isfile(fn): - existing_files.append(fn) - - if existing_files: - LOG.debug("[%s] Exiting. stop file %s existed", - mode, existing_files) - return (None, []) - else: - LOG.debug("Execution continuing, no previous run detected that" - " would allow us to stop early.") - else: - existing = "check" - if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): - existing = "trust" - - init.purge_cache() - # Delete the non-net file as well - util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) - - # Stage 5 - try: - init.fetch(existing=existing) - # if in network mode, and the datasource is local - # then work was done at that stage. - if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: - LOG.debug("[%s] Exiting. datasource %s in local mode", - mode, init.datasource) - return (None, []) - except sources.DataSourceNotFoundException: - # In the case of 'cloud-init init' without '--local' it is a bit - # more likely that the user would consider it failure if nothing was - # found. When using upstart it will also mentions job failure - # in console log if exit code is != 0. - if mode == sources.DSMODE_LOCAL: - LOG.debug("No local datasource found") - else: - util.logexc(LOG, ("No instance datasource found!" - " Likely bad things to come!")) - if not args.force: - init.apply_network_config(bring_up=not args.local) - LOG.debug("[%s] Exiting without datasource in local mode", mode) - if mode == sources.DSMODE_LOCAL: - return (None, []) - else: - return (None, ["No instance datasource found."]) - else: - LOG.debug("[%s] barreling on in force mode without datasource", - mode) - - # Stage 6 - iid = init.instancify() - LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", - mode, name, iid, init.is_new_instance()) - - init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) - - if mode == sources.DSMODE_LOCAL: - if init.datasource.dsmode != mode: - LOG.debug("[%s] Exiting. datasource %s not in local mode.", - mode, init.datasource) - return (init.datasource, []) - else: - LOG.debug("[%s] %s is in local mode, will apply init modules now.", - mode, init.datasource) - - # update fully realizes user-data (pulling in #include if necessary) - init.update() - # Stage 7 - try: - # Attempt to consume the data per instance. - # This may run user-data handlers and/or perform - # url downloads and such as needed. - (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) - if not ran: - # Just consume anything that is set to run per-always - # if nothing ran in the per-instance code - # - # See: https://bugs.launchpad.net/bugs/819507 for a little - # reason behind this... - init.consume_data(PER_ALWAYS) - except Exception: - util.logexc(LOG, "Consuming user data failed!") - return (init.datasource, ["Consuming user data failed!"]) - - apply_reporting_cfg(init.cfg) - - # Stage 8 - re-read and apply relevant cloud-config to include user-data - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 9 - try: - outfmt_orig = outfmt - errfmt_orig = errfmt - (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) - if outfmt_orig != outfmt or errfmt_orig != errfmt: - LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) - (outfmt, errfmt) = util.fixup_output(mods.cfg, name) - except Exception: - util.logexc(LOG, "Failed to re-adjust output redirection!") - logging.setupLogging(mods.cfg) - - # Stage 10 - return (init.datasource, run_module_section(mods, name, name)) - - -def main_modules(action_name, args): - name = args.mode - # Cloud-init 'modules' stages are broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Get the datasource from the init object, if it does - # not exist then that means the main_init stage never - # worked, and thus this stage can not run. - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the modules for the given stage name - # 6. Done! - w_msg = welcome_format("%s:%s" % (action_name, name)) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, theres nothing to do - msg = ('Can not apply stage %s, no datasource found! Likely bad ' - 'things to come!' % name) - util.logexc(LOG, msg) - print_exc(msg) - if not args.force: - return [(msg)] - # Stage 3 - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, name) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - return run_module_section(mods, name, name) - - -def main_query(name, _args): - raise NotImplementedError(("Action '%s' is not" - " currently implemented") % (name)) - - -def main_single(name, args): - # Cloud-init single stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Attempt to fetch the datasource (warn if it doesn't work) - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the single module - # 6. Done! - mod_name = args.name - w_msg = welcome_format(name) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, - # that might be bad (or ok) depending on - # the module being ran (so continue on) - util.logexc(LOG, ("Failed to fetch your datasource," - " likely bad things to come!")) - print_exc(("Failed to fetch your datasource," - " likely bad things to come!")) - if not args.force: - return 1 - # Stage 3 - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - mod_args = args.module_args - if mod_args: - LOG.debug("Using passed in arguments %s", mod_args) - mod_freq = args.frequency - if mod_freq: - LOG.debug("Using passed in frequency %s", mod_freq) - mod_freq = FREQ_SHORT_NAMES.get(mod_freq) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, None) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - (which_ran, failures) = mods.run_single(mod_name, - mod_args, - mod_freq) - if failures: - LOG.warn("Ran %s but it failed!", mod_name) - return 1 - elif not which_ran: - LOG.warn("Did not run %s, does it exist?", mod_name) - return 1 - else: - # Guess it worked - return 0 - - -def atomic_write_file(path, content, mode='w'): - tf = None - try: - tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), - delete=False, mode=mode) - tf.write(content) - tf.close() - os.rename(tf.name, path) - except Exception as e: - if tf is not None: - os.unlink(tf.name) - raise e - - -def atomic_write_json(path, data): - return atomic_write_file(path, json.dumps(data, indent=1) + "\n") - - -def status_wrapper(name, args, data_d=None, link_d=None): - if data_d is None: - data_d = os.path.normpath("/var/lib/cloud/data") - if link_d is None: - link_d = os.path.normpath("/run/cloud-init") - - status_path = os.path.join(data_d, "status.json") - status_link = os.path.join(link_d, "status.json") - result_path = os.path.join(data_d, "result.json") - result_link = os.path.join(link_d, "result.json") - - util.ensure_dirs((data_d, link_d,)) - - (_name, functor) = args.action - - if name == "init": - if args.local: - mode = "init-local" - else: - mode = "init" - elif name == "modules": - mode = "modules-%s" % args.mode - else: - raise ValueError("unknown name: %s" % name) - - modes = ('init', 'init-local', 'modules-config', 'modules-final') - - status = None - if mode == 'init-local': - for f in (status_link, result_link, status_path, result_path): - util.del_file(f) - else: - try: - status = json.loads(util.load_file(status_path)) - except Exception: - pass - - if status is None: - nullstatus = { - 'errors': [], - 'start': None, - 'finished': None, - } - status = {'v1': {}} - for m in modes: - status['v1'][m] = nullstatus.copy() - status['v1']['datasource'] = None - - v1 = status['v1'] - v1['stage'] = mode - v1[mode]['start'] = time.time() - - atomic_write_json(status_path, status) - util.sym_link(os.path.relpath(status_path, link_d), status_link, - force=True) - - try: - ret = functor(name, args) - if mode in ('init', 'init-local'): - (datasource, errors) = ret - if datasource is not None: - v1['datasource'] = str(datasource) - else: - errors = ret - - v1[mode]['errors'] = [str(e) for e in errors] - - except Exception as e: - util.logexc(LOG, "failed stage %s", mode) - print_exc("failed run of stage %s" % mode) - v1[mode]['errors'] = [str(e)] - - v1[mode]['finished'] = time.time() - v1['stage'] = None - - atomic_write_json(status_path, status) - - if mode == "modules-final": - # write the 'finished' file - errors = [] - for m in modes: - if v1[m]['errors']: - errors.extend(v1[m].get('errors', [])) - - atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], - 'errors': errors}}) - util.sym_link(os.path.relpath(result_path, link_d), result_link, - force=True) - - return len(v1[mode]['errors']) - - -def main(sysv_args=None): - if sysv_args is not None: - parser = argparse.ArgumentParser(prog=sysv_args[0]) - sysv_args = sysv_args[1:] - else: - parser = argparse.ArgumentParser() - - # Top level args - parser.add_argument('--version', '-v', action='version', - version='%(prog)s ' + (version.version_string())) - parser.add_argument('--file', '-f', action='append', - dest='files', - help=('additional yaml configuration' - ' files to use'), - type=argparse.FileType('rb')) - parser.add_argument('--debug', '-d', action='store_true', - help=('show additional pre-action' - ' logging (default: %(default)s)'), - default=False) - parser.add_argument('--force', action='store_true', - help=('force running even if no datasource is' - ' found (use at your own risk)'), - dest='force', - default=False) - - parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers() - - # Each action and its sub-options (if any) - parser_init = subparsers.add_parser('init', - help=('initializes cloud-init and' - ' performs initial modules')) - parser_init.add_argument("--local", '-l', action='store_true', - help="start in local mode (default: %(default)s)", - default=False) - # This is used so that we can know which action is selected + - # the functor to use to run this subcommand - parser_init.set_defaults(action=('init', main_init)) - - # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser('modules', - help=('activates modules using ' - 'a given configuration key')) - parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) - parser_mod.set_defaults(action=('modules', main_modules)) - - # These settings are used when you want to query information - # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', - help=('query information stored ' - 'in cloud-init')) - parser_query.add_argument("--name", '-n', action="store", - help="item name to query on", - required=True, - choices=QUERY_DATA_TYPES) - parser_query.set_defaults(action=('query', main_query)) - - # This subcommand allows you to run a single module - parser_single = subparsers.add_parser('single', - help=('run a single module ')) - parser_single.set_defaults(action=('single', main_single)) - parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) - parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) - parser_single.add_argument("--report", action="store_true", - help="enable reporting", - required=False) - parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) - parser_single.set_defaults(action=('single', main_single)) - - args = parser.parse_args(args=sysv_args) - - try: - (name, functor) = args.action - except AttributeError: - parser.error('too few arguments') - - # Setup basic logging to start (until reinitialized) - # iff in debug mode... - if args.debug: - logging.setupBasicLogging() - - # Setup signal handlers before running - signal_handler.attach_handlers() - - if name in ("modules", "init"): - functor = status_wrapper - - report_on = True - if name == "init": - if args.local: - rname, rdesc = ("init-local", "searching for local datasources") - else: - rname, rdesc = ("init-network", - "searching for network datasources") - elif name == "modules": - rname, rdesc = ("modules-%s" % args.mode, - "running modules for %s" % args.mode) - elif name == "single": - rname, rdesc = ("single/%s" % args.name, - "running single module %s" % args.name) - report_on = args.report - - args.reporter = events.ReportEventStack( - rname, rdesc, reporting_enabled=report_on) - with args.reporter: - return util.log_time( - logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, - get_uptime=True, func=functor, args=(name, args)) diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py deleted file mode 100644 index d57453be..00000000 --- a/cloudinit/config/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2008-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Chuck Short -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) - -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - -# This prefix is used to make it less -# of a chance that when importing -# we will not find something else with the same -# name in the lookup path... -MOD_PREFIX = "cc_" - - -def form_module_name(name): - canon_name = name.replace("-", "_") - if canon_name.lower().endswith(".py"): - canon_name = canon_name[0:(len(canon_name) - 3)] - canon_name = canon_name.strip() - if not canon_name: - return None - if not canon_name.startswith(MOD_PREFIX): - canon_name = '%s%s' % (MOD_PREFIX, canon_name) - return canon_name - - -def fixup_module(mod, def_freq=PER_INSTANCE): - if not hasattr(mod, 'frequency'): - setattr(mod, 'frequency', def_freq) - else: - freq = mod.frequency - if freq and freq not in FREQUENCIES: - LOG.warn("Module %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'distros'): - setattr(mod, 'distros', []) - if not hasattr(mod, 'osfamilies'): - setattr(mod, 'osfamilies', []) - return mod diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py deleted file mode 100644 index 05ad4b03..00000000 --- a/cloudinit/config/cc_apt_configure.py +++ /dev/null @@ -1,319 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import glob -import os -import re - -from cloudinit import gpg -from cloudinit import templater -from cloudinit import util - -distros = ['ubuntu', 'debian'] - -PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" -APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" -APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" - -# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') -ADD_APT_REPO_MATCH = r"^[\w-]+:\w" - - -def handle(name, cfg, cloud, log, _args): - if util.is_false(cfg.get('apt_configure_enabled', True)): - log.debug("Skipping module named %s, disabled by config.", name) - return - - release = get_release() - mirrors = find_apt_mirror_info(cloud, cfg) - if not mirrors or "primary" not in mirrors: - log.debug(("Skipping module named %s," - " no package 'mirror' located"), name) - return - - # backwards compatibility - mirror = mirrors["primary"] - mirrors["mirror"] = mirror - - log.debug("Mirror info: %s" % mirrors) - - if not util.get_cfg_option_bool(cfg, - 'apt_preserve_sources_list', False): - generate_sources_list(cfg, release, mirrors, cloud, log) - old_mirrors = cfg.get('apt_old_mirrors', - {"primary": "archive.ubuntu.com/ubuntu", - "security": "security.ubuntu.com/ubuntu"}) - rename_apt_lists(old_mirrors, mirrors) - - try: - apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) - except Exception as e: - log.warn("failed to proxy or apt config info: %s", e) - - # Process 'apt_sources' - if 'apt_sources' in cfg: - params = mirrors - params['RELEASE'] = release - params['MIRROR'] = mirror - - matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) - if matchcfg: - matcher = re.compile(matchcfg).search - else: - def matcher(x): - return False - - errors = add_apt_sources(cfg['apt_sources'], params, - aa_repo_match=matcher) - for e in errors: - log.warn("Add source error: %s", ':'.join(e)) - - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("Setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except Exception: - util.logexc(log, "Failed to run debconf-set-selections") - - -def mirrorurl_to_apt_fileprefix(mirror): - string = mirror - # take off http:// or ftp:// - if string.endswith("/"): - string = string[0:-1] - pos = string.find("://") - if pos >= 0: - string = string[pos + 3:] - string = string.replace("/", "_") - return string - - -def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): - for (name, omirror) in old_mirrors.items(): - nmirror = new_mirrors.get(name) - if not nmirror: - continue - oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror)) - if oprefix == nprefix: - continue - olen = len(oprefix) - for filename in glob.glob("%s_*" % oprefix): - util.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - (stdout, _stderr) = util.subp(['lsb_release', '-cs']) - return stdout.strip() - - -def generate_sources_list(cfg, codename, mirrors, cloud, log): - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] - - custtmpl = cfg.get('apt_custom_sources_list', None) - if custtmpl is not None: - templater.render_string_to_file(custtmpl, - '/etc/apt/sources.list', params) - return - - template_fn = cloud.get_template_filename('sources.list.%s' % - (cloud.distro.name)) - if not template_fn: - template_fn = cloud.get_template_filename('sources.list') - if not template_fn: - log.warn("No template found, not rendering /etc/apt/sources.list") - return - - templater.render_to_file(template_fn, '/etc/apt/sources.list', params) - - -def add_apt_key_raw(key): - """ - actual adding of a key as defined in key argument - to the system - """ - try: - util.subp(('apt-key', 'add', '-'), key) - except util.ProcessExecutionError: - raise ValueError('failed to add apt GPG Key to apt keyring') - - -def add_apt_key(ent): - """ - add key to the system as defined in ent (if any) - supports raw keys or keyid's - The latter will as a first step fetch the raw key from a keyserver - """ - if 'keyid' in ent and 'key' not in ent: - keyserver = "keyserver.ubuntu.com" - if 'keyserver' in ent: - keyserver = ent['keyserver'] - ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver) - - if 'key' in ent: - add_apt_key_raw(ent['key']) - - -def convert_to_new_format(srclist): - """convert_to_new_format - convert the old list based format to the new dict based one - """ - srcdict = {} - if isinstance(srclist, list): - for srcent in srclist: - if 'filename' not in srcent: - # file collides for multiple !filename cases for compatibility - # yet we need them all processed, so not same dictionary key - srcent['filename'] = "cloud_config_sources.list" - key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - else: - # all with filename use that as key (matching new format) - key = srcent['filename'] - srcdict[key] = srcent - elif isinstance(srclist, dict): - srcdict = srclist - else: - raise ValueError("unknown apt_sources format") - - return srcdict - - -def add_apt_sources(srclist, template_params=None, aa_repo_match=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList - """ - if template_params is None: - template_params = {} - - if aa_repo_match is None: - def _aa_repo_match(x): - return False - aa_repo_match = _aa_repo_match - - errorlist = [] - srcdict = convert_to_new_format(srclist) - - for filename in srcdict: - ent = srcdict[filename] - if 'filename' not in ent: - ent['filename'] = filename - - # keys can be added without specifying a source - try: - add_apt_key(ent) - except ValueError as detail: - errorlist.append([ent, detail]) - - if 'source' not in ent: - errorlist.append(["", "missing source"]) - continue - source = ent['source'] - source = templater.render_string(source, template_params) - - if not ent['filename'].startswith(os.path.sep): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - - if aa_repo_match(source): - try: - util.subp(["add-apt-repository", source]) - except util.ProcessExecutionError as e: - errorlist.append([source, - ("add-apt-repository failed. " + str(e))]) - continue - - try: - contents = "%s\n" % (source) - util.write_file(ent['filename'], contents, omode="ab") - except Exception: - errorlist.append([source, - "failed write to file %s" % ent['filename']]) - - return errorlist - - -def find_apt_mirror_info(cloud, cfg): - """find an apt_mirror given the cloud and cfg provided.""" - - mirror = None - - # this is less preferred way of specifying mirror preferred would be to - # use the distro's search or package_mirror. - mirror = cfg.get("apt_mirror", None) - - search = cfg.get("apt_mirror_search", None) - if not mirror and search: - mirror = util.search_for_mirror(search) - - if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): - mydom = "" - doms = [] - - # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - mydom = ".".join(fqdn.split(".")[1:]) - if mydom: - doms.append(".%s" % mydom) - - doms.extend((".localdomain", "",)) - - mirror_list = [] - distro = cloud.distro.name - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) - for post in doms: - mirror_list.append(mirrorfmt % (post)) - - mirror = util.search_for_mirror(mirror_list) - - mirror_info = cloud.datasource.get_package_mirror_info() - - # this is a bit strange. - # if mirror is set, then one of the legacy options above set it - # but they do not cover security. so we need to get that from - # get_package_mirror_info - if mirror: - mirror_info.update({'primary': mirror}) - - return mirror_info - - -def apply_apt_config(cfg, proxy_fname, config_fname): - # Set up any apt proxy - cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'), - ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'), - ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'), - ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";')) - - proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] - if len(proxies): - util.write_file(proxy_fname, '\n'.join(proxies) + '\n') - elif os.path.isfile(proxy_fname): - util.del_file(proxy_fname) - - if cfg.get('apt_config', None): - util.write_file(config_fname, cfg.get('apt_config')) - elif os.path.isfile(config_fname): - util.del_file(config_fname) diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py deleted file mode 100644 index 40c32c84..00000000 --- a/cloudinit/config/cc_apt_pipelining.py +++ /dev/null @@ -1,57 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -frequency = PER_INSTANCE - -distros = ['ubuntu', 'debian'] - -DEFAULT_FILE = "/etc/apt/apt.conf.d/90cloud-init-pipelining" - -APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" - 'Acquire::http::Pipeline-Depth "%s";\n') - -# Acquire::http::Pipeline-Depth can be a value -# from 0 to 5 indicating how many outstanding requests APT should send. -# A value of zero MUST be specified if the remote host does not properly linger -# on TCP connections - otherwise data corruption will occur. - - -def handle(_name, cfg, _cloud, log, _args): - - apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) - apt_pipe_value_s = str(apt_pipe_value).lower().strip() - - if apt_pipe_value_s == "false": - write_apt_snippet("0", log, DEFAULT_FILE) - elif apt_pipe_value_s in ("none", "unchanged", "os"): - return - elif apt_pipe_value_s in [str(b) for b in range(0, 6)]: - write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE) - else: - log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value) - - -def write_apt_snippet(setting, log, f_name): - """Writes f_name with apt pipeline depth 'setting'.""" - - file_contents = APT_PIPE_TPL % (setting) - util.write_file(f_name, file_contents) - log.debug("Wrote %s with apt pipeline depth setting %s", f_name, setting) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py deleted file mode 100644 index b763a3c3..00000000 --- a/cloudinit/config/cc_bootcmd.py +++ /dev/null @@ -1,54 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit.settings import PER_ALWAYS -from cloudinit import util - -frequency = PER_ALWAYS - - -def handle(name, cfg, cloud, log, _args): - - if "bootcmd" not in cfg: - log.debug(("Skipping module named %s," - " no 'bootcmd' key in configuration"), name) - return - - with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: - try: - content = util.shellify(cfg["bootcmd"]) - tmpf.write(util.encode_text(content)) - tmpf.flush() - except Exception: - util.logexc(log, "Failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - iid = cloud.get_instance_id() - if iid: - env['INSTANCE_ID'] = str(iid) - cmd = ['/bin/sh', tmpf.name] - util.subp(cmd, env=env, capture=False) - except Exception: - util.logexc(log, "Failed to run bootcmd module %s", name) - raise diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py deleted file mode 100644 index ef0ce7ab..00000000 --- a/cloudinit/config/cc_byobu.py +++ /dev/null @@ -1,80 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - -from cloudinit import util - -distros = ['ubuntu', 'debian'] - - -def handle(name, cfg, cloud, log, args): - if len(args) != 0: - value = args[0] - else: - value = util.get_cfg_option_str(cfg, "byobu_by_default", "") - - if not value: - log.debug("Skipping module named %s, no 'byobu' values found", name) - return - - if value == "user" or value == "system": - value = "enable-%s" % value - - valid = ("enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable") - if value not in valid: - log.warn("Unknown value %s for byobu_by_default", value) - - mod_user = value.endswith("-user") - mod_sys = value.endswith("-system") - if value.startswith("enable"): - bl_inst = "install" - dc_val = "byobu byobu/launch-by-default boolean true" - mod_sys = True - else: - if value == "disable": - mod_user = True - mod_sys = True - bl_inst = "uninstall" - dc_val = "byobu byobu/launch-by-default boolean false" - - shcmd = "" - if mod_user: - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - (user, _user_config) = ds.extract_default(users) - if not user: - log.warn(("No default byobu user provided, " - "can not launch %s for the default user"), bl_inst) - else: - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) - shcmd += " || X=$(($X+1)); " - if mod_sys: - shcmd += "echo \"%s\" | debconf-set-selections" % dc_val - shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" - shcmd += " || X=$(($X+1)); " - - if len(shcmd): - cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - log.debug("Setting byobu to %s", value) - util.subp(cmd, capture=False) diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py deleted file mode 100644 index 8248b020..00000000 --- a/cloudinit/config/cc_ca_certs.py +++ /dev/null @@ -1,104 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Mike Milner -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" -CA_CERT_FULL_PATH = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - -distros = ['ubuntu', 'debian'] - - -def update_ca_certs(): - """ - Updates the CA certificate cache on the current machine. - """ - util.subp(["update-ca-certificates"], capture=False) - - -def add_ca_certs(certs): - """ - Adds certificates to the system. To actually apply the new certificates - you must also call L{update_ca_certs}. - - @param certs: A list of certificate strings. - """ - if certs: - # First ensure they are strings... - cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) - - # Append cert filename to CA_CERT_CONFIG file. - # We have to strip the content because blank lines in the file - # causes subsequent entries to be ignored. (LP: #1077020) - orig = util.load_file(CA_CERT_CONFIG) - cur_cont = '\n'.join([l for l in orig.splitlines() - if l != CA_CERT_FILENAME]) - out = "%s\n%s\n" % (cur_cont.rstrip(), CA_CERT_FILENAME) - util.write_file(CA_CERT_CONFIG, out, omode="wb") - - -def remove_default_ca_certs(): - """ - Removes all default trusted CA certificates from the system. To actually - apply the change you must also call L{update_ca_certs}. - """ - util.delete_dir_contents(CA_CERT_PATH) - util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0o644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - util.subp(('debconf-set-selections', '-'), debconf_sel) - - -def handle(name, cfg, _cloud, log, _args): - """ - Call to handle ca-cert sections in cloud-config file. - - @param name: The module name "ca-cert" from cloud.cfg - @param cfg: A nested dict containing the entire cloud config contents. - @param cloud: The L{CloudInit} object in use. - @param log: Pre-initialized Python logger object to use for logging. - @param args: Any module arguments from cloud.cfg - """ - # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: - log.debug(("Skipping module named %s," - " no 'ca-certs' key in configuration"), name) - return - - ca_cert_cfg = cfg['ca-certs'] - - # If there is a remove-defaults option set to true, remove the system - # default trusted CA certs first. - if ca_cert_cfg.get("remove-defaults", False): - log.debug("Removing default certificates") - remove_default_ca_certs() - - # If we are given any new trusted CA certs to add, add them. - if "trusted" in ca_cert_cfg: - trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") - if trusted_certs: - log.debug("Adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) - - # Update the system with the new cert configuration. - log.debug("Updating certificates") - update_ca_certs() diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py deleted file mode 100644 index 4c28be6a..00000000 --- a/cloudinit/config/cc_chef.py +++ /dev/null @@ -1,342 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Avishai Ish-Shalom -# Author: Mike Moulton -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -**Summary:** module that configures, starts and installs chef. - -**Description:** This module enables chef to be installed (from packages or -from gems, or from omnibus). Before this occurs chef configurations are -written to disk (validation.pem, client.pem, firstboot.json, client.rb), -and needed chef folders/directories are created (/etc/chef and /var/log/chef -and so-on). Then once installing proceeds correctly if configured chef will -be started (in daemon mode or in non-daemon mode) and then once that has -finished (if ran in non-daemon mode this will be when chef finishes -converging, if ran in daemon mode then no further actions are possible since -chef will have forked into its own process) then a post run function can -run that can do finishing activities (such as removing the validation pem -file). - -It can be configured with the following option structure:: - - chef: - directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, - /var/cache/chef, /var/backups/chef, /var/run/chef) - validation_cert: (optional string to be written to file validation_key) - special value 'system' means set use existing file - validation_key: (optional the path for validation_cert. default - /etc/chef/validation.pem) - firstboot_path: (path to write run_list and initial_attributes keys that - should also be present in this configuration, defaults - to /etc/chef/firstboot.json) - exec: boolean to run or not run chef (defaults to false, unless - a gem installed is requested - where this will then default - to true) - - chef.rb template keys (if falsey, then will be skipped and not - written to /etc/chef/client.rb) - - chef: - client_key: - environment: - file_backup_path: - file_cache_path: - json_attribs: - log_level: - log_location: - node_name: - pid_file: - server_url: - show_time: - ssl_verify_mode: - validation_cert: - validation_key: - validation_name: -""" - -import itertools -import json -import os - -from cloudinit import templater -from cloudinit import url_helper -from cloudinit import util - -import six - -RUBY_VERSION_DEFAULT = "1.8" - -CHEF_DIRS = tuple([ - '/etc/chef', - '/var/log/chef', - '/var/lib/chef', - '/var/cache/chef', - '/var/backups/chef', - '/var/run/chef', -]) -REQUIRED_CHEF_DIRS = tuple([ - '/etc/chef', -]) - -# Used if fetching chef from a omnibus style package -OMNIBUS_URL = "https://www.getchef.com/chef/install.sh" -OMNIBUS_URL_RETRIES = 5 - -CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' -CHEF_FB_PATH = '/etc/chef/firstboot.json' -CHEF_RB_TPL_DEFAULTS = { - # These are ruby symbols... - 'ssl_verify_mode': ':verify_none', - 'log_level': ':info', - # These are not symbols... - 'log_location': '/var/log/chef/client.log', - 'validation_key': CHEF_VALIDATION_PEM_PATH, - 'validation_cert': None, - 'client_key': "/etc/chef/client.pem", - 'json_attribs': CHEF_FB_PATH, - 'file_cache_path': "/var/cache/chef", - 'file_backup_path': "/var/backups/chef", - 'pid_file': "/var/run/chef/client.pid", - 'show_time': True, -} -CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) -CHEF_RB_TPL_PATH_KEYS = frozenset([ - 'log_location', - 'validation_key', - 'client_key', - 'file_cache_path', - 'json_attribs', - 'file_cache_path', - 'pid_file', -]) -CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) -CHEF_RB_TPL_KEYS.extend([ - 'server_url', - 'node_name', - 'environment', - 'validation_name', -]) -CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) -CHEF_RB_PATH = '/etc/chef/client.rb' -CHEF_EXEC_PATH = '/usr/bin/chef-client' -CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) - - -def is_installed(): - if not os.path.isfile(CHEF_EXEC_PATH): - return False - if not os.access(CHEF_EXEC_PATH, os.X_OK): - return False - return True - - -def post_run_chef(chef_cfg, log): - delete_pem = util.get_cfg_option_bool(chef_cfg, - 'delete_validation_post_exec', - default=False) - if delete_pem and os.path.isfile(CHEF_VALIDATION_PEM_PATH): - os.unlink(CHEF_VALIDATION_PEM_PATH) - - -def get_template_params(iid, chef_cfg, log): - params = CHEF_RB_TPL_DEFAULTS.copy() - # Allow users to overwrite any of the keys they want (if they so choose), - # when a value is None, then the value will be set to None and no boolean - # or string version will be populated... - for (k, v) in chef_cfg.items(): - if k not in CHEF_RB_TPL_KEYS: - log.debug("Skipping unknown chef template key '%s'", k) - continue - if v is None: - params[k] = None - else: - # This will make the value a boolean or string... - if k in CHEF_RB_TPL_BOOL_KEYS: - params[k] = util.get_cfg_option_bool(chef_cfg, k) - else: - params[k] = util.get_cfg_option_str(chef_cfg, k) - # These ones are overwritten to be exact values... - params.update({ - 'generated_by': util.make_header(), - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - default=iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - default='_default'), - # These two are mandatory... - 'server_url': chef_cfg['server_url'], - 'validation_name': chef_cfg['validation_name'], - }) - return params - - -def handle(name, cfg, cloud, log, _args): - """Handler method activated by cloud-init.""" - - # If there isn't a chef key in the configuration don't do anything - if 'chef' not in cfg: - log.debug(("Skipping module named %s," - " no 'chef' key in configuration"), name) - return - chef_cfg = cfg['chef'] - - # Ensure the chef directories we use exist - chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') - if not chef_dirs: - chef_dirs = list(CHEF_DIRS) - for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): - util.ensure_dir(d) - - vkey_path = chef_cfg.get('validation_key', CHEF_VALIDATION_PEM_PATH) - vcert = chef_cfg.get('validation_cert') - # special value 'system' means do not overwrite the file - # but still render the template to contain 'validation_key' - if vcert: - if vcert != "system": - util.write_file(vkey_path, vcert) - elif not os.path.isfile(vkey_path): - log.warn("chef validation_cert provided as 'system', but " - "validation_key path '%s' does not exist.", - vkey_path) - - # Create the chef config from template - template_fn = cloud.get_template_filename('chef_client.rb') - if template_fn: - iid = str(cloud.datasource.get_instance_id()) - params = get_template_params(iid, chef_cfg, log) - # Do a best effort attempt to ensure that the template values that - # are associated with paths have there parent directory created - # before they are used by the chef-client itself. - param_paths = set() - for (k, v) in params.items(): - if k in CHEF_RB_TPL_PATH_KEYS and v: - param_paths.add(os.path.dirname(v)) - util.ensure_dirs(param_paths) - templater.render_to_file(template_fn, CHEF_RB_PATH, params) - else: - log.warn("No template found, not rendering to %s", - CHEF_RB_PATH) - - # Set the firstboot json - fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', - default=CHEF_FB_PATH) - if not fb_filename: - log.info("First boot path empty, not writing first boot json file") - else: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in list(initial_attributes.keys()): - initial_json[k] = initial_attributes[k] - util.write_file(fb_filename, json.dumps(initial_json)) - - # Try to install chef, if its not already installed... - force_install = util.get_cfg_option_bool(chef_cfg, - 'force_install', default=False) - if not is_installed() or force_install: - run = install_chef(cloud, chef_cfg, log) - elif is_installed(): - run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) - else: - run = False - if run: - run_chef(chef_cfg, log) - post_run_chef(chef_cfg, log) - - -def run_chef(chef_cfg, log): - log.debug('Running chef-client') - cmd = [CHEF_EXEC_PATH] - if 'exec_arguments' in chef_cfg: - cmd_args = chef_cfg['exec_arguments'] - if isinstance(cmd_args, (list, tuple)): - cmd.extend(cmd_args) - elif isinstance(cmd_args, six.string_types): - cmd.append(cmd_args) - else: - log.warn("Unknown type %s provided for chef" - " 'exec_arguments' expected list, tuple," - " or string", type(cmd_args)) - cmd.extend(CHEF_EXEC_DEF_ARGS) - else: - cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) - - -def install_chef(cloud, chef_cfg, log): - # If chef is not installed, we install chef based on 'install_type' - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) - if install_type == "gems": - # This will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - RUBY_VERSION_DEFAULT) - install_chef_from_gems(ruby_version, chef_version, cloud.distro) - # Retain backwards compat, by preferring True instead of False - # when not provided/overriden... - run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True) - elif install_type == 'packages': - # This will install and run the chef-client from packages - cloud.distro.install_packages(('chef',)) - elif install_type == 'omnibus': - # This will install as a omnibus unified package - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - retries = max(0, util.get_cfg_option_int(chef_cfg, - "omnibus_url_retries", - default=OMNIBUS_URL_RETRIES)) - content = url_helper.readurl(url=url, retries=retries) - with util.tempdir() as tmpd: - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, content, mode=0o700) - util.subp([tmpf], capture=False) - else: - log.warn("Unknown chef install type '%s'", install_type) - run = False - return run - - -def get_ruby_packages(version): - # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] - if version == "1.8": - pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return pkgs - - -def install_chef_from_gems(ruby_version, chef_version, distro): - distro.install_packages(get_ruby_packages(ruby_version)) - if not os.path.exists('/usr/bin/gem'): - util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') - if not os.path.exists('/usr/bin/ruby'): - util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') - if chef_version: - util.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) - else: - util.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py deleted file mode 100644 index bdc32fe6..00000000 --- a/cloudinit/config/cc_debug.py +++ /dev/null @@ -1,109 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Yahoo! Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -**Summary:** helper to debug cloud-init *internal* datastructures. - -**Description:** This module will enable for outputting various internal -information that cloud-init sources provide to either a file or to the output -console/log location that this cloud-init has been configured with when -running. - -It can be configured with the following option structure:: - - debug: - verbose: (defaulting to true) - output: (location to write output, defaulting to console + log) - -.. note:: - - Log configurations are not output. -""" - -import copy - -from six import StringIO - -from cloudinit import type_utils -from cloudinit import util - -SKIP_KEYS = frozenset(['log_cfgs']) - - -def _make_header(text): - header = StringIO() - header.write("-" * 80) - header.write("\n") - header.write(text.center(80, ' ')) - header.write("\n") - header.write("-" * 80) - header.write("\n") - return header.getvalue() - - -def _dumps(obj): - text = util.yaml_dumps(obj, explicit_start=False, explicit_end=False) - return text.rstrip() - - -def handle(name, cfg, cloud, log, args): - """Handler method activated by cloud-init.""" - - verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) - if args: - # if args are provided (from cmdline) then explicitly set verbose - out_file = args[0] - verbose = True - else: - out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) - - if not verbose: - log.debug(("Skipping module named %s," - " verbose printing disabled"), name) - return - # Clean out some keys that we just don't care about showing... - dump_cfg = copy.deepcopy(cfg) - for k in SKIP_KEYS: - dump_cfg.pop(k, None) - all_keys = list(dump_cfg) - for k in all_keys: - if k.startswith("_"): - dump_cfg.pop(k, None) - # Now dump it... - to_print = StringIO() - to_print.write(_make_header("Config")) - to_print.write(_dumps(dump_cfg)) - to_print.write("\n") - to_print.write(_make_header("MetaData")) - to_print.write(_dumps(cloud.datasource.metadata)) - to_print.write("\n") - to_print.write(_make_header("Misc")) - to_print.write("Datasource: %s\n" % - (type_utils.obj_name(cloud.datasource))) - to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) - to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) - to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) - to_print.write("Locale: %s\n" % (cloud.get_locale())) - to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) - contents = to_print.getvalue() - content_to_file = [] - for line in contents.splitlines(): - line = "ci-info: %s\n" % (line) - content_to_file.append(line) - if out_file: - util.write_file(out_file, "".join(content_to_file), 0o644, "w") - else: - util.multi_log("".join(content_to_file), console=True, stderr=False) diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py deleted file mode 100644 index 3fd2c20f..00000000 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ /dev/null @@ -1,36 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import util - -from cloudinit.settings import PER_ALWAYS - -frequency = PER_ALWAYS - -REJECT_CMD = ['route', 'add', '-host', '169.254.169.254', 'reject'] - - -def handle(name, cfg, _cloud, log, _args): - disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) - if disabled: - util.subp(REJECT_CMD, capture=False) - else: - log.debug(("Skipping module named %s," - " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py deleted file mode 100644 index b642f1f8..00000000 --- a/cloudinit/config/cc_disk_setup.py +++ /dev/null @@ -1,863 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from cloudinit.settings import PER_INSTANCE -from cloudinit import util -import logging -import os -import shlex - -frequency = PER_INSTANCE - -# Define the commands to use -UDEVADM_CMD = util.which('udevadm') -SFDISK_CMD = util.which("sfdisk") -SGDISK_CMD = util.which("sgdisk") -LSBLK_CMD = util.which("lsblk") -BLKID_CMD = util.which("blkid") -BLKDEV_CMD = util.which("blockdev") -WIPEFS_CMD = util.which("wipefs") - -LOG = logging.getLogger(__name__) - - -def handle(_name, cfg, cloud, log, _args): - """ - See doc/examples/cloud-config_disk-setup.txt for documentation on the - format. - """ - disk_setup = cfg.get("disk_setup") - if isinstance(disk_setup, dict): - update_disk_setup_devices(disk_setup, cloud.device_name_to_device) - log.debug("Partitioning disks: %s", str(disk_setup)) - for disk, definition in disk_setup.items(): - if not isinstance(definition, dict): - log.warn("Invalid disk definition for %s" % disk) - continue - - try: - log.debug("Creating new partition table/disk") - util.log_time(logfunc=LOG.debug, - msg="Creating partition on %s" % disk, - func=mkpart, args=(disk, definition)) - except Exception as e: - util.logexc(LOG, "Failed partitioning operation\n%s" % e) - - fs_setup = cfg.get("fs_setup") - if isinstance(fs_setup, list): - log.debug("setting up filesystems: %s", str(fs_setup)) - update_fs_setup_devices(fs_setup, cloud.device_name_to_device) - for definition in fs_setup: - if not isinstance(definition, dict): - log.warn("Invalid file system definition: %s" % definition) - continue - - try: - log.debug("Creating new filesystem.") - device = definition.get('device') - util.log_time(logfunc=LOG.debug, - msg="Creating fs for %s" % device, - func=mkfs, args=(definition,)) - except Exception as e: - util.logexc(LOG, "Failed during filesystem operation\n%s" % e) - - -def update_disk_setup_devices(disk_setup, tformer): - # update 'disk_setup' dictionary anywhere were a device may occur - # update it with the response from 'tformer' - for origname in disk_setup.keys(): - transformed = tformer(origname) - if transformed is None or transformed == origname: - continue - if transformed in disk_setup: - LOG.info("Replacing %s in disk_setup for translation of %s", - origname, transformed) - del disk_setup[transformed] - - disk_setup[transformed] = disk_setup[origname] - disk_setup[transformed]['_origname'] = origname - del disk_setup[origname] - LOG.debug("updated disk_setup device entry '%s' to '%s'", - origname, transformed) - - -def update_fs_setup_devices(disk_setup, tformer): - # update 'fs_setup' dictionary anywhere were a device may occur - # update it with the response from 'tformer' - for definition in disk_setup: - if not isinstance(definition, dict): - LOG.warn("entry in disk_setup not a dict: %s", definition) - continue - - origname = definition.get('device') - - if origname is None: - continue - - (dev, part) = util.expand_dotted_devname(origname) - - tformed = tformer(dev) - if tformed is not None: - dev = tformed - LOG.debug("%s is mapped to disk=%s part=%s", - origname, tformed, part) - definition['_origname'] = origname - definition['device'] = tformed - - if part and 'partition' in definition: - definition['_partition'] = definition['partition'] - definition['partition'] = part - - -def value_splitter(values, start=None): - """ - Returns the key/value pairs of output sent as string - like: FOO='BAR' HOME='127.0.0.1' - """ - _values = shlex.split(values) - if start: - _values = _values[start:] - - for key, value in [x.split('=') for x in _values]: - yield key, value - - -def enumerate_disk(device, nodeps=False): - """ - Enumerate the elements of a child device. - - Parameters: - device: the kernel device name - nodeps : don't enumerate children devices - - Return a dict describing the disk: - type: the entry type, i.e disk or part - fstype: the filesystem type, if it exists - label: file system label, if it exists - name: the device name, i.e. sda - """ - - lsblk_cmd = [LSBLK_CMD, '--pairs', '--output', 'NAME,TYPE,FSTYPE,LABEL', - device] - - if nodeps: - lsblk_cmd.append('--nodeps') - - info = None - try: - info, _err = util.subp(lsblk_cmd) - except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) - - parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] - - for part in parts: - d = { - 'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } - - for key, value in value_splitter(part): - d[key.lower()] = value - - yield d - - -def device_type(device): - """ - Return the device type of the device by calling lsblk. - """ - - for d in enumerate_disk(device, nodeps=True): - if "type" in d: - return d["type"].lower() - return None - - -def is_device_valid(name, partition=False): - """ - Check if the device is a valid device. - """ - d_type = "" - try: - d_type = device_type(name) - except Exception: - LOG.warn("Query against device %s failed" % name) - return False - - if partition and d_type == 'part': - return True - elif not partition and d_type == 'disk': - return True - return False - - -def check_fs(device): - """ - Check if the device has a filesystem on it - - Output of blkid is generally something like: - /dev/sda: LABEL="Backup500G" UUID="..." TYPE="ext4" - - Return values are device, label, type, uuid - """ - out, label, fs_type, uuid = None, None, None, None - - blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device] - try: - out, _err = util.subp(blkid_cmd, rcs=[0, 2]) - except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) - - if out: - if len(out.splitlines()) == 1: - for key, value in value_splitter(out, start=1): - if key.lower() == 'label': - label = value - elif key.lower() == 'type': - fs_type = value - elif key.lower() == 'uuid': - uuid = value - - return label, fs_type, uuid - - -def is_filesystem(device): - """ - Returns true if the device has a file system. - """ - _, fs_type, _ = check_fs(device) - return fs_type - - -def find_device_node(device, fs_type=None, label=None, valid_targets=None, - label_match=True, replace_fs=None): - """ - Find a device that is either matches the spec, or the first - - The return is value is (, ) where the device is the - device to use and the bool is whether the device matches the - fs_type and label. - - Note: This works with GPT partition tables! - """ - # label of None is same as no label - if label is None: - label = "" - - if not valid_targets: - valid_targets = ['disk', 'part'] - - raw_device_used = False - for d in enumerate_disk(device): - - if d['fstype'] == replace_fs and label_match is False: - # We found a device where we want to replace the FS - return ('/dev/%s' % d['name'], False) - - if (d['fstype'] == fs_type and - ((label_match and d['label'] == label) or not label_match)): - # If we find a matching device, we return that - return ('/dev/%s' % d['name'], True) - - if d['type'] in valid_targets: - - if d['type'] != 'disk' or d['fstype']: - raw_device_used = True - - if d['type'] == 'disk': - # Skip the raw disk, its the default - pass - - elif not d['fstype']: - return ('/dev/%s' % d['name'], False) - - if not raw_device_used: - return (device, False) - - LOG.warn("Failed to find device during available device search.") - return (None, False) - - -def is_disk_used(device): - """ - Check if the device is currently used. Returns true if the device - has either a file system or a partition entry - is no filesystem found on the disk. - """ - - # If the child count is higher 1, then there are child nodes - # such as partition or device mapper nodes - if len(list(enumerate_disk(device))) > 1: - return True - - # If we see a file system, then its used - _, check_fstype, _ = check_fs(device) - if check_fstype: - return True - - return False - - -def get_dyn_func(*args): - """ - Call the appropriate function. - - The first value is the template for function name - The second value is the template replacement - The remain values are passed to the function - - For example: get_dyn_func("foo_%s", 'bar', 1, 2, 3,) - would call "foo_bar" with args of 1, 2, 3 - """ - if len(args) < 2: - raise Exception("Unable to determine dynamic funcation name") - - func_name = (args[0] % args[1]) - func_args = args[2:] - - try: - if func_args: - return globals()[func_name](*func_args) - else: - return globals()[func_name] - - except KeyError: - raise Exception("No such function %s to call!" % func_name) - - -def get_mbr_hdd_size(device): - size_cmd = [SFDISK_CMD, '--show-size', device] - size = None - try: - size, _err = util.subp(size_cmd) - except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) - - return int(size.strip()) - - -def get_gpt_hdd_size(device): - out, _ = util.subp([SGDISK_CMD, '-p', device]) - return out.splitlines()[0].split()[2] - - -def get_hdd_size(table_type, device): - """ - Returns the hard disk size. - This works with any disk type, including GPT. - """ - return get_dyn_func("get_%s_hdd_size", table_type, device) - - -def check_partition_mbr_layout(device, layout): - """ - Returns true if the partition layout matches the one on the disk - - Layout should be a list of values. At this time, this only - verifies that the number of partitions and their labels is correct. - """ - - read_parttbl(device) - prt_cmd = [SFDISK_CMD, "-l", device] - try: - out, _err = util.subp(prt_cmd, data="%s\n" % layout) - except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) - - found_layout = [] - for line in out.splitlines(): - _line = line.split() - if len(_line) == 0: - continue - - if device in _line[0]: - # We don't understand extended partitions yet - if _line[-1].lower() in ['extended', 'empty']: - continue - - # Find the partition types - type_label = None - for x in sorted(range(1, len(_line)), reverse=True): - if _line[x].isdigit() and _line[x] != '/': - type_label = _line[x] - break - - found_layout.append(type_label) - return found_layout - - -def check_partition_gpt_layout(device, layout): - prt_cmd = [SGDISK_CMD, '-p', device] - try: - out, _err = util.subp(prt_cmd) - except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) - - out_lines = iter(out.splitlines()) - # Skip header - for line in out_lines: - if line.strip().startswith('Number'): - break - - return [line.strip().split()[-1] for line in out_lines] - - -def check_partition_layout(table_type, device, layout): - """ - See if the partition lay out matches. - - This is future a future proofing function. In order - to add support for other disk layout schemes, add a - function called check_partition_%s_layout - """ - found_layout = get_dyn_func( - "check_partition_%s_layout", table_type, device, layout) - - if isinstance(layout, bool): - # if we are using auto partitioning, or "True" be happy - # if a single partition exists. - if layout and len(found_layout) >= 1: - return True - return False - - else: - if len(found_layout) != len(layout): - return False - else: - # This just makes sure that the number of requested - # partitions and the type labels are right - for x in range(1, len(layout) + 1): - if isinstance(layout[x - 1], tuple): - _, part_type = layout[x] - if int(found_layout[x]) != int(part_type): - return False - return True - - return False - - -def get_partition_mbr_layout(size, layout): - """ - Calculate the layout of the partition table. Partition sizes - are defined as percentage values or a tuple of percentage and - partition type. - - For example: - [ 33, [66: 82] ] - - Defines the first partition to be a size of 1/3 the disk, - while the remaining 2/3's will be of type Linux Swap. - """ - - if not isinstance(layout, list) and isinstance(layout, bool): - # Create a single partition - return "0," - - if ((len(layout) == 0 and isinstance(layout, list)) or - not isinstance(layout, list)): - raise Exception("Partition layout is invalid") - - last_part_num = len(layout) - if last_part_num > 4: - raise Exception("Only simply partitioning is allowed.") - - part_definition = [] - part_num = 0 - for part in layout: - part_type = 83 # Default to Linux - percent = part - part_num += 1 - - if isinstance(part, list): - if len(part) != 2: - raise Exception("Partition was incorrectly defined: %s" % part) - percent, part_type = part - - part_size = int((float(size) * (float(percent) / 100)) / 1024) - - if part_num == last_part_num: - part_definition.append(",,%s" % part_type) - else: - part_definition.append(",%s,%s" % (part_size, part_type)) - - sfdisk_definition = "\n".join(part_definition) - if len(part_definition) > 4: - raise Exception("Calculated partition definition is too big\n%s" % - sfdisk_definition) - - return sfdisk_definition - - -def get_partition_gpt_layout(size, layout): - if isinstance(layout, bool): - return [(None, [0, 0])] - - partition_specs = [] - for partition in layout: - if isinstance(partition, list): - if len(partition) != 2: - raise Exception( - "Partition was incorrectly defined: %s" % partition) - percent, partition_type = partition - else: - percent = partition - partition_type = None - - part_size = int(float(size) * (float(percent) / 100)) - partition_specs.append((partition_type, [0, '+{}'.format(part_size)])) - - # The last partition should use up all remaining space - partition_specs[-1][-1][-1] = 0 - return partition_specs - - -def purge_disk_ptable(device): - # wipe the first and last megabyte of a disk (or file) - # gpt stores partition table both at front and at end. - null = '\0' - start_len = 1024 * 1024 - end_len = 1024 * 1024 - with open(device, "rb+") as fp: - fp.write(null * (start_len)) - fp.seek(-end_len, os.SEEK_END) - fp.write(null * end_len) - fp.flush() - - read_parttbl(device) - - -def purge_disk(device): - """ - Remove parition table entries - """ - - # wipe any file systems first - for d in enumerate_disk(device): - if d['type'] not in ["disk", "crypt"]: - wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] - try: - LOG.info("Purging filesystem on /dev/%s" % d['name']) - util.subp(wipefs_cmd) - except Exception: - raise Exception("Failed FS purge of /dev/%s" % d['name']) - - purge_disk_ptable(device) - - -def get_partition_layout(table_type, size, layout): - """ - Call the appropriate function for creating the table - definition. Returns the table definition - - This is a future proofing function. To add support for - other layouts, simply add a "get_partition_%s_layout" - function. - """ - return get_dyn_func("get_partition_%s_layout", table_type, size, layout) - - -def read_parttbl(device): - """ - Use partprobe instead of 'udevadm'. Partprobe is the only - reliable way to probe the partition table. - """ - blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device] - udev_cmd = [UDEVADM_CMD, 'settle'] - try: - util.subp(udev_cmd) - util.subp(blkdev_cmd) - util.subp(udev_cmd) - except Exception as e: - util.logexc(LOG, "Failed reading the partition table %s" % e) - - -def exec_mkpart_mbr(device, layout): - """ - Break out of mbr partition to allow for future partition - types, i.e. gpt - """ - # Create the partitions - prt_cmd = [SFDISK_CMD, "--Linux", "-uM", device] - try: - util.subp(prt_cmd, data="%s\n" % layout) - except Exception as e: - raise Exception("Failed to partition device %s\n%s" % (device, e)) - - read_parttbl(device) - - -def exec_mkpart_gpt(device, layout): - try: - util.subp([SGDISK_CMD, '-Z', device]) - for index, (partition_type, (start, end)) in enumerate(layout): - index += 1 - util.subp([SGDISK_CMD, - '-n', '{}:{}:{}'.format(index, start, end), device]) - if partition_type is not None: - util.subp( - [SGDISK_CMD, - '-t', '{}:{}'.format(index, partition_type), device]) - except Exception: - LOG.warn("Failed to partition device %s" % device) - raise - - -def exec_mkpart(table_type, device, layout): - """ - Fetches the function for creating the table type. - This allows to dynamically find which function to call. - - Paramaters: - table_type: type of partition table to use - device: the device to work on - layout: layout definition specific to partition table - """ - return get_dyn_func("exec_mkpart_%s", table_type, device, layout) - - -def mkpart(device, definition): - """ - Creates the partition table. - - Parameters: - definition: dictionary describing how to create the partition. - - The following are supported values in the dict: - overwrite: Should the partition table be created regardless - of any pre-exisiting data? - layout: the layout of the partition table - table_type: Which partition table to use, defaults to MBR - device: the device to work on. - """ - # ensure that we get a real device rather than a symbolic link - device = os.path.realpath(device) - - LOG.debug("Checking values for %s definition" % device) - overwrite = definition.get('overwrite', False) - layout = definition.get('layout', False) - table_type = definition.get('table_type', 'mbr') - - # Check if the default device is a partition or not - LOG.debug("Checking against default devices") - - if (isinstance(layout, bool) and not layout) or not layout: - LOG.debug("Device is not to be partitioned, skipping") - return # Device is not to be partitioned - - # This prevents you from overwriting the device - LOG.debug("Checking if device %s is a valid device", device) - if not is_device_valid(device): - raise Exception("Device %s is not a disk device!", device) - - # Remove the partition table entries - if isinstance(layout, str) and layout.lower() == "remove": - LOG.debug("Instructed to remove partition table entries") - purge_disk(device) - return - - LOG.debug("Checking if device layout matches") - if check_partition_layout(table_type, device, layout): - LOG.debug("Device partitioning layout matches") - return True - - LOG.debug("Checking if device is safe to partition") - if not overwrite and (is_disk_used(device) or is_filesystem(device)): - LOG.debug("Skipping partitioning on configured device %s" % device) - return - - LOG.debug("Checking for device size") - device_size = get_hdd_size(table_type, device) - - LOG.debug("Calculating partition layout") - part_definition = get_partition_layout(table_type, device_size, layout) - LOG.debug(" Layout is: %s" % part_definition) - - LOG.debug("Creating partition table on %s", device) - exec_mkpart(table_type, device, part_definition) - - LOG.debug("Partition table created for %s", device) - - -def lookup_force_flag(fs): - """ - A force flag might be -F or -F, this look it up - """ - flags = { - 'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - } - - if 'ext' in fs.lower(): - fs = 'ext' - - if fs.lower() in flags: - return flags[fs] - - LOG.warn("Force flag for %s is unknown." % fs) - return '' - - -def mkfs(fs_cfg): - """ - Create a file system on the device. - - label: defines the label to use on the device - fs_cfg: defines how the filesystem is to look - The following values are required generally: - device: which device or cloud defined default_device - filesystem: which file system type - overwrite: indiscriminately create the file system - partition: when device does not define a partition, - setting this to a number will mean - device + partition. When set to 'auto', the - first free device or the first device which - matches both label and type will be used. - - 'any' means the first filesystem that matches - on the device. - - When 'cmd' is provided then no other parameter is required. - """ - label = fs_cfg.get('label') - device = fs_cfg.get('device') - partition = str(fs_cfg.get('partition', 'any')) - fs_type = fs_cfg.get('filesystem') - fs_cmd = fs_cfg.get('cmd', []) - fs_opts = fs_cfg.get('extra_opts', []) - fs_replace = fs_cfg.get('replace_fs', False) - overwrite = fs_cfg.get('overwrite', False) - - # ensure that we get a real device rather than a symbolic link - device = os.path.realpath(device) - - # This allows you to define the default ephemeral or swap - LOG.debug("Checking %s against default devices", device) - - if not partition or partition.isdigit(): - # Handle manual definition of partition - if partition.isdigit(): - device = "%s%s" % (device, partition) - LOG.debug("Manual request of partition %s for %s", - partition, device) - - # Check to see if the fs already exists - LOG.debug("Checking device %s", device) - check_label, check_fstype, _ = check_fs(device) - LOG.debug("Device %s has %s %s", device, check_label, check_fstype) - - if check_label == label and check_fstype == fs_type: - LOG.debug("Existing file system found at %s", device) - - if not overwrite: - LOG.debug("Device %s has required file system", device) - return - else: - LOG.warn("Destroying filesystem on %s", device) - - else: - LOG.debug("Device %s is cleared for formating", device) - - elif partition and str(partition).lower() in ('auto', 'any'): - # For auto devices, we match if the filesystem does exist - odevice = device - LOG.debug("Identifying device to create %s filesytem on", label) - - # any mean pick the first match on the device with matching fs_type - label_match = True - if partition.lower() == 'any': - label_match = False - - device, reuse = find_device_node(device, fs_type=fs_type, label=label, - label_match=label_match, - replace_fs=fs_replace) - LOG.debug("Automatic device for %s identified as %s", odevice, device) - - if reuse: - LOG.debug("Found filesystem match, skipping formating.") - return - - if not reuse and fs_replace and device: - LOG.debug("Replacing file system on %s as instructed." % device) - - if not device: - LOG.debug("No device aviable that matches request. " - "Skipping fs creation for %s", fs_cfg) - return - elif not partition or str(partition).lower() == 'none': - LOG.debug("Using the raw device to place filesystem %s on" % label) - - else: - LOG.debug("Error in device identification handling.") - return - - LOG.debug("File system %s will be created on %s", label, device) - - # Make sure the device is defined - if not device: - LOG.warn("Device is not known: %s", device) - return - - # Check that we can create the FS - if not (fs_type or fs_cmd): - raise Exception("No way to create filesystem '%s'. fs_type or fs_cmd " - "must be set.", label) - - # Create the commands - if fs_cmd: - fs_cmd = fs_cfg['cmd'] % { - 'label': label, - 'filesystem': fs_type, - 'device': device, - } - else: - # Find the mkfs command - mkfs_cmd = util.which("mkfs.%s" % fs_type) - if not mkfs_cmd: - mkfs_cmd = util.which("mk%s" % fs_type) - - if not mkfs_cmd: - LOG.warn("Cannot create fstype '%s'. No mkfs.%s command", fs_type, - fs_type) - return - - fs_cmd = [mkfs_cmd, device] - - if label: - fs_cmd.extend(["-L", label]) - - # File systems that support the -F flag - if overwrite or device_type(device) == "disk": - fs_cmd.append(lookup_force_flag(fs_type)) - - # Add the extends FS options - if fs_opts: - fs_cmd.extend(fs_opts) - - LOG.debug("Creating file system %s on %s", label, device) - LOG.debug(" Using cmd: %s", " ".join(fs_cmd)) - try: - util.subp(fs_cmd) - except Exception as e: - raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py deleted file mode 100644 index 98828b9e..00000000 --- a/cloudinit/config/cc_emit_upstart.py +++ /dev/null @@ -1,69 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import log as logging -from cloudinit.settings import PER_ALWAYS -from cloudinit import util - -frequency = PER_ALWAYS - -distros = ['ubuntu', 'debian'] -LOG = logging.getLogger(__name__) - - -def is_upstart_system(): - if not os.path.isfile("/sbin/initctl"): - LOG.debug("no /sbin/initctl located") - return False - - myenv = os.environ.copy() - if 'UPSTART_SESSION' in myenv: - del myenv['UPSTART_SESSION'] - check_cmd = ['initctl', 'version'] - try: - (out, err) = util.subp(check_cmd, env=myenv) - return 'upstart' in out - except util.ProcessExecutionError as e: - LOG.debug("'%s' returned '%s', not using upstart", - ' '.join(check_cmd), e.exit_code) - return False - - -def handle(name, _cfg, cloud, log, args): - event_names = args - if not event_names: - # Default to the 'cloud-config' - # event for backwards compat. - event_names = ['cloud-config'] - - if not is_upstart_system(): - log.debug("not upstart system, '%s' disabled", name) - return - - cfgpath = cloud.paths.get_ipath_cur("cloud_config") - for n in event_names: - cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] - try: - util.subp(cmd) - except Exception as e: - # TODO(harlowja), use log exception from utils?? - log.warn("Emission of upstart event %s failed due to: %s", n, e) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py deleted file mode 100644 index 545fee22..00000000 --- a/cloudinit/config/cc_fan.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -fan module allows configuration of Ubuntu Fan - https://wiki.ubuntu.com/FanNetworking - -Example config: - #cloud-config - fan: - config: | - # fan 240 - 10.0.0.0/8 eth0/16 dhcp - 10.0.0.0/8 eth1/16 dhcp off - # fan 241 - 241.0.0.0/8 eth0/16 dhcp - config_path: /etc/network/fan - -If cloud-init sees a 'fan' entry in cloud-config it will - a.) write 'config_path' with the contents - b.) install the package 'ubuntu-fan' if it is not installed - c.) ensure the service is started (or restarted if was previously running) -""" - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -LOG = logging.getLogger(__name__) - -frequency = PER_INSTANCE - -BUILTIN_CFG = { - 'config': None, - 'config_path': '/etc/network/fan', -} - - -def stop_update_start(service, config_file, content, systemd=False): - if systemd: - cmds = {'stop': ['systemctl', 'stop', service], - 'start': ['systemctl', 'start', service], - 'enable': ['systemctl', 'enable', service]} - else: - cmds = {'stop': ['service', 'stop'], - 'start': ['service', 'start']} - - def run(cmd, msg): - try: - return util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: - LOG.warn("failed: %s (%s): %s", service, cmd, e) - return False - - stop_failed = not run(cmds['stop'], msg='stop %s' % service) - if not content.endswith('\n'): - content += '\n' - util.write_file(config_file, content, omode="w") - - ret = run(cmds['start'], msg='start %s' % service) - if ret and stop_failed: - LOG.warn("success: %s started", service) - - if 'enable' in cmds: - ret = run(cmds['enable'], msg='enable %s' % service) - - return ret - - -def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('fan') - if not cfgin: - cfgin = {} - mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - - if not mycfg.get('config'): - LOG.debug("%s: no 'fan' config entry. disabling", name) - return - - util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") - distro = cloud.distro - if not util.which('fanctl'): - distro.install_packages(['ubuntu-fan']) - - stop_update_start( - service='ubuntu-fan', config_file=mycfg.get('config_path'), - content=mycfg.get('config'), systemd=distro.uses_systemd()) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py deleted file mode 100644 index c9021eb1..00000000 --- a/cloudinit/config/cc_final_message.py +++ /dev/null @@ -1,73 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import templater -from cloudinit import util -from cloudinit import version - -from cloudinit.settings import PER_ALWAYS - -frequency = PER_ALWAYS - -# Jinja formated default message -FINAL_MESSAGE_DEF = ( - "## template: jinja\n" - "Cloud-init v. {{version}} finished at {{timestamp}}." - " Datasource {{datasource}}. Up {{uptime}} seconds" -) - - -def handle(_name, cfg, cloud, log, args): - - msg_in = '' - if len(args) != 0: - msg_in = str(args[0]) - else: - msg_in = util.get_cfg_option_str(cfg, "final_message", "") - - msg_in = msg_in.strip() - if not msg_in: - msg_in = FINAL_MESSAGE_DEF - - uptime = util.uptime() - ts = util.time_rfc2822() - cver = version.version_string() - try: - subs = { - 'uptime': uptime, - 'timestamp': ts, - 'version': cver, - 'datasource': str(cloud.datasource), - } - subs.update(dict([(k.upper(), v) for k, v in subs.items()])) - util.multi_log("%s\n" % (templater.render_string(msg_in, subs)), - console=False, stderr=True, log=log) - except Exception: - util.logexc(log, "Failed to render final message template") - - boot_fin_fn = cloud.paths.boot_finished - try: - contents = "%s - %s - v. %s\n" % (uptime, ts, cver) - util.write_file(boot_fin_fn, contents) - except Exception: - util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) - - if cloud.datasource.is_disconnected: - log.warn("Used fallback datasource") diff --git a/cloudinit/config/cc_foo.py b/cloudinit/config/cc_foo.py deleted file mode 100644 index 95aab4dd..00000000 --- a/cloudinit/config/cc_foo.py +++ /dev/null @@ -1,52 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.settings import PER_INSTANCE - -# Modules are expected to have the following attributes. -# 1. A required 'handle' method which takes the following params. -# a) The name will not be this files name, but instead -# the name specified in configuration (which is the name -# which will be used to find this module). -# b) A configuration object that is the result of the merging -# of cloud configs configuration with legacy configuration -# as well as any datasource provided configuration -# c) A cloud object that can be used to access various -# datasource and paths for the given distro and data provided -# by the various datasource instance types. -# d) A argument list that may or may not be empty to this module. -# Typically those are from module configuration where the module -# is defined with some extra configuration that will eventually -# be translated from yaml into arguments to this module. -# 2. A optional 'frequency' that defines how often this module should be ran. -# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not -# provided PER_INSTANCE will be assumed. -# See settings.py for these constants. -# 3. A optional 'distros' array/set/tuple that defines the known distros -# this module will work with (if not all of them). This is used to write -# a warning out if a module is being ran on a untested distribution for -# informational purposes. If non existent all distros are assumed and -# no warning occurs. - -frequency = PER_INSTANCE - - -def handle(name, _cfg, _cloud, log, _args): - log.debug("Hi from module %s", name) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py deleted file mode 100644 index 40560f11..00000000 --- a/cloudinit/config/cc_growpart.py +++ /dev/null @@ -1,300 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import re -import stat - -from cloudinit import log as logging -from cloudinit.settings import PER_ALWAYS -from cloudinit import util - -frequency = PER_ALWAYS - -DEFAULT_CONFIG = { - 'mode': 'auto', - 'devices': ['/'], - 'ignore_growroot_disabled': False, -} - - -class RESIZE(object): - SKIPPED = "SKIPPED" - CHANGED = "CHANGED" - NOCHANGE = "NOCHANGE" - FAILED = "FAILED" - - -LOG = logging.getLogger(__name__) - - -def resizer_factory(mode): - resize_class = None - if mode == "auto": - for (_name, resizer) in RESIZERS: - cur = resizer() - if cur.available(): - resize_class = cur - break - - if not resize_class: - raise ValueError("No resizers available") - - else: - mmap = {} - for (k, v) in RESIZERS: - mmap[k] = v - - if mode not in mmap: - raise TypeError("unknown resize mode %s" % mode) - - mclass = mmap[mode]() - if mclass.available(): - resize_class = mclass - - if not resize_class: - raise ValueError("mode %s not available" % mode) - - return resize_class - - -class ResizeFailedException(Exception): - pass - - -class ResizeGrowPart(object): - def available(self): - myenv = os.environ.copy() - myenv['LANG'] = 'C' - - try: - (out, _err) = util.subp(["growpart", "--help"], env=myenv) - if re.search(r"--update\s+", out, re.DOTALL): - return True - - except util.ProcessExecutionError: - pass - return False - - def resize(self, diskdev, partnum, partdev): - before = get_size(partdev) - try: - util.subp(["growpart", '--dry-run', diskdev, partnum]) - except util.ProcessExecutionError as e: - if e.exit_code != 1: - util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", - diskdev, partnum) - raise ResizeFailedException(e) - return (before, before) - - try: - util.subp(["growpart", diskdev, partnum]) - except util.ProcessExecutionError as e: - util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) - raise ResizeFailedException(e) - - return (before, get_size(partdev)) - - -class ResizeGpart(object): - def available(self): - if not util.which('gpart'): - return False - return True - - def resize(self, diskdev, partnum, partdev): - """ - GPT disks store metadata at the beginning (primary) and at the - end (secondary) of the disk. When launching an image with a - larger disk compared to the original image, the secondary copy - is lost. Thus, the metadata will be marked CORRUPT, and need to - be recovered. - """ - try: - util.subp(["gpart", "recover", diskdev]) - except util.ProcessExecutionError as e: - if e.exit_code != 0: - util.logexc(LOG, "Failed: gpart recover %s", diskdev) - raise ResizeFailedException(e) - - before = get_size(partdev) - try: - util.subp(["gpart", "resize", "-i", partnum, diskdev]) - except util.ProcessExecutionError as e: - util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) - raise ResizeFailedException(e) - - # Since growing the FS requires a reboot, make sure we reboot - # first when this module has finished. - open('/var/run/reboot-required', 'a').close() - - return (before, get_size(partdev)) - - -def get_size(filename): - fd = os.open(filename, os.O_RDONLY) - try: - return os.lseek(fd, 0, os.SEEK_END) - finally: - os.close(fd) - - -def device_part_info(devpath): - # convert an entry in /dev/ to parent disk and partition number - - # input of /dev/vdb or /dev/disk/by-label/foo - # rpath is hopefully a real-ish path in /dev (vda, sdb..) - rpath = os.path.realpath(devpath) - - bname = os.path.basename(rpath) - syspath = "/sys/class/block/%s" % bname - - # FreeBSD doesn't know of sysfs so just get everything we need from - # the device, like /dev/vtbd0p2. - if util.system_info()["platform"].startswith('FreeBSD'): - m = re.search('^(/dev/.+)p([0-9])$', devpath) - return (m.group(1), m.group(2)) - - if not os.path.exists(syspath): - raise ValueError("%s had no syspath (%s)" % (devpath, syspath)) - - ptpath = os.path.join(syspath, "partition") - if not os.path.exists(ptpath): - raise TypeError("%s not a partition" % devpath) - - ptnum = util.load_file(ptpath).rstrip() - - # for a partition, real syspath is something like: - # /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1 - rsyspath = os.path.realpath(syspath) - disksyspath = os.path.dirname(rsyspath) - - diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip() - diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin) - - # diskdevpath has something like 253:0 - # and udev has put links in /dev/block/253:0 to the device name in /dev/ - return (diskdevpath, ptnum) - - -def devent2dev(devent): - if devent.startswith("/dev/"): - return devent - else: - result = util.get_mount_info(devent) - if not result: - raise ValueError("Could not determine device of '%s' % dev_ent") - return result[0] - - -def resize_devices(resizer, devices): - # returns a tuple of tuples containing (entry-in-devices, action, message) - info = [] - for devent in devices: - try: - blockdev = devent2dev(devent) - except ValueError as e: - info.append((devent, RESIZE.SKIPPED, - "unable to convert to device: %s" % e,)) - continue - - try: - statret = os.stat(blockdev) - except OSError as e: - info.append((devent, RESIZE.SKIPPED, - "stat of '%s' failed: %s" % (blockdev, e),)) - continue - - if (not stat.S_ISBLK(statret.st_mode) and - not stat.S_ISCHR(statret.st_mode)): - info.append((devent, RESIZE.SKIPPED, - "device '%s' not a block device" % blockdev,)) - continue - - try: - (disk, ptnum) = device_part_info(blockdev) - except (TypeError, ValueError) as e: - info.append((devent, RESIZE.SKIPPED, - "device_part_info(%s) failed: %s" % (blockdev, e),)) - continue - - try: - (old, new) = resizer.resize(disk, ptnum, blockdev) - if old == new: - info.append((devent, RESIZE.NOCHANGE, - "no change necessary (%s, %s)" % (disk, ptnum),)) - else: - info.append((devent, RESIZE.CHANGED, - "changed (%s, %s) from %s to %s" % - (disk, ptnum, old, new),)) - - except ResizeFailedException as e: - info.append((devent, RESIZE.FAILED, - "failed to resize: disk=%s, ptnum=%s: %s" % - (disk, ptnum, e),)) - - return info - - -def handle(_name, cfg, _cloud, log, _args): - if 'growpart' not in cfg: - log.debug("No 'growpart' entry in cfg. Using default: %s" % - DEFAULT_CONFIG) - cfg['growpart'] = DEFAULT_CONFIG - - mycfg = cfg.get('growpart') - if not isinstance(mycfg, dict): - log.warn("'growpart' in config was not a dict") - return - - mode = mycfg.get('mode', "auto") - if util.is_false(mode): - log.debug("growpart disabled: mode=%s" % mode) - return - - if util.is_false(mycfg.get('ignore_growroot_disabled', False)): - if os.path.isfile("/etc/growroot-disabled"): - log.debug("growpart disabled: /etc/growroot-disabled exists") - log.debug("use ignore_growroot_disabled to ignore") - return - - devices = util.get_cfg_option_list(mycfg, "devices", ["/"]) - if not len(devices): - log.debug("growpart: empty device list") - return - - try: - resizer = resizer_factory(mode) - except (ValueError, TypeError) as e: - log.debug("growpart unable to find resizer for '%s': %s" % (mode, e)) - if mode != "auto": - raise e - return - - resized = util.log_time(logfunc=log.debug, msg="resize_devices", - func=resize_devices, args=(resizer, devices)) - for (entry, action, msg) in resized: - if action == RESIZE.CHANGED: - log.info("'%s' resized: %s" % (entry, msg)) - else: - log.debug("'%s' %s: %s" % (entry, action, msg)) - -RESIZERS = (('growpart', ResizeGrowPart), ('gpart', ResizeGpart)) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py deleted file mode 100644 index 156722d9..00000000 --- a/cloudinit/config/cc_grub_dpkg.py +++ /dev/null @@ -1,73 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -distros = ['ubuntu', 'debian'] - - -def handle(name, cfg, _cloud, log, _args): - - mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) - if not mycfg: - mycfg = {} - - enabled = mycfg.get('enabled', True) - if util.is_false(enabled): - log.debug("%s disabled by config grub_dpkg/enabled=%s", name, enabled) - return - - idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str( - mycfg, "grub-pc/install_devices_empty", None) - - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs is None: - idevs = "" - if idevs_empty is None: - idevs_empty = "true" - else: - if idevs_empty is None: - idevs_empty = "false" - if idevs is None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/xvda", - "/dev/sda1", "/dev/vda1", "/dev/xvda1"): - if os.path.exists(dev): - idevs = dev - break - - # now idevs and idevs_empty are set to determined values - # or, those set by user - - dconf_sel = (("grub-pc grub-pc/install_devices string %s\n" - "grub-pc grub-pc/install_devices_empty boolean %s\n") % - (idevs, idevs_empty)) - - log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) - - try: - util.subp(['debconf-set-selections'], dconf_sel) - except Exception: - util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py deleted file mode 100644 index 9a02f056..00000000 --- a/cloudinit/config/cc_keys_to_console.py +++ /dev/null @@ -1,62 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -frequency = PER_INSTANCE - -# This is a tool that cloud init provides -HELPER_TOOL_TPL = '%s/cloud-init/write-ssh-key-fingerprints' - - -def _get_helper_tool_path(distro): - try: - base_lib = distro.usr_lib_exec - except AttributeError: - base_lib = '/usr/lib' - return HELPER_TOOL_TPL % base_lib - - -def handle(name, cfg, cloud, log, _args): - helper_path = _get_helper_tool_path(cloud.distro) - if not os.path.exists(helper_path): - log.warn(("Unable to activate module %s," - " helper tool not found at %s"), name, helper_path) - return - - fp_blacklist = util.get_cfg_option_list(cfg, - "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list(cfg, - "ssh_key_console_blacklist", - ["ssh-dss"]) - - try: - cmd = [helper_path] - cmd.append(','.join(fp_blacklist)) - cmd.append(','.join(key_blacklist)) - (stdout, _stderr) = util.subp(cmd) - util.multi_log("%s\n" % (stdout.strip()), - stderr=False, console=True) - except Exception: - log.warn("Writing keys to the system console failed!") - raise diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py deleted file mode 100644 index 68fcb27f..00000000 --- a/cloudinit/config/cc_landscape.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from six import StringIO - -from configobj import ConfigObj - -from cloudinit import type_utils -from cloudinit import util - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - -LSC_CLIENT_CFG_FILE = "/etc/landscape/client.conf" -LS_DEFAULT_FILE = "/etc/default/landscape-client" - -distros = ['ubuntu'] - -# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 -LSC_BUILTIN_CFG = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", - } -} - - -def handle(_name, cfg, cloud, log, _args): - """ - Basically turn a top level 'landscape' entry with a 'client' dict - and render it to ConfigObj format under '[client]' section in - /etc/landscape/client.conf - """ - - ls_cloudcfg = cfg.get("landscape", {}) - - if not isinstance(ls_cloudcfg, (dict)): - raise RuntimeError(("'landscape' key existed in config," - " but not a dictionary type," - " is a %s instead"), - type_utils.obj_name(ls_cloudcfg)) - if not ls_cloudcfg: - return - - cloud.distro.install_packages(('landscape-client',)) - - merge_data = [ - LSC_BUILTIN_CFG, - LSC_CLIENT_CFG_FILE, - ls_cloudcfg, - ] - merged = merge_together(merge_data) - contents = StringIO() - merged.write(contents) - - util.ensure_dir(os.path.dirname(LSC_CLIENT_CFG_FILE)) - util.write_file(LSC_CLIENT_CFG_FILE, contents.getvalue()) - log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE) - - util.write_file(LS_DEFAULT_FILE, "RUN=1\n") - util.subp(["service", "landscape-client", "restart"]) - - -def merge_together(objs): - """ - merge together ConfigObj objects or things that ConfigObj() will take in - later entries override earlier - """ - cfg = ConfigObj({}) - for obj in objs: - if not obj: - continue - if isinstance(obj, ConfigObj): - cfg.merge(obj) - else: - cfg.merge(ConfigObj(obj)) - return cfg diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py deleted file mode 100644 index bbe5fcae..00000000 --- a/cloudinit/config/cc_locale.py +++ /dev/null @@ -1,37 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import util - - -def handle(name, cfg, cloud, log, args): - if len(args) != 0: - locale = args[0] - else: - locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - - if util.is_false(locale): - log.debug("Skipping module named %s, disabled by config: %s", - name, locale) - return - - log.debug("Setting locale to %s", locale) - locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile") - cloud.distro.apply_locale(locale, locale_cfgfile) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py deleted file mode 100644 index 70d4e7c3..00000000 --- a/cloudinit/config/cc_lxd.py +++ /dev/null @@ -1,177 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# -# Author: Wesley Wiedenmeier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -This module initializes lxd using 'lxd init' - -Example config: - #cloud-config - lxd: - init: - network_address: - network_port: - storage_backend: - storage_create_device: - storage_create_loop: - storage_pool: - trust_password: - bridge: - mode: - name: - ipv4_address: - ipv4_netmask: - ipv4_dhcp_first: - ipv4_dhcp_last: - ipv4_dhcp_leases: - ipv4_nat: - ipv6_address: - ipv6_netmask: - ipv6_nat: - domain: -""" - -from cloudinit import util - - -def handle(name, cfg, cloud, log, args): - # Get config - lxd_cfg = cfg.get('lxd') - if not lxd_cfg: - log.debug("Skipping module named %s, not present or disabled by cfg", - name) - return - if not isinstance(lxd_cfg, dict): - log.warn("lxd config must be a dictionary. found a '%s'", - type(lxd_cfg)) - return - - # Grab the configuration - init_cfg = lxd_cfg.get('init') - if not isinstance(init_cfg, dict): - log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) - init_cfg = {} - - bridge_cfg = lxd_cfg.get('bridge') - if not isinstance(bridge_cfg, dict): - log.warn("lxd/bridge config must be a dictionary. found a '%s'", - type(bridge_cfg)) - bridge_cfg = {} - - # Install the needed packages - packages = [] - if not util.which("lxd"): - packages.append('lxd') - - if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): - packages.append('zfs') - - if len(packages): - try: - cloud.distro.install_packages(packages) - except util.ProcessExecutionError as exc: - log.warn("failed to install packages %s: %s", packages, exc) - return - - # Set up lxd if init config is given - if init_cfg: - init_keys = ( - 'network_address', 'network_port', 'storage_backend', - 'storage_create_device', 'storage_create_loop', - 'storage_pool', 'trust_password') - cmd = ['lxd', 'init', '--auto'] - for k in init_keys: - if init_cfg.get(k): - cmd.extend(["--%s=%s" % - (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) - - # Set up lxd-bridge if bridge config is given - dconf_comm = "debconf-communicate" - if bridge_cfg and util.which(dconf_comm): - debconf = bridge_to_debconf(bridge_cfg) - - # Update debconf database - try: - log.debug("Setting lxd debconf via " + dconf_comm) - data = "\n".join(["set %s %s" % (k, v) - for k, v in debconf.items()]) + "\n" - util.subp(['debconf-communicate'], data) - except Exception: - util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) - - # Remove the existing configuration file (forces re-generation) - util.del_file("/etc/default/lxd-bridge") - - # Run reconfigure - log.debug("Running dpkg-reconfigure for lxd") - util.subp(['dpkg-reconfigure', 'lxd', - '--frontend=noninteractive']) - elif bridge_cfg: - raise RuntimeError( - "Unable to configure lxd bridge without %s." + dconf_comm) - - -def bridge_to_debconf(bridge_cfg): - debconf = {} - - if bridge_cfg.get("mode") == "none": - debconf["lxd/setup-bridge"] = "false" - debconf["lxd/bridge-name"] = "" - - elif bridge_cfg.get("mode") == "existing": - debconf["lxd/setup-bridge"] = "false" - debconf["lxd/use-existing-bridge"] = "true" - debconf["lxd/bridge-name"] = bridge_cfg.get("name") - - elif bridge_cfg.get("mode") == "new": - debconf["lxd/setup-bridge"] = "true" - if bridge_cfg.get("name"): - debconf["lxd/bridge-name"] = bridge_cfg.get("name") - - if bridge_cfg.get("ipv4_address"): - debconf["lxd/bridge-ipv4"] = "true" - debconf["lxd/bridge-ipv4-address"] = \ - bridge_cfg.get("ipv4_address") - debconf["lxd/bridge-ipv4-netmask"] = \ - bridge_cfg.get("ipv4_netmask") - debconf["lxd/bridge-ipv4-dhcp-first"] = \ - bridge_cfg.get("ipv4_dhcp_first") - debconf["lxd/bridge-ipv4-dhcp-last"] = \ - bridge_cfg.get("ipv4_dhcp_last") - debconf["lxd/bridge-ipv4-dhcp-leases"] = \ - bridge_cfg.get("ipv4_dhcp_leases") - debconf["lxd/bridge-ipv4-nat"] = \ - bridge_cfg.get("ipv4_nat", "true") - - if bridge_cfg.get("ipv6_address"): - debconf["lxd/bridge-ipv6"] = "true" - debconf["lxd/bridge-ipv6-address"] = \ - bridge_cfg.get("ipv6_address") - debconf["lxd/bridge-ipv6-netmask"] = \ - bridge_cfg.get("ipv6_netmask") - debconf["lxd/bridge-ipv6-nat"] = \ - bridge_cfg.get("ipv6_nat", "false") - - if bridge_cfg.get("domain"): - debconf["lxd/bridge-domain"] = bridge_cfg.get("domain") - - else: - raise Exception("invalid bridge mode \"%s\"" % bridge_cfg.get("mode")) - - return debconf diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py deleted file mode 100644 index ada535f8..00000000 --- a/cloudinit/config/cc_mcollective.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Marc Cluet -# Based on code by Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import six -from six import BytesIO - -# Used since this can maintain comments -# and doesn't need a top level section -from configobj import ConfigObj - -from cloudinit import log as logging -from cloudinit import util - -PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" -PRICERT_FILE = "/etc/mcollective/ssl/server-private.pem" -SERVER_CFG = '/etc/mcollective/server.cfg' - -LOG = logging.getLogger(__name__) - - -def configure(config, server_cfg=SERVER_CFG, - pubcert_file=PUBCERT_FILE, pricert_file=PRICERT_FILE): - # Read server.cfg values from the - # original file in order to be able to mix the rest up - try: - mcollective_config = ConfigObj(server_cfg, file_error=True) - existed = True - except IOError: - LOG.debug("Did not find file %s", server_cfg) - mcollective_config = ConfigObj() - existed = False - - for (cfg_name, cfg) in config.items(): - if cfg_name == 'public-cert': - util.write_file(pubcert_file, cfg, mode=0o644) - mcollective_config[ - 'plugin.ssl_server_public'] = pubcert_file - mcollective_config['securityprovider'] = 'ssl' - elif cfg_name == 'private-cert': - util.write_file(pricert_file, cfg, mode=0o600) - mcollective_config[ - 'plugin.ssl_server_private'] = pricert_file - mcollective_config['securityprovider'] = 'ssl' - else: - if isinstance(cfg, six.string_types): - # Just set it in the 'main' section - mcollective_config[cfg_name] = cfg - elif isinstance(cfg, (dict)): - # Iterate through the config items, create a section if - # it is needed and then add/or create items as needed - if cfg_name not in mcollective_config.sections: - mcollective_config[cfg_name] = {} - for (o, v) in cfg.items(): - mcollective_config[cfg_name][o] = v - else: - # Otherwise just try to convert it to a string - mcollective_config[cfg_name] = str(cfg) - - if existed: - # We got all our config as wanted we'll rename - # the previous server.cfg and create our new one - util.rename(server_cfg, "%s.old" % (server_cfg)) - - # Now we got the whole file, write to disk... - contents = BytesIO() - mcollective_config.write(contents) - util.write_file(server_cfg, contents.getvalue(), mode=0o644) - - -def handle(name, cfg, cloud, log, _args): - - # If there isn't a mcollective key in the configuration don't do anything - if 'mcollective' not in cfg: - log.debug(("Skipping module named %s, " - "no 'mcollective' key in configuration"), name) - return - - mcollective_cfg = cfg['mcollective'] - - # Start by installing the mcollective package ... - cloud.distro.install_packages(("mcollective",)) - - # ... and then update the mcollective configuration - if 'conf' in mcollective_cfg: - configure(config=mcollective_cfg['conf']) - - # restart mcollective to handle updated config - util.subp(['service', 'mcollective', 'restart'], capture=False) diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py deleted file mode 100644 index facaa538..00000000 --- a/cloudinit/config/cc_migrator.py +++ /dev/null @@ -1,85 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import shutil - -from cloudinit import helpers -from cloudinit import util - -from cloudinit.settings import PER_ALWAYS - -frequency = PER_ALWAYS - - -def _migrate_canon_sems(cloud): - paths = (cloud.paths.get_ipath('sem'), cloud.paths.get_cpath('sem')) - am_adjusted = 0 - for sem_path in paths: - if not sem_path or not os.path.exists(sem_path): - continue - for p in os.listdir(sem_path): - full_path = os.path.join(sem_path, p) - if os.path.isfile(full_path): - (name, ext) = os.path.splitext(p) - canon_name = helpers.canon_sem_name(name) - if canon_name != name: - new_path = os.path.join(sem_path, canon_name + ext) - shutil.move(full_path, new_path) - am_adjusted += 1 - return am_adjusted - - -def _migrate_legacy_sems(cloud, log): - legacy_adjust = { - 'apt-update-upgrade': [ - 'apt-configure', - 'package-update-upgrade-install', - ], - } - paths = (cloud.paths.get_ipath('sem'), cloud.paths.get_cpath('sem')) - for sem_path in paths: - if not sem_path or not os.path.exists(sem_path): - continue - sem_helper = helpers.FileSemaphores(sem_path) - for (mod_name, migrate_to) in legacy_adjust.items(): - possibles = [mod_name, helpers.canon_sem_name(mod_name)] - old_exists = [] - for p in os.listdir(sem_path): - (name, _ext) = os.path.splitext(p) - if name in possibles and os.path.isfile(p): - old_exists.append(p) - for p in old_exists: - util.del_file(os.path.join(sem_path, p)) - (_name, freq) = os.path.splitext(p) - for m in migrate_to: - log.debug("Migrating %s => %s with the same frequency", - p, m) - with sem_helper.lock(m, freq): - pass - - -def handle(name, cfg, cloud, log, _args): - do_migrate = util.get_cfg_option_str(cfg, "migrate", True) - if not util.translate_bool(do_migrate): - log.debug("Skipping module named %s, migration disabled", name) - return - sems_moved = _migrate_canon_sems(cloud) - log.debug("Migrated %s semaphore files to there canonicalized names", - sems_moved) - _migrate_legacy_sems(cloud, log) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py deleted file mode 100644 index 2b981935..00000000 --- a/cloudinit/config/cc_mounts.py +++ /dev/null @@ -1,405 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from string import whitespace - -import logging -import os.path -import re - -from cloudinit import type_utils -from cloudinit import util - -# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 -DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" -DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) -WS = re.compile("[%s]+" % (whitespace)) -FSTAB_PATH = "/etc/fstab" - -LOG = logging.getLogger(__name__) - - -def is_meta_device_name(name): - # return true if this is a metadata service name - if name in ["ami", "root", "swap"]: - return True - # names 'ephemeral0' or 'ephemeral1' - # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' - for enumname in ("ephemeral", "ebs"): - if name.startswith(enumname) and name.find(":") == -1: - return True - return False - - -def _get_nth_partition_for_device(device_path, partition_number): - potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), - '-part%s' % (partition_number,)] - for suffix in potential_suffixes: - potential_partition_device = '%s%s' % (device_path, suffix) - if os.path.exists(potential_partition_device): - return potential_partition_device - return None - - -def _is_block_device(device_path, partition_path=None): - device_name = os.path.realpath(device_path).split('/')[-1] - sys_path = os.path.join('/sys/block/', device_name) - if partition_path is not None: - sys_path = os.path.join( - sys_path, os.path.realpath(partition_path).split('/')[-1]) - return os.path.exists(sys_path) - - -def sanitize_devname(startname, transformer, log): - log.debug("Attempting to determine the real name of %s", startname) - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - devname = startname - if devname == "ephemeral": - devname = "ephemeral0" - log.debug("Adjusted mount option from ephemeral to ephemeral0") - - device_path, partition_number = util.expand_dotted_devname(devname) - - if is_meta_device_name(device_path): - orig = device_path - device_path = transformer(device_path) - if not device_path: - return None - if not device_path.startswith("/"): - device_path = "/dev/%s" % (device_path,) - log.debug("Mapped metadata name %s to %s", orig, device_path) - else: - if DEVICE_NAME_RE.match(startname): - device_path = "/dev/%s" % (device_path,) - - partition_path = None - if partition_number is None: - partition_path = _get_nth_partition_for_device(device_path, 1) - else: - partition_path = _get_nth_partition_for_device(device_path, - partition_number) - if partition_path is None: - return None - - if _is_block_device(device_path, partition_path): - if partition_path is not None: - return partition_path - return device_path - return None - - -def suggested_swapsize(memsize=None, maxsize=None, fsys=None): - # make a suggestion on the size of swap for this system. - if memsize is None: - memsize = util.read_meminfo()['total'] - - GB = 2 ** 30 - sugg_max = 8 * GB - - info = {'avail': 'na', 'max_in': maxsize, 'mem': memsize} - - if fsys is None and maxsize is None: - # set max to 8GB default if no filesystem given - maxsize = sugg_max - elif fsys: - statvfs = os.statvfs(fsys) - avail = statvfs.f_frsize * statvfs.f_bfree - info['avail'] = avail - - if maxsize is None: - # set to 25% of filesystem space - maxsize = min(int(avail / 4), sugg_max) - elif maxsize > ((avail * .9)): - # set to 90% of available disk space - maxsize = int(avail * .9) - elif maxsize is None: - maxsize = sugg_max - - info['max'] = maxsize - - formulas = [ - # < 1G: swap = double memory - (1 * GB, lambda x: x * 2), - # < 2G: swap = 2G - (2 * GB, lambda x: 2 * GB), - # < 4G: swap = memory - (4 * GB, lambda x: x), - # < 16G: 4G - (16 * GB, lambda x: 4 * GB), - # < 64G: 1/2 M up to max - (64 * GB, lambda x: x / 2), - ] - - size = None - for top, func in formulas: - if memsize <= top: - size = min(func(memsize), maxsize) - # if less than 1/2 memory and not much, return 0 - if size < (memsize / 2) and size < 4 * GB: - size = 0 - break - break - - if size is not None: - size = maxsize - - info['size'] = size - - MB = 2 ** 20 - pinfo = {} - for k, v in info.items(): - if isinstance(v, int): - pinfo[k] = "%s MB" % (v / MB) - else: - pinfo[k] = v - - LOG.debug("suggest %(size)s swap for %(mem)s memory with '%(avail)s'" - " disk given max=%(max_in)s [max=%(max)s]'" % pinfo) - return size - - -def setup_swapfile(fname, size=None, maxsize=None): - """ - fname: full path string of filename to setup - size: the size to create. set to "auto" for recommended - maxsize: the maximum size - """ - tdir = os.path.dirname(fname) - if str(size).lower() == "auto": - try: - memsize = util.read_meminfo()['total'] - except IOError as e: - LOG.debug("Not creating swap. failed to read meminfo") - return - - util.ensure_dir(tdir) - size = suggested_swapsize(fsys=tdir, maxsize=maxsize, - memsize=memsize) - - if not size: - LOG.debug("Not creating swap: suggested size was 0") - return - - mbsize = str(int(size / (2 ** 20))) - msg = "creating swap file '%s' of %sMB" % (fname, mbsize) - try: - util.ensure_dir(tdir) - util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) - - except Exception as e: - raise IOError("Failed %s: %s" % (msg, e)) - - return fname - - -def handle_swapcfg(swapcfg): - """handle the swap config, calling setup_swap if necessary. - return None or (filename, size) - """ - if not isinstance(swapcfg, dict): - LOG.warn("input for swap config was not a dict.") - return None - - fname = swapcfg.get('filename', '/swap.img') - size = swapcfg.get('size', 0) - maxsize = swapcfg.get('maxsize', None) - - if not (size and fname): - LOG.debug("no need to setup swap") - return - - if os.path.exists(fname): - if not os.path.exists("/proc/swaps"): - LOG.debug("swap file %s existed. no /proc/swaps. Being safe.", - fname) - return fname - try: - for line in util.load_file("/proc/swaps").splitlines(): - if line.startswith(fname + " "): - LOG.debug("swap file %s already in use.", fname) - return fname - LOG.debug("swap file %s existed, but not in /proc/swaps", fname) - except Exception: - LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) - return fname - - try: - if isinstance(size, str) and size != "auto": - size = util.human2bytes(size) - if isinstance(maxsize, str): - maxsize = util.human2bytes(maxsize) - return setup_swapfile(fname=fname, size=size, maxsize=maxsize) - - except Exception as e: - LOG.warn("failed to setup swap: %s", e) - - return None - - -def handle(_name, cfg, cloud, log, _args): - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - def_mnt_opts = "defaults,nobootwait" - if cloud.distro.uses_systemd(): - def_mnt_opts = "defaults,nofail" - - defvals = [None, None, "auto", def_mnt_opts, "0", "2"] - defvals = cfg.get("mount_default_fields", defvals) - - # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], - ["swap", "none", "swap", "sw", "0", "0"]] - - cfgmnt = [] - if "mounts" in cfg: - cfgmnt = cfg["mounts"] - - for i in range(len(cfgmnt)): - # skip something that wasn't a list - if not isinstance(cfgmnt[i], list): - log.warn("Mount option %s not a list, got a %s instead", - (i + 1), type_utils.obj_name(cfgmnt[i])) - continue - - start = str(cfgmnt[i][0]) - sanitized = sanitize_devname(start, cloud.device_name_to_device, log) - if sanitized is None: - log.debug("Ignorming nonexistant named mount %s", start) - continue - - if sanitized != start: - log.debug("changed %s => %s" % (start, sanitized)) - cfgmnt[i][0] = sanitized - - # in case the user did not quote a field (likely fs-freq, fs_passno) - # but do not convert None to 'None' (LP: #898365) - for j in range(len(cfgmnt[i])): - if cfgmnt[i][j] is None: - continue - else: - cfgmnt[i][j] = str(cfgmnt[i][j]) - - for i in range(len(cfgmnt)): - # fill in values with defaults from defvals above - for j in range(len(defvals)): - if len(cfgmnt[i]) <= j: - cfgmnt[i].append(defvals[j]) - elif cfgmnt[i][j] is None: - cfgmnt[i][j] = defvals[j] - - # if the second entry in the list is 'None' this - # clears all previous entries of that same 'fs_spec' - # (fs_spec is the first field in /etc/fstab, ie, that device) - if cfgmnt[i][1] is None: - for j in range(i): - if cfgmnt[j][0] == cfgmnt[i][0]: - cfgmnt[j][1] = None - - # for each of the "default" mounts, add them only if no other - # entry has the same device name - for defmnt in defmnts: - start = defmnt[0] - sanitized = sanitize_devname(start, cloud.device_name_to_device, log) - if sanitized is None: - log.debug("Ignoring nonexistant default named mount %s", start) - continue - if sanitized != start: - log.debug("changed default device %s => %s" % (start, sanitized)) - defmnt[0] = sanitized - - cfgmnt_has = False - for cfgm in cfgmnt: - if cfgm[0] == defmnt[0]: - cfgmnt_has = True - break - - if cfgmnt_has: - log.debug(("Not including %s, already" - " previously included"), start) - continue - cfgmnt.append(defmnt) - - # now, each entry in the cfgmnt list has all fstab values - # if the second field is None (not the string, the value) we skip it - actlist = [] - for x in cfgmnt: - if x[1] is None: - log.debug("Skipping non-existent device named %s", x[0]) - else: - actlist.append(x) - - swapret = handle_swapcfg(cfg.get('swap', {})) - if swapret: - actlist.append([swapret, "none", "swap", "sw", "0", "0"]) - - if len(actlist) == 0: - log.debug("No modifications to fstab needed.") - return - - comment = "comment=cloudconfig" - cc_lines = [] - needswap = False - dirs = [] - for line in actlist: - # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,%s" % (line[3], comment) - if line[2] == "swap": - needswap = True - if line[1].startswith("/"): - dirs.append(line[1]) - cc_lines.append('\t'.join(line)) - - fstab_lines = [] - for line in util.load_file(FSTAB_PATH).splitlines(): - try: - toks = WS.split(line) - if toks[3].find(comment) != -1: - continue - except Exception: - pass - fstab_lines.append(line) - - fstab_lines.extend(cc_lines) - contents = "%s\n" % ('\n'.join(fstab_lines)) - util.write_file(FSTAB_PATH, contents) - - if needswap: - try: - util.subp(("swapon", "-a")) - except Exception: - util.logexc(log, "Activating swap via 'swapon -a' failed") - - for d in dirs: - try: - util.ensure_dir(d) - except Exception: - util.logexc(log, "Failed to make '%s' config-mount", d) - - try: - util.subp(("mount", "-a")) - except Exception: - util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py deleted file mode 100644 index 73b0e30d..00000000 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import time - -from cloudinit import log as logging -from cloudinit import util - -REBOOT_FILE = "/var/run/reboot-required" -REBOOT_CMD = ["/sbin/reboot"] - - -def _multi_cfg_bool_get(cfg, *keys): - for k in keys: - if util.get_cfg_option_bool(cfg, k, False): - return True - return False - - -def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) - start = time.time() - wait_time = initial_sleep - for _i in range(0, wait_attempts): - time.sleep(wait_time) - wait_time *= backoff - elapsed = time.time() - start - log.debug("Rebooted, but still running after %s seconds", int(elapsed)) - # If we got here, not good - elapsed = time.time() - start - raise RuntimeError(("Reboot did not happen" - " after %s seconds!") % (int(elapsed))) - - -def handle(_name, cfg, cloud, log, _args): - # Handle the old style + new config names - update = _multi_cfg_bool_get(cfg, 'apt_update', 'package_update') - upgrade = _multi_cfg_bool_get(cfg, 'package_upgrade', 'apt_upgrade') - reboot_if_required = _multi_cfg_bool_get(cfg, 'apt_reboot_if_required', - 'package_reboot_if_required') - pkglist = util.get_cfg_option_list(cfg, 'packages', []) - - errors = [] - if update or len(pkglist) or upgrade: - try: - cloud.distro.update_package_sources() - except Exception as e: - util.logexc(log, "Package update failed") - errors.append(e) - - if upgrade: - try: - cloud.distro.package_command("upgrade") - except Exception as e: - util.logexc(log, "Package upgrade failed") - errors.append(e) - - if len(pkglist): - try: - cloud.distro.install_packages(pkglist) - except Exception as e: - util.logexc(log, "Failed to install packages: %s", pkglist) - errors.append(e) - - # TODO(smoser): handle this less violently - # kernel and openssl (possibly some other packages) - # write a file /var/run/reboot-required after upgrading. - # if that file exists and configured, then just stop right now and reboot - reboot_fn_exists = os.path.isfile(REBOOT_FILE) - if (upgrade or pkglist) and reboot_if_required and reboot_fn_exists: - try: - log.warn("Rebooting after upgrade or install per %s", REBOOT_FILE) - # Flush the above warning + anything else out... - logging.flushLoggers(log) - _fire_reboot(log) - except Exception as e: - util.logexc(log, "Requested reboot did not happen!") - errors.append(e) - - if len(errors): - log.warn("%s failed with exceptions, re-raising the last one", - len(errors)) - raise errors[-1] diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py deleted file mode 100644 index 72176d42..00000000 --- a/cloudinit/config/cc_phone_home.py +++ /dev/null @@ -1,122 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import templater -from cloudinit import util - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - -POST_LIST_ALL = [ - 'pub_key_dsa', - 'pub_key_rsa', - 'pub_key_ecdsa', - 'instance_id', - 'hostname', - 'fdqn' -] - - -# phone_home: -# url: http://my.foo.bar/$INSTANCE/ -# post: all -# tries: 10 -# -# phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ -# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id, hostname, -# fqdn ] -# -def handle(name, cfg, cloud, log, args): - if len(args) != 0: - ph_cfg = util.read_conf(args[0]) - else: - if 'phone_home' not in cfg: - log.debug(("Skipping module named %s, " - "no 'phone_home' configuration found"), name) - return - ph_cfg = cfg['phone_home'] - - if 'url' not in ph_cfg: - log.warn(("Skipping module named %s, " - "no 'url' found in 'phone_home' configuration"), name) - return - - url = ph_cfg['url'] - post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries') - try: - tries = int(tries) - except Exception: - tries = 10 - util.logexc(log, "Configuration entry 'tries' is not an integer, " - "using %s instead", tries) - - if post_list == "all": - post_list = POST_LIST_ALL - - all_keys = {} - all_keys['instance_id'] = cloud.get_instance_id() - all_keys['hostname'] = cloud.get_hostname() - all_keys['fqdn'] = cloud.get_hostname(fqdn=True) - - pubkeys = { - 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', - 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', - 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', - } - - for (n, path) in pubkeys.items(): - try: - all_keys[n] = util.load_file(path) - except Exception: - util.logexc(log, "%s: failed to open, can not phone home that " - "data!", path) - - submit_keys = {} - for k in post_list: - if k in all_keys: - submit_keys[k] = all_keys[k] - else: - submit_keys[k] = None - log.warn(("Requested key %s from 'post'" - " configuration list not available"), k) - - # Get them read to be posted - real_submit_keys = {} - for (k, v) in submit_keys.items(): - if v is None: - real_submit_keys[k] = 'N/A' - else: - real_submit_keys[k] = str(v) - - # Incase the url is parameterized - url_params = { - 'INSTANCE_ID': all_keys['instance_id'], - } - url = templater.render_string(url, url_params) - try: - util.read_file_or_url(url, data=real_submit_keys, - retries=tries, sec_between=3, - ssl_details=util.fetch_ssl_details(cloud.paths)) - except Exception: - util.logexc(log, "Failed to post phone home data to %s in %s tries", - url, tries) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py deleted file mode 100644 index cc3f7f70..00000000 --- a/cloudinit/config/cc_power_state_change.py +++ /dev/null @@ -1,223 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -import errno -import os -import re -import six -import subprocess -import time - -frequency = PER_INSTANCE - -EXIT_FAIL = 254 - - -def givecmdline(pid): - # Returns the cmdline for the given process id. In Linux we can use procfs - # for this but on BSD there is /usr/bin/procstat. - try: - # Example output from procstat -c 1 - # PID COMM ARGS - # 1 init /bin/init -- - if util.system_info()["platform"].startswith('FreeBSD'): - (output, _err) = util.subp(['procstat', '-c', str(pid)]) - line = output.splitlines()[1] - m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line) - return m.group(2) - else: - return util.load_file("/proc/%s/cmdline" % pid) - except IOError: - return None - - -def check_condition(cond, log=None): - if isinstance(cond, bool): - if log: - log.debug("Static Condition: %s" % cond) - return cond - - pre = "check_condition command (%s): " % cond - try: - proc = subprocess.Popen(cond, shell=not isinstance(cond, list)) - proc.communicate() - ret = proc.returncode - if ret == 0: - if log: - log.debug(pre + "exited 0. condition met.") - return True - elif ret == 1: - if log: - log.debug(pre + "exited 1. condition not met.") - return False - else: - if log: - log.warn(pre + "unexpected exit %s. " % ret + - "do not apply change.") - return False - except Exception as e: - if log: - log.warn(pre + "Unexpected error: %s" % e) - return False - - -def handle(_name, cfg, _cloud, log, _args): - - try: - (args, timeout, condition) = load_power_state(cfg) - if args is None: - log.debug("no power_state provided. doing nothing") - return - except Exception as e: - log.warn("%s Not performing power state change!" % str(e)) - return - - if condition is False: - log.debug("Condition was false. Will not perform state change.") - return - - mypid = os.getpid() - - cmdline = givecmdline(mypid) - if not cmdline: - log.warn("power_state: failed to get cmdline of current process") - return - - devnull_fp = open(os.devnull, "w") - - log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, - condition, execmd, [args, devnull_fp]) - - -def load_power_state(cfg): - # returns a tuple of shutdown_command, timeout - # shutdown_command is None if no config found - pstate = cfg.get('power_state') - - if pstate is None: - return (None, None, None) - - if not isinstance(pstate, dict): - raise TypeError("power_state is not a dict.") - - opt_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} - - mode = pstate.get("mode") - if mode not in opt_map: - raise TypeError( - "power_state[mode] required, must be one of: %s. found: '%s'." % - (','.join(opt_map.keys()), mode)) - - delay = pstate.get("delay", "now") - # convert integer 30 or string '30' to '+30' - try: - delay = "+%s" % int(delay) - except ValueError: - pass - - if delay != "now" and not re.match(r"\+[0-9]+", delay): - raise TypeError( - "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay) - - args = ["shutdown", opt_map[mode], delay] - if pstate.get("message"): - args.append(pstate.get("message")) - - try: - timeout = float(pstate.get('timeout', 30.0)) - except ValueError: - raise ValueError("failed to convert timeout '%s' to float." % - pstate['timeout']) - - condition = pstate.get("condition", True) - if not isinstance(condition, six.string_types + (list, bool)): - raise TypeError("condition type %s invalid. must be list, bool, str") - return (args, timeout, condition) - - -def doexit(sysexit): - os._exit(sysexit) - - -def execmd(exe_args, output=None, data_in=None): - try: - proc = subprocess.Popen(exe_args, stdin=subprocess.PIPE, - stdout=output, stderr=subprocess.STDOUT) - proc.communicate(data_in) - ret = proc.returncode - except Exception: - doexit(EXIT_FAIL) - doexit(ret) - - -def run_after_pid_gone(pid, pidcmdline, timeout, log, condition, func, args): - # wait until pid, with /proc/pid/cmdline contents of pidcmdline - # is no longer alive. After it is gone, or timeout has passed - # execute func(args) - msg = None - end_time = time.time() + timeout - - def fatal(msg): - if log: - log.warn(msg) - doexit(EXIT_FAIL) - - known_errnos = (errno.ENOENT, errno.ESRCH) - - while True: - if time.time() > end_time: - msg = "timeout reached before %s ended" % pid - break - - try: - cmdline = givecmdline(pid) - if cmdline != pidcmdline: - msg = "cmdline changed for %s [now: %s]" % (pid, cmdline) - break - - except IOError as ioerr: - if ioerr.errno in known_errnos: - msg = "pidfile gone [%d]" % ioerr.errno - else: - fatal("IOError during wait: %s" % ioerr) - break - - except Exception as e: - fatal("Unexpected Exception: %s" % e) - - time.sleep(.25) - - if not msg: - fatal("Unexpected error in run_after_pid_gone") - - if log: - log.debug(msg) - - try: - if not check_condition(condition, log): - return - except Exception as e: - fatal("Unexpected Exception when checking condition: %s" % e) - - func(*args) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py deleted file mode 100644 index 774d3322..00000000 --- a/cloudinit/config/cc_puppet.py +++ /dev/null @@ -1,118 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from six import StringIO - -import os -import socket - -from cloudinit import helpers -from cloudinit import util - -PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' -PUPPET_SSL_CERT_DIR = '/var/lib/puppet/ssl/certs/' -PUPPET_SSL_DIR = '/var/lib/puppet/ssl' -PUPPET_SSL_CERT_PATH = '/var/lib/puppet/ssl/certs/ca.pem' - - -def _autostart_puppet(log): - # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) - elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], - capture=False) - elif os.path.exists('/sbin/chkconfig'): - util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) - else: - log.warn(("Sorry we do not know how to enable" - " puppet services on this system")) - - -def handle(name, cfg, cloud, log, _args): - # If there isn't a puppet key in the configuration don't do anything - if 'puppet' not in cfg: - log.debug(("Skipping module named %s," - " no 'puppet' configuration found"), name) - return - - puppet_cfg = cfg['puppet'] - - # Start by installing the puppet package if necessary... - install = util.get_cfg_option_bool(puppet_cfg, 'install', True) - version = util.get_cfg_option_str(puppet_cfg, 'version', None) - if not install and version: - log.warn(("Puppet install set false but version supplied," - " doing nothing.")) - elif install: - log.debug(("Attempting to install puppet %s,"), - version if version else 'latest') - cloud.distro.install_packages(('puppet', version)) - - # ... and then update the puppet configuration - if 'conf' in puppet_cfg: - # Add all sections from the conf object to puppet.conf - contents = util.load_file(PUPPET_CONF_PATH) - # Create object for reading puppet.conf values - puppet_config = helpers.DefaultingConfigParser() - # Read puppet.conf values from original file in order to be able to - # mix the rest up. First clean them up - # (TODO(harlowja) is this really needed??) - cleaned_lines = [i.lstrip() for i in contents.splitlines()] - cleaned_contents = '\n'.join(cleaned_lines) - puppet_config.readfp(StringIO(cleaned_contents), - filename=PUPPET_CONF_PATH) - for (cfg_name, cfg) in puppet_cfg['conf'].items(): - # Cert configuration is a special case - # Dump the puppet master ca certificate in the correct place - if cfg_name == 'ca_cert': - # Puppet ssl sub-directory isn't created yet - # Create it with the proper permissions and ownership - util.ensure_dir(PUPPET_SSL_DIR, 0o771) - util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') - util.ensure_dir(PUPPET_SSL_CERT_DIR) - util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') - util.write_file(PUPPET_SSL_CERT_PATH, cfg) - util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') - else: - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for (o, v) in cfg.items(): - if o == 'certname': - # Expand %f as the fqdn - # TODO(harlowja) should this use the cloud fqdn?? - v = v.replace("%f", socket.getfqdn()) - # Expand %i as the instance id - v = v.replace("%i", cloud.get_instance_id()) - # certname needs to be downcased - v = v.lower() - puppet_config.set(cfg_name, o, v) - # We got all our config as wanted we'll rename - # the previous puppet.conf and create our new one - util.rename(PUPPET_CONF_PATH, "%s.old" % (PUPPET_CONF_PATH)) - util.write_file(PUPPET_CONF_PATH, puppet_config.stringify()) - - # Set it up so it autostarts - _autostart_puppet(log) - - # Start puppetd - util.subp(['service', 'puppet', 'start'], capture=False) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py deleted file mode 100644 index 2a2a9f59..00000000 --- a/cloudinit/config/cc_resizefs.py +++ /dev/null @@ -1,185 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import errno -import os -import stat - -from cloudinit.settings import PER_ALWAYS -from cloudinit import util - -frequency = PER_ALWAYS - - -def _resize_btrfs(mount_point, devpth): - return ('btrfs', 'filesystem', 'resize', 'max', mount_point) - - -def _resize_ext(mount_point, devpth): - return ('resize2fs', devpth) - - -def _resize_xfs(mount_point, devpth): - return ('xfs_growfs', devpth) - - -def _resize_ufs(mount_point, devpth): - return ('growfs', devpth) - -# Do not use a dictionary as these commands should be able to be used -# for multiple filesystem types if possible, e.g. one command for -# ext2, ext3 and ext4. -RESIZE_FS_PREFIXES_CMDS = [ - ('btrfs', _resize_btrfs), - ('ext', _resize_ext), - ('xfs', _resize_xfs), - ('ufs', _resize_ufs), -] - -NOBLOCK = "noblock" - - -def rootdev_from_cmdline(cmdline): - found = None - for tok in cmdline.split(): - if tok.startswith("root="): - found = tok[5:] - break - if found is None: - return None - - if found.startswith("/dev/"): - return found - if found.startswith("LABEL="): - return "/dev/disk/by-label/" + found[len("LABEL="):] - if found.startswith("UUID="): - return "/dev/disk/by-uuid/" + found[len("UUID="):] - - return "/dev/" + found - - -def handle(name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = args[0] - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if not util.translate_bool(resize_root, addons=[NOBLOCK]): - log.debug("Skipping module named %s, resizing disabled", name) - return - - # TODO(harlowja) is the directory ok to be used?? - resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") - util.ensure_dir(resize_root_d) - - # TODO(harlowja): allow what is to be resized to be configurable?? - resize_what = "/" - result = util.get_mount_info(resize_what, log) - if not result: - log.warn("Could not determine filesystem type of %s", resize_what) - return - - (devpth, fs_type, mount_point) = result - - info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) - log.debug("resize_info: %s" % info) - - container = util.is_container() - - # Ensure the path is a block device. - if (devpth == "/dev/root" and not os.path.exists(devpth) and - not container): - devpth = rootdev_from_cmdline(util.get_cmdline()) - if devpth is None: - log.warn("Unable to find device '/dev/root'") - return - log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth) - - try: - statret = os.stat(devpth) - except OSError as exc: - if container and exc.errno == errno.ENOENT: - log.debug("Device '%s' did not exist in container. " - "cannot resize: %s", devpth, info) - elif exc.errno == errno.ENOENT: - log.warn("Device '%s' did not exist. cannot resize: %s", - devpth, info) - else: - raise exc - return - - if not os.access(devpth, os.W_OK): - if container: - log.debug("'%s' not writable in container. cannot resize: %s", - devpth, info) - else: - log.warn("'%s' not writable. cannot resize: %s", devpth, info) - return - - if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): - if container: - log.debug("device '%s' not a block device in container." - " cannot resize: %s" % (devpth, info)) - else: - log.warn("device '%s' not a block device. cannot resize: %s" % - (devpth, info)) - return - - resizer = None - fstype_lc = fs_type.lower() - for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS: - if fstype_lc.startswith(pfix): - resizer = root_cmd - break - - if not resizer: - log.warn("Not resizing unknown filesystem type %s for %s", - fs_type, resize_what) - return - - resize_cmd = resizer(resize_what, devpth) - log.debug("Resizing %s (%s) using %s", resize_what, fs_type, - ' '.join(resize_cmd)) - - if resize_root == NOBLOCK: - # Fork to a child that will run - # the resize command - util.fork_cb( - util.log_time, logfunc=log.debug, msg="backgrounded Resizing", - func=do_resize, args=(resize_cmd, log)) - else: - util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) - - action = 'Resized' - if resize_root == NOBLOCK: - action = 'Resizing (via forking)' - log.debug("%s root filesystem (type=%s, val=%s)", action, fs_type, - resize_root) - - -def do_resize(resize_cmd, log): - try: - util.subp(resize_cmd) - except util.ProcessExecutionError: - util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) - raise - # TODO(harlowja): Should we add a fsck check after this to make - # sure we didn't corrupt anything? diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py deleted file mode 100644 index 71d9e3a7..00000000 --- a/cloudinit/config/cc_resolv_conf.py +++ /dev/null @@ -1,116 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Craig Tracey -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Craig Tracey -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Note: -# This module is intended to manage resolv.conf in environments where -# early configuration of resolv.conf is necessary for further -# bootstrapping and/or where configuration management such as puppet or -# chef own dns configuration. As Debian/Ubuntu will, by default, utilize -# resovlconf, and similarly RedHat will use sysconfig, this module is -# likely to be of little use unless those are configured correctly. -# -# For RedHat with sysconfig, be sure to set PEERDNS=no for all DHCP -# enabled NICs. And, in Ubuntu/Debian it is recommended that DNS -# be configured via the standard /etc/network/interfaces configuration -# file. -# -# -# Usage Example: -# -# #cloud-config -# manage_resolv_conf: true -# -# resolv_conf: -# nameservers: ['8.8.4.4', '8.8.8.8'] -# searchdomains: -# - foo.example.com -# - bar.example.com -# domain: example.com -# options: -# rotate: true -# timeout: 1 -# - - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import templater -from cloudinit import util - -LOG = logging.getLogger(__name__) - -frequency = PER_INSTANCE - -distros = ['fedora', 'rhel', 'sles'] - - -def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): - flags = [] - false_flags = [] - - if 'options' in params: - for key, val in params['options'].items(): - if isinstance(val, bool): - if val: - flags.append(key) - else: - false_flags.append(key) - - for flag in flags + false_flags: - del params['options'][flag] - - if not params.get('options'): - params['options'] = {} - - params['flags'] = flags - LOG.debug("Writing resolv.conf from template %s" % template_fn) - templater.render_to_file(template_fn, target_fname, params) - - -def handle(name, cfg, cloud, log, _args): - """ - Handler for resolv.conf - - @param name: The module name "resolv-conf" from cloud.cfg - @param cfg: A nested dict containing the entire cloud config contents. - @param cloud: The L{CloudInit} object in use. - @param log: Pre-initialized Python logger object to use for logging. - @param args: Any module arguments from cloud.cfg - """ - if "manage_resolv_conf" not in cfg: - log.debug(("Skipping module named %s," - " no 'manage_resolv_conf' key in configuration"), name) - return - - if not util.get_cfg_option_bool(cfg, "manage_resolv_conf", False): - log.debug(("Skipping module named %s," - " 'manage_resolv_conf' present but set to False"), name) - return - - if "resolv_conf" not in cfg: - log.warn("manage_resolv_conf True but no parameters provided!") - - template_fn = cloud.get_template_filename('resolv.conf') - if not template_fn: - log.warn("No template found, not rendering /etc/resolv.conf") - return - - generate_resolv_conf(template_fn=template_fn, params=cfg["resolv_conf"]) - return diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py deleted file mode 100644 index 3a113aea..00000000 --- a/cloudinit/config/cc_rh_subscription.py +++ /dev/null @@ -1,408 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Red Hat, Inc. -# -# Author: Brent Baude -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import util - - -def handle(name, cfg, _cloud, log, _args): - sm = SubscriptionManager(cfg) - sm.log = log - if not sm.is_configured(): - log.debug("%s: module not configured.", name) - return None - - if not sm.is_registered(): - try: - verify, verify_msg = sm._verify_keys() - if verify is not True: - raise SubscriptionError(verify_msg) - cont = sm.rhn_register() - if not cont: - raise SubscriptionError("Registration failed or did not " - "run completely") - - # Splitting up the registration, auto-attach, and servicelevel - # commands because the error codes, messages from subman are not - # specific enough. - - # Attempt to change the service level - if sm.auto_attach and sm.servicelevel is not None: - if not sm._set_service_level(): - raise SubscriptionError("Setting of service-level " - "failed") - else: - sm.log.debug("Completed auto-attach with service level") - elif sm.auto_attach: - if not sm._set_auto_attach(): - raise SubscriptionError("Setting auto-attach failed") - else: - sm.log.debug("Completed auto-attach") - - if sm.pools is not None: - if not isinstance(sm.pools, list): - pool_fail = "Pools must in the format of a list" - raise SubscriptionError(pool_fail) - - return_stat = sm.addPool(sm.pools) - if not return_stat: - raise SubscriptionError("Unable to attach pools {0}" - .format(sm.pools)) - if (sm.enable_repo is not None) or (sm.disable_repo is not None): - return_stat = sm.update_repos(sm.enable_repo, sm.disable_repo) - if not return_stat: - raise SubscriptionError("Unable to add or remove repos") - sm.log_success("rh_subscription plugin completed successfully") - except SubscriptionError as e: - sm.log_warn(str(e)) - sm.log_warn("rh_subscription plugin did not complete successfully") - else: - sm.log_success("System is already registered") - - -class SubscriptionError(Exception): - pass - - -class SubscriptionManager(object): - valid_rh_keys = ['org', 'activation-key', 'username', 'password', - 'disable-repo', 'enable-repo', 'add-pool', - 'rhsm-baseurl', 'server-hostname', - 'auto-attach', 'service-level'] - - def __init__(self, cfg): - self.cfg = cfg - self.rhel_cfg = self.cfg.get('rh_subscription', {}) - self.rhsm_baseurl = self.rhel_cfg.get('rhsm-baseurl') - self.server_hostname = self.rhel_cfg.get('server-hostname') - self.pools = self.rhel_cfg.get('add-pool') - self.activation_key = self.rhel_cfg.get('activation-key') - self.org = self.rhel_cfg.get('org') - self.userid = self.rhel_cfg.get('username') - self.password = self.rhel_cfg.get('password') - self.auto_attach = self.rhel_cfg.get('auto-attach') - self.enable_repo = self.rhel_cfg.get('enable-repo') - self.disable_repo = self.rhel_cfg.get('disable-repo') - self.servicelevel = self.rhel_cfg.get('service-level') - self.subman = ['subscription-manager'] - - def log_success(self, msg): - '''Simple wrapper for logging info messages. Useful for unittests''' - self.log.info(msg) - - def log_warn(self, msg): - '''Simple wrapper for logging warning messages. Useful for unittests''' - self.log.warn(msg) - - def _verify_keys(self): - ''' - Checks that the keys in the rh_subscription dict from the user-data - are what we expect. - ''' - - for k in self.rhel_cfg: - if k not in self.valid_rh_keys: - bad_key = "{0} is not a valid key for rh_subscription. "\ - "Valid keys are: "\ - "{1}".format(k, ', '.join(self.valid_rh_keys)) - return False, bad_key - - # Check for bad auto-attach value - if (self.auto_attach is not None) and \ - not (util.is_true(self.auto_attach) or - util.is_false(self.auto_attach)): - not_bool = "The key auto-attach must be a boolean value "\ - "(True/False " - return False, not_bool - - if (self.servicelevel is not None) and ((not self.auto_attach) or - (util.is_false(str(self.auto_attach)))): - no_auto = ("The service-level key must be used in conjunction " - "with the auto-attach key. Please re-run with " - "auto-attach: True") - return False, no_auto - return True, None - - def is_registered(self): - ''' - Checks if the system is already registered and returns - True if so, else False - ''' - cmd = ['identity'] - - try: - self._sub_man_cli(cmd) - except util.ProcessExecutionError: - return False - - return True - - def _sub_man_cli(self, cmd, logstring_val=False): - ''' - Uses the prefered cloud-init subprocess def of util.subp - and runs subscription-manager. Breaking this to a - separate function for later use in mocking and unittests - ''' - cmd = self.subman + cmd - return util.subp(cmd, logstring=logstring_val) - - def rhn_register(self): - ''' - Registers the system by userid and password or activation key - and org. Returns True when successful False when not. - ''' - - if (self.activation_key is not None) and (self.org is not None): - # register by activation key - cmd = ['register', '--activationkey={0}'. - format(self.activation_key), '--org={0}'.format(self.org)] - - # If the baseurl and/or server url are passed in, we register - # with them. - - if self.rhsm_baseurl is not None: - cmd.append("--baseurl={0}".format(self.rhsm_baseurl)) - - if self.server_hostname is not None: - cmd.append("--serverurl={0}".format(self.server_hostname)) - - try: - return_out, return_err = self._sub_man_cli(cmd, - logstring_val=True) - except util.ProcessExecutionError as e: - if e.stdout == "": - self.log_warn("Registration failed due " - "to: {0}".format(e.stderr)) - return False - - elif (self.userid is not None) and (self.password is not None): - # register by username and password - cmd = ['register', '--username={0}'.format(self.userid), - '--password={0}'.format(self.password)] - - # If the baseurl and/or server url are passed in, we register - # with them. - - if self.rhsm_baseurl is not None: - cmd.append("--baseurl={0}".format(self.rhsm_baseurl)) - - if self.server_hostname is not None: - cmd.append("--serverurl={0}".format(self.server_hostname)) - - # Attempting to register the system only - try: - return_out, return_err = self._sub_man_cli(cmd, - logstring_val=True) - except util.ProcessExecutionError as e: - if e.stdout == "": - self.log_warn("Registration failed due " - "to: {0}".format(e.stderr)) - return False - - else: - self.log_warn("Unable to register system due to incomplete " - "information.") - self.log_warn("Use either activationkey and org *or* userid " - "and password") - return False - - reg_id = return_out.split("ID: ")[1].rstrip() - self.log.debug("Registered successfully with ID {0}".format(reg_id)) - return True - - def _set_service_level(self): - cmd = ['attach', '--auto', '--servicelevel={0}' - .format(self.servicelevel)] - - try: - return_out, return_err = self._sub_man_cli(cmd) - except util.ProcessExecutionError as e: - if e.stdout.rstrip() != '': - for line in e.stdout.split("\n"): - if line is not '': - self.log_warn(line) - else: - self.log_warn("Setting the service level failed with: " - "{0}".format(e.stderr.strip())) - return False - for line in return_out.split("\n"): - if line is not "": - self.log.debug(line) - return True - - def _set_auto_attach(self): - cmd = ['attach', '--auto'] - try: - return_out, return_err = self._sub_man_cli(cmd) - except util.ProcessExecutionError: - self.log_warn("Auto-attach failed with: " - "{0}]".format(return_err.strip())) - return False - for line in return_out.split("\n"): - if line is not "": - self.log.debug(line) - return True - - def _getPools(self): - ''' - Gets the list pools for the active subscription and returns them - in list form. - ''' - available = [] - consumed = [] - - # Get all available pools - cmd = ['list', '--available', '--pool-only'] - results, errors = self._sub_man_cli(cmd) - available = (results.rstrip()).split("\n") - - # Get all consumed pools - cmd = ['list', '--consumed', '--pool-only'] - results, errors = self._sub_man_cli(cmd) - consumed = (results.rstrip()).split("\n") - - return available, consumed - - def _getRepos(self): - ''' - Obtains the current list of active yum repositories and returns - them in list form. - ''' - - cmd = ['repos', '--list-enabled'] - return_out, return_err = self._sub_man_cli(cmd) - active_repos = [] - for repo in return_out.split("\n"): - if "Repo ID:" in repo: - active_repos.append((repo.split(':')[1]).strip()) - - cmd = ['repos', '--list-disabled'] - return_out, return_err = self._sub_man_cli(cmd) - - inactive_repos = [] - for repo in return_out.split("\n"): - if "Repo ID:" in repo: - inactive_repos.append((repo.split(':')[1]).strip()) - return active_repos, inactive_repos - - def addPool(self, pools): - ''' - Takes a list of subscription pools and "attaches" them to the - current subscription - ''' - - # An empty list was passed - if len(pools) == 0: - self.log.debug("No pools to attach") - return True - - pool_available, pool_consumed = self._getPools() - pool_list = [] - cmd = ['attach'] - for pool in pools: - if (pool not in pool_consumed) and (pool in pool_available): - pool_list.append('--pool={0}'.format(pool)) - else: - self.log_warn("Pool {0} is not available".format(pool)) - if len(pool_list) > 0: - cmd.extend(pool_list) - try: - self._sub_man_cli(cmd) - self.log.debug("Attached the following pools to your " - "system: %s" % (", ".join(pool_list)) - .replace('--pool=', '')) - return True - except util.ProcessExecutionError as e: - self.log_warn("Unable to attach pool {0} " - "due to {1}".format(pool, e)) - return False - - def update_repos(self, erepos, drepos): - ''' - Takes a list of yum repo ids that need to be disabled or enabled; then - it verifies if they are already enabled or disabled and finally - executes the action to disable or enable - ''' - - if (erepos is not None) and (not isinstance(erepos, list)): - self.log_warn("Repo IDs must in the format of a list.") - return False - - if (drepos is not None) and (not isinstance(drepos, list)): - self.log_warn("Repo IDs must in the format of a list.") - return False - - # Bail if both lists are not populated - if (len(erepos) == 0) and (len(drepos) == 0): - self.log.debug("No repo IDs to enable or disable") - return True - - active_repos, inactive_repos = self._getRepos() - # Creating a list of repoids to be enabled - enable_list = [] - enable_list_fail = [] - for repoid in erepos: - if (repoid in inactive_repos): - enable_list.append("--enable={0}".format(repoid)) - else: - enable_list_fail.append(repoid) - - # Creating a list of repoids to be disabled - disable_list = [] - disable_list_fail = [] - for repoid in drepos: - if repoid in active_repos: - disable_list.append("--disable={0}".format(repoid)) - else: - disable_list_fail.append(repoid) - - # Logging any repos that are already enabled or disabled - if len(enable_list_fail) > 0: - for fail in enable_list_fail: - # Check if the repo exists or not - if fail in active_repos: - self.log.debug("Repo {0} is already enabled".format(fail)) - else: - self.log_warn("Repo {0} does not appear to " - "exist".format(fail)) - if len(disable_list_fail) > 0: - for fail in disable_list_fail: - self.log.debug("Repo {0} not disabled " - "because it is not enabled".format(fail)) - - cmd = ['repos'] - if len(enable_list) > 0: - cmd.extend(enable_list) - if len(disable_list) > 0: - cmd.extend(disable_list) - - try: - self._sub_man_cli(cmd) - except util.ProcessExecutionError as e: - self.log_warn("Unable to alter repos due to {0}".format(e)) - return False - - if len(enable_list) > 0: - self.log.debug("Enabled the following repos: %s" % - (", ".join(enable_list)).replace('--enable=', '')) - if len(disable_list) > 0: - self.log.debug("Disabled the following repos: %s" % - (", ".join(disable_list)).replace('--disable=', '')) - return True - - def is_configured(self): - return bool((self.userid and self.password) or self.activation_key) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py deleted file mode 100644 index 8118fac4..00000000 --- a/cloudinit/config/cc_rightscale_userdata.py +++ /dev/null @@ -1,102 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# -# The purpose of this script is to allow cloud-init to consume -# rightscale style userdata. rightscale user data is key-value pairs -# in a url-query-string like format. -# -# for cloud-init support, there will be a key named -# 'CLOUD_INIT_REMOTE_HOOK'. -# -# This cloud-config module will -# - read the blob of data from raw user data, and parse it as key/value -# - for each key that is found, download the content to -# the local instance/scripts directory and set them executable. -# - the files in that directory will be run by the user-scripts module -# Therefore, this must run before that. -# -# - -import os - -from cloudinit.settings import PER_INSTANCE -from cloudinit import url_helper as uhelp -from cloudinit import util - -from six.moves.urllib_parse import parse_qs - -frequency = PER_INSTANCE - -MY_NAME = "cc_rightscale_userdata" -MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' - - -def handle(name, _cfg, cloud, log, _args): - try: - ud = cloud.get_userdata_raw() - except Exception: - log.debug("Failed to get raw userdata in module %s", name) - return - - try: - mdict = parse_qs(ud) - if not mdict or MY_HOOKNAME not in mdict: - log.debug(("Skipping module %s, " - "did not find %s in parsed" - " raw userdata"), name, MY_HOOKNAME) - return - except Exception: - util.logexc(log, "Failed to parse query string %s into a dictionary", - ud) - raise - - wrote_fns = [] - captured_excps = [] - - # These will eventually be then ran by the cc_scripts_user - # TODO(harlowja): maybe this should just be a new user data handler?? - # Instead of a late module that acts like a user data handler? - scripts_d = cloud.get_ipath_cur('scripts') - urls = mdict[MY_HOOKNAME] - for (i, url) in enumerate(urls): - fname = os.path.join(scripts_d, "rightscale-%02i" % (i)) - try: - resp = uhelp.readurl(url) - # Ensure its a valid http response (and something gotten) - if resp.ok() and resp.contents: - util.write_file(fname, resp, mode=0o700) - wrote_fns.append(fname) - except Exception as e: - captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", MY_NAME, url, - fname) - - if wrote_fns: - log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) - - if len(wrote_fns) != len(urls): - skipped = len(urls) - len(wrote_fns) - log.debug("%s urls were skipped or failed", skipped) - - if captured_excps: - log.warn("%s failed with exceptions, re-raising the last one", - len(captured_excps)) - raise captured_excps[-1] diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py deleted file mode 100644 index b8642d65..00000000 --- a/cloudinit/config/cc_rsyslog.py +++ /dev/null @@ -1,366 +0,0 @@ -# vi: ts=4 expandtab syntax=python -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -rsyslog module allows configuration of syslog logging via rsyslog -Configuration is done under the cloud-config top level 'rsyslog'. - -Under 'rsyslog' you can define: - - configs: [default=[]] - this is a list. entries in it are a string or a dictionary. - each entry has 2 parts: - * content - * filename - if the entry is a string, then it is assigned to 'content'. - for each entry, content is written to the provided filename. - if filename is not provided, its default is read from 'config_filename' - - Content here can be any valid rsyslog configuration. No format - specific format is enforced. - - For simply logging to an existing remote syslog server, via udp: - configs: ["*.* @192.168.1.1"] - - - remotes: [default={}] - This is a dictionary of name / value pairs. - In comparison to 'config's, it is more focused in that it only supports - remote syslog configuration. It is not rsyslog specific, and could - convert to other syslog implementations. - - Each entry in remotes is a 'name' and a 'value'. - * name: an string identifying the entry. good practice would indicate - using a consistent and identifiable string for the producer. - For example, the MAAS service could use 'maas' as the key. - * value consists of the following parts: - * optional filter for log messages - default if not present: *.* - * optional leading '@' or '@@' (indicates udp or tcp respectively). - default if not present (udp): @ - This is rsyslog format for that. if not present, is '@'. - * ipv4 or ipv6 or hostname - ipv6 addresses must be in [::1] format. (@[fd00::1]:514) - * optional port - port defaults to 514 - - - config_filename: [default=20-cloud-config.conf] - this is the file name to use if none is provided in a config entry. - - - config_dir: [default=/etc/rsyslog.d] - this directory is used for filenames that are not absolute paths. - - - service_reload_command: [default="auto"] - this command is executed if files have been written and thus the syslog - daemon needs to be told. - -Note, since cloud-init 0.5 a legacy version of rsyslog config has been -present and is still supported. See below for the mappings between old -value and new value: - old value -> new value - 'rsyslog' -> rsyslog/configs - 'rsyslog_filename' -> rsyslog/config_filename - 'rsyslog_dir' -> rsyslog/config_dir - -the legacy config does not support 'service_reload_command'. - -Example config: - #cloud-config - rsyslog: - configs: - - "*.* @@192.158.1.1" - - content: "*.* @@192.0.2.1:10514" - filename: 01-example.conf - - content: | - *.* @@syslogd.example.com - remotes: - maas: "192.168.1.1" - juju: "10.0.4.1" - config_dir: config_dir - config_filename: config_filename - service_reload_command: [your, syslog, restart, command] - -Example Legacy config: - #cloud-config - rsyslog: - - "*.* @@192.158.1.1" - rsyslog_dir: /etc/rsyslog-config.d/ - rsyslog_filename: 99-local.conf -""" - -import os -import re -import six - -from cloudinit import log as logging -from cloudinit import util - -DEF_FILENAME = "20-cloud-config.conf" -DEF_DIR = "/etc/rsyslog.d" -DEF_RELOAD = "auto" -DEF_REMOTES = {} - -KEYNAME_CONFIGS = 'configs' -KEYNAME_FILENAME = 'config_filename' -KEYNAME_DIR = 'config_dir' -KEYNAME_RELOAD = 'service_reload_command' -KEYNAME_LEGACY_FILENAME = 'rsyslog_filename' -KEYNAME_LEGACY_DIR = 'rsyslog_dir' -KEYNAME_REMOTES = 'remotes' - -LOG = logging.getLogger(__name__) - -COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') -HOST_PORT_RE = re.compile( - r'^(?P[@]{0,2})' - '(([[](?P[^\]]*)[\]])|(?P[^:]*))' - '([:](?P[0-9]+))?$') - - -def reload_syslog(command=DEF_RELOAD, systemd=False): - service = 'rsyslog' - if command == DEF_RELOAD: - if systemd: - cmd = ['systemctl', 'reload-or-try-restart', service] - else: - cmd = ['service', service, 'restart'] - else: - cmd = command - util.subp(cmd, capture=True) - - -def load_config(cfg): - # return an updated config with entries of the correct type - # support converting the old top level format into new format - mycfg = cfg.get('rsyslog', {}) - - if isinstance(cfg.get('rsyslog'), list): - mycfg = {KEYNAME_CONFIGS: cfg.get('rsyslog')} - if KEYNAME_LEGACY_FILENAME in cfg: - mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME] - if KEYNAME_LEGACY_DIR in cfg: - mycfg[KEYNAME_DIR] = cfg[KEYNAME_LEGACY_DIR] - - fillup = ( - (KEYNAME_CONFIGS, [], list), - (KEYNAME_DIR, DEF_DIR, six.string_types), - (KEYNAME_FILENAME, DEF_FILENAME, six.string_types), - (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,)), - (KEYNAME_REMOTES, DEF_REMOTES, dict)) - - for key, default, vtypes in fillup: - if key not in mycfg or not isinstance(mycfg[key], vtypes): - mycfg[key] = default - - return mycfg - - -def apply_rsyslog_changes(configs, def_fname, cfg_dir): - # apply the changes in 'configs' to the paths in def_fname and cfg_dir - # return a list of the files changed - files = [] - for cur_pos, ent in enumerate(configs): - if isinstance(ent, dict): - if "content" not in ent: - LOG.warn("No 'content' entry in config entry %s", cur_pos + 1) - continue - content = ent['content'] - filename = ent.get("filename", def_fname) - else: - content = ent - filename = def_fname - - filename = filename.strip() - if not filename: - LOG.warn("Entry %s has an empty filename", cur_pos + 1) - continue - - filename = os.path.join(cfg_dir, filename) - - # Truncate filename first time you see it - omode = "ab" - if filename not in files: - omode = "wb" - files.append(filename) - - try: - endl = "" - if not content.endswith("\n"): - endl = "\n" - util.write_file(filename, content + endl, omode=omode) - except Exception: - util.logexc(LOG, "Failed to write to %s", filename) - - return files - - -def parse_remotes_line(line, name=None): - try: - data, comment = COMMENT_RE.split(line) - comment = comment.strip() - except ValueError: - data, comment = (line, None) - - toks = data.strip().split() - match = None - if len(toks) == 1: - host_port = data - elif len(toks) == 2: - match, host_port = toks - else: - raise ValueError("line had multiple spaces: %s" % data) - - toks = HOST_PORT_RE.match(host_port) - - if not toks: - raise ValueError("Invalid host specification '%s'" % host_port) - - proto = toks.group('proto') - addr = toks.group('addr') or toks.group('bracket_addr') - port = toks.group('port') - - if addr.startswith("[") and not addr.endswith("]"): - raise ValueError("host spec had invalid brackets: %s" % addr) - - if comment and not name: - name = comment - - t = SyslogRemotesLine(name=name, match=match, proto=proto, - addr=addr, port=port) - t.validate() - return t - - -class SyslogRemotesLine(object): - def __init__(self, name=None, match=None, proto=None, addr=None, - port=None): - if not match: - match = "*.*" - self.name = name - self.match = match - if not proto: - proto = "udp" - if proto == "@": - proto = "udp" - elif proto == "@@": - proto = "tcp" - self.proto = proto - - self.addr = addr - if port: - self.port = int(port) - else: - self.port = None - - def validate(self): - if self.port: - try: - int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) - - if not self.addr: - raise ValueError("address is required") - - def __repr__(self): - return "[name=%s match=%s proto=%s address=%s port=%s]" % ( - self.name, self.match, self.proto, self.addr, self.port - ) - - def __str__(self): - buf = self.match + " " - if self.proto == "udp": - buf += "@" - elif self.proto == "tcp": - buf += "@@" - - if ":" in self.addr: - buf += "[" + self.addr + "]" - else: - buf += self.addr - - if self.port: - buf += ":%s" % self.port - - if self.name: - buf += " # %s" % self.name - return buf - - -def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): - if not remotes: - return None - lines = [] - if header is not None: - lines.append(header) - for name, line in remotes.items(): - if not line: - continue - try: - lines.append(str(parse_remotes_line(line, name=name))) - except ValueError as e: - LOG.warn("failed loading remote %s: %s [%s]", name, line, e) - if footer is not None: - lines.append(footer) - return '\n'.join(lines) + "\n" - - -def handle(name, cfg, cloud, log, _args): - if 'rsyslog' not in cfg: - log.debug(("Skipping module named %s," - " no 'rsyslog' key in configuration"), name) - return - - mycfg = load_config(cfg) - configs = mycfg[KEYNAME_CONFIGS] - - if mycfg[KEYNAME_REMOTES]: - configs.append( - remotes_to_rsyslog_cfg( - mycfg[KEYNAME_REMOTES], - header="# begin remotes", - footer="# end remotes", - )) - - if not mycfg['configs']: - log.debug("Empty config rsyslog['configs'], nothing to do") - return - - changes = apply_rsyslog_changes( - configs=mycfg[KEYNAME_CONFIGS], - def_fname=mycfg[KEYNAME_FILENAME], - cfg_dir=mycfg[KEYNAME_DIR]) - - if not changes: - log.debug("restart of syslog not necessary, no changes made") - return - - try: - restarted = reload_syslog( - command=mycfg[KEYNAME_RELOAD], - systemd=cloud.distro.uses_systemd()), - except util.ProcessExecutionError as e: - restarted = False - log.warn("Failed to reload syslog", e) - - if restarted: - # This only needs to run if we *actually* restarted - # syslog above. - cloud.cycle_logging() - # This should now use rsyslog if - # the logging was setup to use it... - log.debug("%s configured %s files", name, changes) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py deleted file mode 100644 index bc09d38c..00000000 --- a/cloudinit/config/cc_runcmd.py +++ /dev/null @@ -1,38 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - - -def handle(name, cfg, cloud, log, _args): - if "runcmd" not in cfg: - log.debug(("Skipping module named %s," - " no 'runcmd' key in configuration"), name) - return - - out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd") - cmd = cfg["runcmd"] - try: - content = util.shellify(cmd) - util.write_file(out_fn, content, 0o700) - except Exception: - util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py deleted file mode 100644 index f5786a31..00000000 --- a/cloudinit/config/cc_salt_minion.py +++ /dev/null @@ -1,59 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Jeff Bauer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -# Note: see http://saltstack.org/topics/installation/ - - -def handle(name, cfg, cloud, log, _args): - # If there isn't a salt key in the configuration don't do anything - if 'salt_minion' not in cfg: - log.debug(("Skipping module named %s," - " no 'salt_minion' key in configuration"), name) - return - - salt_cfg = cfg['salt_minion'] - - # Start by installing the salt package ... - cloud.distro.install_packages(('salt-minion',)) - - # Ensure we can configure files at the right dir - config_dir = salt_cfg.get("config_dir", '/etc/salt') - util.ensure_dir(config_dir) - - # ... and then update the salt configuration - if 'conf' in salt_cfg: - # Add all sections from the conf object to /etc/salt/minion - minion_config = os.path.join(config_dir, 'minion') - minion_data = util.yaml_dumps(salt_cfg.get('conf')) - util.write_file(minion_config, minion_data) - - # ... copy the key pair if specified - if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') - with util.umask(0o77): - util.ensure_dir(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - util.write_file(pub_name, salt_cfg['public_key']) - util.write_file(pem_name, salt_cfg['private_key']) - - # restart salt-minion. 'service' will start even if not started. if it - # was started, it needs to be restarted for config change. - util.subp(['service', 'salt-minion', 'restart'], capture=False) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py deleted file mode 100644 index ee3b6c9f..00000000 --- a/cloudinit/config/cc_scripts_per_boot.py +++ /dev/null @@ -1,41 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -from cloudinit.settings import PER_ALWAYS - -frequency = PER_ALWAYS - -SCRIPT_SUBDIR = 'per-boot' - - -def handle(name, _cfg, cloud, log, _args): - # Comes from the following: - # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) - try: - util.runparts(runparts_path) - except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) - raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py deleted file mode 100644 index c0d62b12..00000000 --- a/cloudinit/config/cc_scripts_per_instance.py +++ /dev/null @@ -1,41 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - -SCRIPT_SUBDIR = 'per-instance' - - -def handle(name, _cfg, cloud, log, _args): - # Comes from the following: - # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) - try: - util.runparts(runparts_path) - except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) - raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py deleted file mode 100644 index ecb527f6..00000000 --- a/cloudinit/config/cc_scripts_per_once.py +++ /dev/null @@ -1,41 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -from cloudinit.settings import PER_ONCE - -frequency = PER_ONCE - -SCRIPT_SUBDIR = 'per-once' - - -def handle(name, _cfg, cloud, log, _args): - # Comes from the following: - # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) - try: - util.runparts(runparts_path) - except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) - raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py deleted file mode 100644 index 699857d1..00000000 --- a/cloudinit/config/cc_scripts_user.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - -SCRIPT_SUBDIR = 'scripts' - - -def handle(name, _cfg, cloud, log, _args): - # This is written to by the user data handlers - # Ie, any custom shell scripts that come down - # go here... - runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) - try: - util.runparts(runparts_path) - except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) - raise diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py deleted file mode 100644 index 80bf10ff..00000000 --- a/cloudinit/config/cc_scripts_vendor.py +++ /dev/null @@ -1,43 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import util - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - -SCRIPT_SUBDIR = 'vendor' - - -def handle(name, cfg, cloud, log, _args): - # This is written to by the vendor data handlers - # any vendor data shell scripts get placed in runparts_path - runparts_path = os.path.join(cloud.get_ipath_cur(), 'scripts', - SCRIPT_SUBDIR) - - prefix = util.get_cfg_by_path(cfg, ('vendor_data', 'prefix'), []) - - try: - util.runparts(runparts_path, exe_prefix=prefix) - except Exception: - log.warn("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) - raise diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py deleted file mode 100644 index 5085c23a..00000000 --- a/cloudinit/config/cc_seed_random.py +++ /dev/null @@ -1,94 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Yahoo! Inc. -# Copyright (C) 2014 Canonical, Ltd -# -# Author: Joshua Harlow -# Author: Dustin Kirkland -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 -import os - -from six import BytesIO - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -frequency = PER_INSTANCE -LOG = logging.getLogger(__name__) - - -def _decode(data, encoding=None): - if not data: - return b'' - if not encoding or encoding.lower() in ['raw']: - return util.encode_text(data) - elif encoding.lower() in ['base64', 'b64']: - return base64.b64decode(data) - elif encoding.lower() in ['gzip', 'gz']: - return util.decomp_gzip(data, quiet=False, decode=None) - else: - raise IOError("Unknown random_seed encoding: %s" % (encoding)) - - -def handle_random_seed_command(command, required, env=None): - if not command and required: - raise ValueError("no command found but required=true") - elif not command: - LOG.debug("no command provided") - return - - cmd = command[0] - if not util.which(cmd): - if required: - raise ValueError("command '%s' not found but required=true", cmd) - else: - LOG.debug("command '%s' not found for seed_command", cmd) - return - util.subp(command, env=env, capture=False) - - -def handle(name, cfg, cloud, log, _args): - mycfg = cfg.get('random_seed', {}) - seed_path = mycfg.get('file', '/dev/urandom') - seed_data = mycfg.get('data', b'') - - seed_buf = BytesIO() - if seed_data: - seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding'))) - - # 'random_seed' is set up by Azure datasource, and comes already in - # openstack meta_data.json - metadata = cloud.datasource.metadata - if metadata and 'random_seed' in metadata: - seed_buf.write(util.encode_text(metadata['random_seed'])) - - seed_data = seed_buf.getvalue() - if len(seed_data): - log.debug("%s: adding %s bytes of random seed entropy to %s", name, - len(seed_data), seed_path) - util.append_file(seed_path, seed_data) - - command = mycfg.get('command', None) - req = mycfg.get('command_required', False) - try: - env = os.environ.copy() - env['RANDOM_SEED_FILE'] = seed_path - handle_random_seed_command(command=command, required=req, env=env) - except ValueError as e: - log.warn("handling random command [%s] failed: %s", command, e) - raise e diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py deleted file mode 100644 index f43d8d5a..00000000 --- a/cloudinit/config/cc_set_hostname.py +++ /dev/null @@ -1,37 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import util - - -def handle(name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) - return - - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - log.debug("Setting the hostname to %s (%s)", fqdn, hostname) - cloud.distro.set_hostname(hostname, fqdn) - except Exception: - util.logexc(log, "Failed to set the hostname to %s (%s)", fqdn, - hostname) - raise diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py deleted file mode 100644 index 5c8c23b8..00000000 --- a/cloudinit/config/cc_set_passwords.py +++ /dev/null @@ -1,167 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - -from cloudinit import ssh_util -from cloudinit import util - -from string import ascii_letters, digits - -# We are removing certain 'painful' letters/numbers -PW_SET = (''.join([x for x in ascii_letters + digits - if x not in 'loLOI01'])) - - -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - # if run from command line, and give args, wipe the chpasswd['list'] - password = args[0] - if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: - del cfg['chpasswd']['list'] - else: - password = util.get_cfg_option_str(cfg, "password", None) - - expire = True - plist = None - - if 'chpasswd' in cfg: - chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg, 'list', plist) - expire = util.get_cfg_option_bool(chfg, 'expire', expire) - - if not plist and password: - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - (user, _user_config) = ds.extract_default(users) - if user: - plist = "%s:%s" % (user, password) - else: - log.warn("No default or defined user to change password for.") - - errors = [] - if plist: - plist_in = [] - randlist = [] - users = [] - for line in plist.splitlines(): - u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) - - ch_in = '\n'.join(plist_in) + '\n' - try: - log.debug("Changing password for %s:", users) - util.subp(['chpasswd'], ch_in) - except Exception as e: - errors.append(e) - util.logexc(log, "Failed to set passwords with chpasswd for %s", - users) - - if len(randlist): - blurb = ("Set the following 'random' passwords\n", - '\n'.join(randlist)) - sys.stderr.write("%s\n%s\n" % blurb) - - if expire: - expired_users = [] - for u in users: - try: - util.subp(['passwd', '--expire', u]) - expired_users.append(u) - except Exception as e: - errors.append(e) - util.logexc(log, "Failed to set 'expire' for %s", u) - if expired_users: - log.debug("Expired passwords for: %s users", expired_users) - - change_pwauth = False - pw_auth = None - if 'ssh_pwauth' in cfg: - if util.is_true(cfg['ssh_pwauth']): - change_pwauth = True - pw_auth = 'yes' - elif util.is_false(cfg['ssh_pwauth']): - change_pwauth = True - pw_auth = 'no' - elif str(cfg['ssh_pwauth']).lower() == 'unchanged': - log.debug('Leaving auth line unchanged') - change_pwauth = False - elif not str(cfg['ssh_pwauth']).strip(): - log.debug('Leaving auth line unchanged') - change_pwauth = False - elif not cfg['ssh_pwauth']: - log.debug('Leaving auth line unchanged') - change_pwauth = False - else: - msg = 'Unrecognized value %s for ssh_pwauth' % cfg['ssh_pwauth'] - util.logexc(log, msg) - - if change_pwauth: - replaced_auth = False - - # See: man sshd_config - old_lines = ssh_util.parse_ssh_config(ssh_util.DEF_SSHD_CFG) - new_lines = [] - i = 0 - for (i, line) in enumerate(old_lines): - # Keywords are case-insensitive and arguments are case-sensitive - if line.key == 'passwordauthentication': - log.debug("Replacing auth line %s with %s", i + 1, pw_auth) - replaced_auth = True - line.value = pw_auth - new_lines.append(line) - - if not replaced_auth: - log.debug("Adding new auth line %s", i + 1) - replaced_auth = True - new_lines.append(ssh_util.SshdConfigLine('', - 'PasswordAuthentication', - pw_auth)) - - lines = [str(l) for l in new_lines] - util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) - - try: - cmd = cloud.distro.init_cmd # Default service - cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) - cmd.append('restart') - if 'systemctl' in cmd: # Switch action ordering - cmd[1], cmd[2] = cmd[2], cmd[1] - cmd = filter(None, cmd) # Remove empty arguments - util.subp(cmd) - log.debug("Restarted the ssh daemon") - except Exception: - util.logexc(log, "Restarting of the ssh daemon failed") - - if len(errors): - log.debug("%s errors occured, re-raising the last one", len(errors)) - raise errors[-1] - - -def rand_user_password(pwlen=9): - return util.rand_str(pwlen, select_from=PW_SET) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py deleted file mode 100644 index 1a485ee6..00000000 --- a/cloudinit/config/cc_snappy.py +++ /dev/null @@ -1,304 +0,0 @@ -# vi: ts=4 expandtab -# -""" -snappy modules allows configuration of snappy. -Example config: - #cloud-config - snappy: - system_snappy: auto - ssh_enabled: auto - packages: [etcd, pkg2.smoser] - config: - pkgname: - key2: value2 - pkg2: - key1: value1 - packages_dir: '/writable/user-data/cloud-init/snaps' - - - ssh_enabled: - This controls the system's ssh service. The default value is 'auto'. - True: enable ssh service - False: disable ssh service - auto: enable ssh service if either ssh keys have been provided - or user has requested password authentication (ssh_pwauth). - - - snap installation and config - The above would install 'etcd', and then install 'pkg2.smoser' with a - '' argument where 'config-file' has 'config-blob' inside it. - If 'pkgname' is installed already, then 'snappy config pkgname ' - will be called where 'file' has 'pkgname-config-blob' as its content. - - Entries in 'config' can be namespaced or non-namespaced for a package. - In either case, the config provided to snappy command is non-namespaced. - The package name is provided as it appears. - - If 'packages_dir' has files in it that end in '.snap', then they are - installed. Given 3 files: - /foo.snap - /foo.config - /bar.snap - cloud-init will invoke: - snappy install /foo.snap /foo.config - snappy install /bar.snap - - Note, that if provided a 'config' entry for 'ubuntu-core', then - cloud-init will invoke: snappy config ubuntu-core - Allowing you to configure ubuntu-core in this way. -""" - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -import glob -import os -import tempfile - -LOG = logging.getLogger(__name__) - -frequency = PER_INSTANCE -SNAPPY_CMD = "snappy" -NAMESPACE_DELIM = '.' - -BUILTIN_CFG = { - 'packages': [], - 'packages_dir': '/writable/user-data/cloud-init/snaps', - 'ssh_enabled': "auto", - 'system_snappy': "auto", - 'config': {}, -} - - -def parse_filename(fname): - fname = os.path.basename(fname) - fname_noext = fname.rpartition(".")[0] - name = fname_noext.partition("_")[0] - shortname = name.partition(".")[0] - return(name, shortname, fname_noext) - - -def get_fs_package_ops(fspath): - if not fspath: - return [] - ops = [] - for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): - (name, shortname, fname_noext) = parse_filename(snapfile) - cfg = None - for cand in (fname_noext, name, shortname): - fpcand = os.path.sep.join([fspath, cand]) + ".config" - if os.path.isfile(fpcand): - cfg = fpcand - break - ops.append(makeop('install', name, config=None, - path=snapfile, cfgfile=cfg)) - return ops - - -def makeop(op, name, config=None, path=None, cfgfile=None): - return({'op': op, 'name': name, 'config': config, 'path': path, - 'cfgfile': cfgfile}) - - -def get_package_config(configs, name): - # load the package's config from the configs dict. - # prefer full-name entry (config-example.canonical) - # over short name entry (config-example) - if name in configs: - return configs[name] - return configs.get(name.partition(NAMESPACE_DELIM)[0]) - - -def get_package_ops(packages, configs, installed=None, fspath=None): - # get the install an config operations that should be done - if installed is None: - installed = read_installed_packages() - short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] - - if not packages: - packages = [] - if not configs: - configs = {} - - ops = [] - ops += get_fs_package_ops(fspath) - - for name in packages: - ops.append(makeop('install', name, get_package_config(configs, name))) - - to_install = [f['name'] for f in ops] - short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] - - for name in configs: - if name in to_install: - continue - shortname = name.partition(NAMESPACE_DELIM)[0] - if shortname in short_to_install: - continue - if name in installed or shortname in short_installed: - ops.append(makeop('config', name, - config=get_package_config(configs, name))) - - # prefer config entries to filepath entries - for op in ops: - if op['op'] != 'install' or not op['cfgfile']: - continue - name = op['name'] - fromcfg = get_package_config(configs, op['name']) - if fromcfg: - LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) - op['cfgfile'] = None - op['config'] = fromcfg - - return ops - - -def render_snap_op(op, name, path=None, cfgfile=None, config=None): - if op not in ('install', 'config'): - raise ValueError("cannot render op '%s'" % op) - - shortname = name.partition(NAMESPACE_DELIM)[0] - try: - cfg_tmpf = None - if config is not None: - # input to 'snappy config packagename' must have nested data. odd. - # config: - # packagename: - # config - # Note, however, we do not touch config files on disk. - nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = tempfile.mkstemp() - os.write(fd, util.yaml_dumps(nested_cfg).encode()) - os.close(fd) - cfgfile = cfg_tmpf - - cmd = [SNAPPY_CMD, op] - if op == 'install': - if path: - cmd.append("--allow-unauthenticated") - cmd.append(path) - else: - cmd.append(name) - if cfgfile: - cmd.append(cfgfile) - elif op == 'config': - cmd += [name, cfgfile] - - util.subp(cmd) - - finally: - if cfg_tmpf: - os.unlink(cfg_tmpf) - - -def read_installed_packages(): - ret = [] - for (name, date, version, dev) in read_pkg_data(): - if dev: - ret.append(NAMESPACE_DELIM.join([name, dev])) - else: - ret.append(name) - return ret - - -def read_pkg_data(): - out, err = util.subp([SNAPPY_CMD, "list"]) - pkg_data = [] - for line in out.splitlines()[1:]: - toks = line.split(sep=None, maxsplit=3) - if len(toks) == 3: - (name, date, version) = toks - dev = None - else: - (name, date, version, dev) = toks - pkg_data.append((name, date, version, dev,)) - return pkg_data - - -def disable_enable_ssh(enabled): - LOG.debug("setting enablement of ssh to: %s", enabled) - # do something here that would enable or disable - not_to_be_run = "/etc/ssh/sshd_not_to_be_run" - if enabled: - util.del_file(not_to_be_run) - # this is an indempotent operation - util.subp(["systemctl", "start", "ssh"]) - else: - # this is an indempotent operation - util.subp(["systemctl", "stop", "ssh"]) - util.write_file(not_to_be_run, "cloud-init\n") - - -def system_is_snappy(): - # channel.ini is configparser loadable. - # snappy will move to using /etc/system-image/config.d/*.ini - # this is certainly not a perfect test, but good enough for now. - content = util.load_file("/etc/system-image/channel.ini", quiet=True) - if 'ubuntu-core' in content.lower(): - return True - if os.path.isdir("/etc/system-image/config.d/"): - return True - return False - - -def set_snappy_command(): - global SNAPPY_CMD - if util.which("snappy-go"): - SNAPPY_CMD = "snappy-go" - else: - SNAPPY_CMD = "snappy" - LOG.debug("snappy command is '%s'", SNAPPY_CMD) - - -def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('snappy') - if not cfgin: - cfgin = {} - mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - - sys_snappy = str(mycfg.get("system_snappy", "auto")) - if util.is_false(sys_snappy): - LOG.debug("%s: System is not snappy. disabling", name) - return - - if sys_snappy.lower() == "auto" and not(system_is_snappy()): - LOG.debug("%s: 'auto' mode, and system not snappy", name) - return - - set_snappy_command() - - pkg_ops = get_package_ops(packages=mycfg['packages'], - configs=mycfg['config'], - fspath=mycfg['packages_dir']) - - fails = [] - for pkg_op in pkg_ops: - try: - render_snap_op(**pkg_op) - except Exception as e: - fails.append((pkg_op, e,)) - LOG.warn("'%s' failed for '%s': %s", - pkg_op['op'], pkg_op['name'], e) - - # Default to disabling SSH - ssh_enabled = mycfg.get('ssh_enabled', "auto") - - # If the user has not explicitly enabled or disabled SSH, then enable it - # when password SSH authentication is requested or there are SSH keys - if ssh_enabled == "auto": - user_ssh_keys = cloud.get_public_ssh_keys() or None - password_auth_enabled = cfg.get('ssh_pwauth', False) - if user_ssh_keys: - LOG.debug("Enabling SSH, ssh keys found in datasource") - ssh_enabled = True - elif cfg.get('ssh_authorized_keys'): - LOG.debug("Enabling SSH, ssh keys found in config") - elif password_auth_enabled: - LOG.debug("Enabling SSH, password authentication requested") - ssh_enabled = True - elif ssh_enabled not in (True, False): - LOG.warn("Unknown value '%s' in ssh_enabled", ssh_enabled) - - disable_enable_ssh(ssh_enabled) - - if fails: - raise Exception("failed to install/configure snaps") diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py deleted file mode 100644 index cb9b70aa..00000000 --- a/cloudinit/config/cc_ssh.py +++ /dev/null @@ -1,142 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import glob -import os -import sys - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - -from cloudinit import ssh_util -from cloudinit import util - -DISABLE_ROOT_OPTS = ( - "no-port-forwarding,no-agent-forwarding," - "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\"" - " rather than the user \\\"root\\\".\';echo;sleep 10\"") - -GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] -KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' - -CONFIG_KEY_TO_FILE = {} -PRIV_TO_PUB = {} -for k in GENERATE_KEY_NAMES: - CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) - CONFIG_KEY_TO_FILE.update( - {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) - PRIV_TO_PUB["%s_private" % k] = "%s_public" % k - -KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' - - -def handle(_name, cfg, cloud, log, _args): - - # remove the static keys from the pristine image - if cfg.get("ssh_deletekeys", True): - key_pth = os.path.join("/etc/ssh/", "ssh_host_*key*") - for f in glob.glob(key_pth): - try: - util.del_file(f) - except Exception: - util.logexc(log, "Failed deleting key file %s", f) - - if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them - for (key, val) in cfg["ssh_keys"].items(): - if key in CONFIG_KEY_TO_FILE: - tgt_fn = CONFIG_KEY_TO_FILE[key][0] - tgt_perms = CONFIG_KEY_TO_FILE[key][1] - util.write_file(tgt_fn, val, tgt_perms) - - for (priv, pub) in PRIV_TO_PUB.items(): - if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: - continue - pair = (CONFIG_KEY_TO_FILE[priv][0], CONFIG_KEY_TO_FILE[pub][0]) - cmd = ['sh', '-xc', KEY_GEN_TPL % pair] - try: - # TODO(harlowja): Is this guard needed? - with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) - log.debug("Generated a key for %s from %s", pair[0], pair[1]) - except Exception: - util.logexc(log, "Failed generated a key for %s from %s", - pair[0], pair[1]) - else: - # if not, generate them - genkeys = util.get_cfg_option_list(cfg, - 'ssh_genkeytypes', - GENERATE_KEY_NAMES) - lang_c = os.environ.copy() - lang_c['LANG'] = 'C' - for keytype in genkeys: - keyfile = KEY_FILE_TPL % (keytype) - if os.path.exists(keyfile): - continue - util.ensure_dir(os.path.dirname(keyfile)) - cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] - - # TODO(harlowja): Is this guard needed? - with util.SeLinuxGuard("/etc/ssh", recursive=True): - try: - out, err = util.subp(cmd, capture=True, env=lang_c) - sys.stdout.write(util.decode_binary(out)) - except util.ProcessExecutionError as e: - err = util.decode_binary(e.stderr).lower() - if (e.exit_code == 1 and - err.lower().startswith("unknown key")): - log.debug("ssh-keygen: unknown key type '%s'", keytype) - else: - util.logexc(log, "Failed generating key type %s to " - "file %s", keytype, keyfile) - - try: - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - (user, _user_config) = ds.extract_default(users) - disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) - disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) - - keys = cloud.get_public_ssh_keys() or [] - if "ssh_authorized_keys" in cfg: - cfgkeys = cfg["ssh_authorized_keys"] - keys.extend(cfgkeys) - - apply_credentials(keys, user, disable_root, disable_root_opts) - except Exception: - util.logexc(log, "Applying ssh credentials failed!") - - -def apply_credentials(keys, user, disable_root, disable_root_opts): - - keys = set(keys) - if user: - ssh_util.setup_user_keys(keys, user) - - if disable_root: - if not user: - user = "NONE" - key_prefix = disable_root_opts.replace('$USER', user) - else: - key_prefix = '' - - ssh_util.setup_user_keys(keys, 'root', options=key_prefix) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py deleted file mode 100644 index 6ce831bc..00000000 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ /dev/null @@ -1,105 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 -import hashlib - -from prettytable import PrettyTable - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - -from cloudinit import ssh_util -from cloudinit import util - - -def _split_hash(bin_hash): - split_up = [] - for i in range(0, len(bin_hash), 2): - split_up.append(bin_hash[i:i + 2]) - return split_up - - -def _gen_fingerprint(b64_text, hash_meth='md5'): - if not b64_text: - return '' - # TBD(harlowja): Maybe we should feed this into 'ssh -lf'? - try: - hasher = hashlib.new(hash_meth) - hasher.update(base64.b64decode(b64_text)) - return ":".join(_split_hash(hasher.hexdigest())) - except (TypeError, ValueError): - # Raised when b64 not really b64... - # or when the hash type is not really - # a known/supported hash type... - return '?' - - -def _is_printable_key(entry): - if any([entry.keytype, entry.base64, entry.comment, entry.options]): - if (entry.keytype and - entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): - return True - return False - - -def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', - prefix='ci-info: '): - if not key_entries: - message = ("%sno authorized ssh keys fingerprints found for user %s.\n" - % (prefix, user)) - util.multi_log(message) - return - tbl_fields = ['Keytype', 'Fingerprint (%s)' % (hash_meth), 'Options', - 'Comment'] - tbl = PrettyTable(tbl_fields) - for entry in key_entries: - if _is_printable_key(entry): - row = [] - row.append(entry.keytype or '-') - row.append(_gen_fingerprint(entry.base64, hash_meth) or '-') - row.append(entry.options or '-') - row.append(entry.comment or '-') - tbl.add_row(row) - authtbl_s = tbl.get_string() - authtbl_lines = authtbl_s.splitlines() - max_len = len(max(authtbl_lines, key=len)) - lines = [ - util.center("Authorized keys from %s for user %s" % - (key_fn, user), "+", max_len), - ] - lines.extend(authtbl_lines) - for line in lines: - util.multi_log(text="%s%s\n" % (prefix, line), - stderr=False, console=True) - - -def handle(name, cfg, cloud, log, _args): - if util.is_true(cfg.get('no_ssh_fingerprints', False)): - log.debug(("Skipping module named %s, " - "logging of ssh fingerprints disabled"), name) - return - - hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - for (user_name, _cfg) in users.items(): - (key_fn, key_entries) = ssh_util.extract_authorized_keys(user_name) - _pprint_key_entries(user_name, key_fn, - key_entries, hash_meth) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py deleted file mode 100644 index 28c4585b..00000000 --- a/cloudinit/config/cc_ssh_import_id.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - -from cloudinit import util -import pwd - -# https://launchpad.net/ssh-import-id -distros = ['ubuntu', 'debian'] - - -def handle(_name, cfg, cloud, log, args): - - # import for "user: XXXXX" - if len(args) != 0: - user = args[0] - ids = [] - if len(args) > 1: - ids = args[1:] - - import_ssh_ids(ids, user, log) - return - - # import for cloudinit created users - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - elist = [] - for (user, user_cfg) in users.items(): - import_ids = [] - if user_cfg['default']: - import_ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) - else: - try: - import_ids = user_cfg['ssh_import_id'] - except Exception: - log.debug("User %s is not configured for ssh_import_id", user) - continue - - try: - import_ids = util.uniq_merge(import_ids) - import_ids = [str(i) for i in import_ids] - except Exception: - log.debug("User %s is not correctly configured for ssh_import_id", - user) - continue - - if not len(import_ids): - continue - - try: - import_ssh_ids(import_ids, user, log) - except Exception as exc: - util.logexc(log, "ssh-import-id failed for: %s %s", user, - import_ids) - elist.append(exc) - - if len(elist): - raise elist[0] - - -def import_ssh_ids(ids, user, log): - - if not (user and ids): - log.debug("empty user(%s) or ids(%s). not importing", user, ids) - return - - try: - pwd.getpwnam(user) - except KeyError as exc: - raise exc - - cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - log.debug("Importing ssh ids for user %s.", user) - - try: - util.subp(cmd, capture=False) - except util.ProcessExecutionError as exc: - util.logexc(log, "Failed to run command to import %s ssh ids", user) - raise exc diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py deleted file mode 100644 index b9eb85b2..00000000 --- a/cloudinit/config/cc_timezone.py +++ /dev/null @@ -1,39 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import util - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - - -def handle(name, cfg, cloud, log, args): - if len(args) != 0: - timezone = args[0] - else: - timezone = util.get_cfg_option_str(cfg, "timezone", False) - - if not timezone: - log.debug("Skipping module named %s, no 'timezone' specified", name) - return - - # Let the distro handle settings its timezone - cloud.distro.set_timezone(timezone) diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py deleted file mode 100644 index 884d79f1..00000000 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ /dev/null @@ -1,162 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -**Summary:** reboot system into another init. - -**Description:** This module provides a way for the user to boot with systemd -even if the image is set to boot with upstart. It should be run as one of the -first ``cloud_init_modules``, and will switch the init system and then issue a -reboot. The next boot will come up in the target init system and no action will -be taken. - -This should be inert on non-ubuntu systems, and also exit quickly. - -It can be configured with the following option structure:: - - init_switch: - target: systemd (can be 'systemd' or 'upstart') - reboot: true (reboot if a change was made, or false to not reboot) - -.. note:: - - Best effort is made, but it's possible - this system will break, and probably won't interact well with any other - mechanism you've used to switch the init system. -""" - -from cloudinit.distros import ubuntu -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -import os -import time - -frequency = PER_INSTANCE -REBOOT_CMD = ["/sbin/reboot", "--force"] - -DEFAULT_CONFIG = { - 'init_switch': {'target': None, 'reboot': True} -} - -SWITCH_INIT = """ -#!/bin/sh -# switch_init: [upstart | systemd] - -is_systemd() { - [ "$(dpkg-divert --listpackage /sbin/init)" = "systemd-sysv" ] -} -debug() { echo "$@" 1>&2; } -fail() { echo "$@" 1>&2; exit 1; } - -if [ "$1" = "systemd" ]; then - if is_systemd; then - debug "already systemd, nothing to do" - else - [ -f /lib/systemd/systemd ] || fail "no systemd available"; - dpkg-divert --package systemd-sysv --divert /sbin/init.diverted \\ - --rename /sbin/init - fi - [ -f /sbin/init ] || ln /lib/systemd/systemd /sbin/init -elif [ "$1" = "upstart" ]; then - if is_systemd; then - rm -f /sbin/init - dpkg-divert --package systemd-sysv --rename --remove /sbin/init - else - debug "already upstart, nothing to do." - fi -else - fail "Error. expect 'upstart' or 'systemd'" -fi -""" - - -def handle(name, cfg, cloud, log, args): - """Handler method activated by cloud-init.""" - - if not isinstance(cloud.distro, ubuntu.Distro): - log.debug("%s: distro is '%s', not ubuntu. returning", - name, cloud.distro.__class__) - return - - cfg = util.mergemanydict([cfg, DEFAULT_CONFIG]) - target = cfg['init_switch']['target'] - reboot = cfg['init_switch']['reboot'] - - if len(args) != 0: - target = args[0] - if len(args) > 1: - reboot = util.is_true(args[1]) - - if not target: - log.debug("%s: target=%s. nothing to do", name, target) - return - - if not util.which('dpkg'): - log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name) - return - - supported = ('upstart', 'systemd') - if target not in supported: - log.warn("%s: target set to %s, expected one of: %s", - name, target, str(supported)) - - if os.path.exists("/run/systemd/system"): - current = "systemd" - else: - current = "upstart" - - if current == target: - log.debug("%s: current = target = %s. nothing to do", name, target) - return - - try: - util.subp(['sh', '-s', target], data=SWITCH_INIT) - except util.ProcessExecutionError as e: - log.warn("%s: Failed to switch to init '%s'. %s", name, target, e) - return - - if util.is_false(reboot): - log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.", - name, current, target) - return - - try: - log.warn("%s: switched '%s' to '%s'. rebooting.", - name, current, target) - logging.flushLoggers(log) - _fire_reboot(log, wait_attempts=4, initial_sleep=4) - except Exception as e: - util.logexc(log, "Requested reboot did not happen!") - raise - - -def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) - start = time.time() - wait_time = initial_sleep - for _i in range(0, wait_attempts): - time.sleep(wait_time) - wait_time *= backoff - elapsed = time.time() - start - log.debug("Rebooted, but still running after %s seconds", int(elapsed)) - # If we got here, not good - elapsed = time.time() - start - raise RuntimeError(("Reboot did not happen" - " after %s seconds!") % (int(elapsed))) diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py deleted file mode 100644 index 15703efe..00000000 --- a/cloudinit/config/cc_update_etc_hosts.py +++ /dev/null @@ -1,60 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import templater -from cloudinit import util - -from cloudinit.settings import PER_ALWAYS - -frequency = PER_ALWAYS - - -def handle(name, cfg, cloud, log, _args): - manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if util.translate_bool(manage_hosts, addons=['template']): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - if not hostname: - log.warn(("Option 'manage_etc_hosts' was set," - " but no hostname was found")) - return - - # Render from a template file - tpl_fn_name = cloud.get_template_filename("hosts.%s" % - (cloud.distro.osfamily)) - if not tpl_fn_name: - raise RuntimeError(("No hosts template could be" - " found for distro %s") % - (cloud.distro.osfamily)) - - templater.render_to_file(tpl_fn_name, '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) - - elif manage_hosts == "localhost": - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - if not hostname: - log.warn(("Option 'manage_etc_hosts' was set," - " but no hostname was found")) - return - - log.debug("Managing localhost in /etc/hosts") - cloud.distro.update_etc_hosts(hostname, fqdn) - else: - log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py deleted file mode 100644 index 5b78afe1..00000000 --- a/cloudinit/config/cc_update_hostname.py +++ /dev/null @@ -1,43 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit.settings import PER_ALWAYS -from cloudinit import util - -frequency = PER_ALWAYS - - -def handle(name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) - return - - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") - log.debug("Updating hostname to %s (%s)", fqdn, hostname) - cloud.distro.update_hostname(hostname, fqdn, prev_fn) - except Exception: - util.logexc(log, "Failed to update the hostname to %s (%s)", fqdn, - hostname) - raise diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py deleted file mode 100644 index bf5b4581..00000000 --- a/cloudinit/config/cc_users_groups.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - - -def handle(name, cfg, cloud, _log, _args): - (users, groups) = ds.normalize_users_groups(cfg, cloud.distro) - for (name, members) in groups.items(): - cloud.distro.create_group(name, members) - for (user, config) in users.items(): - cloud.distro.create_user(user, **config) diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py deleted file mode 100644 index b1096b9b..00000000 --- a/cloudinit/config/cc_write_files.py +++ /dev/null @@ -1,105 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 -import os -import six - -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -frequency = PER_INSTANCE - -DEFAULT_OWNER = "root:root" -DEFAULT_PERMS = 0o644 -UNKNOWN_ENC = 'text/plain' - - -def handle(name, cfg, _cloud, log, _args): - files = cfg.get('write_files') - if not files: - log.debug(("Skipping module named %s," - " no/empty 'write_files' key in configuration"), name) - return - write_files(name, files, log) - - -def canonicalize_extraction(encoding_type, log): - if not encoding_type: - encoding_type = '' - encoding_type = encoding_type.lower().strip() - if encoding_type in ['gz', 'gzip']: - return ['application/x-gzip'] - if encoding_type in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']: - return ['application/base64', 'application/x-gzip'] - # Yaml already encodes binary data as base64 if it is given to the - # yaml file as binary, so those will be automatically decoded for you. - # But the above b64 is just for people that are more 'comfortable' - # specifing it manually (which might be a possiblity) - if encoding_type in ['b64', 'base64']: - return ['application/base64'] - if encoding_type: - log.warn("Unknown encoding type %s, assuming %s", - encoding_type, UNKNOWN_ENC) - return [UNKNOWN_ENC] - - -def write_files(name, files, log): - if not files: - return - - for (i, f_info) in enumerate(files): - path = f_info.get('path') - if not path: - log.warn("No path provided to write for entry %s in module %s", - i + 1, name) - continue - path = os.path.abspath(path) - extractions = canonicalize_extraction(f_info.get('encoding'), log) - contents = extract_contents(f_info.get('content', ''), extractions) - (u, g) = util.extract_usergroup(f_info.get('owner', DEFAULT_OWNER)) - perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS, log) - util.write_file(path, contents, mode=perms) - util.chownbyname(path, u, g) - - -def decode_perms(perm, default, log): - if perm is None: - return default - try: - if isinstance(perm, six.integer_types + (float,)): - # Just 'downcast' it (if a float) - return int(perm) - else: - # Force to string and try octal conversion - return int(str(perm), 8) - except (TypeError, ValueError): - log.warn("Undecodable permissions %s, assuming %s", perm, default) - return default - - -def extract_contents(contents, extraction_types): - result = contents - for t in extraction_types: - if t == 'application/x-gzip': - result = util.decomp_gzip(result, quiet=False, decode=False) - elif t == 'application/base64': - result = base64.b64decode(result) - elif t == UNKNOWN_ENC: - pass - return result diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py deleted file mode 100644 index 64fba869..00000000 --- a/cloudinit/config/cc_yum_add_repo.py +++ /dev/null @@ -1,107 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -import configobj -import six - -from cloudinit import util - - -def _canonicalize_id(repo_id): - repo_id = repo_id.lower().replace("-", "_") - repo_id = repo_id.replace(" ", "_") - return repo_id - - -def _format_repo_value(val): - if isinstance(val, (bool)): - # Seems like yum prefers 1/0 - return str(int(val)) - if isinstance(val, (list, tuple)): - # Can handle 'lists' in certain cases - # See: http://bit.ly/Qqrf1t - return "\n ".join([_format_repo_value(v) for v in val]) - if not isinstance(val, six.string_types): - return str(val) - return val - - -# TODO(harlowja): move to distro? -# See man yum.conf -def _format_repository_config(repo_id, repo_config): - to_be = configobj.ConfigObj() - to_be[repo_id] = {} - # Do basic translation of the items -> values - for (k, v) in repo_config.items(): - # For now assume that people using this know - # the format of yum and don't verify keys/values further - to_be[repo_id][k] = _format_repo_value(v) - lines = to_be.write() - lines.insert(0, "# Created by cloud-init on %s" % (util.time_rfc2822())) - return "\n".join(lines) - - -def handle(name, cfg, _cloud, log, _args): - repos = cfg.get('yum_repos') - if not repos: - log.debug(("Skipping module named %s," - " no 'yum_repos' configuration found"), name) - return - repo_base_path = util.get_cfg_option_str(cfg, 'yum_repo_dir', - '/etc/yum.repos.d/') - repo_locations = {} - repo_configs = {} - for (repo_id, repo_config) in repos.items(): - canon_repo_id = _canonicalize_id(repo_id) - repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) - if os.path.exists(repo_fn_pth): - log.info("Skipping repo %s, file %s already exists!", - repo_id, repo_fn_pth) - continue - elif canon_repo_id in repo_locations: - log.info("Skipping repo %s, file %s already pending!", - repo_id, repo_fn_pth) - continue - if not repo_config: - repo_config = {} - # Do some basic sanity checks/cleaning - n_repo_config = {} - for (k, v) in repo_config.items(): - k = k.lower().strip().replace("-", "_") - if k: - n_repo_config[k] = v - repo_config = n_repo_config - missing_required = 0 - for req_field in ['baseurl']: - if req_field not in repo_config: - log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), - repo_id, req_field) - missing_required += 1 - if not missing_required: - repo_configs[canon_repo_id] = repo_config - repo_locations[canon_repo_id] = repo_fn_pth - else: - log.warn("Repository %s is missing %s required fields, skipping!", - repo_id, missing_required) - for (c_repo_id, path) in repo_locations.items(): - repo_blob = _format_repository_config(c_repo_id, - repo_configs.get(c_repo_id)) - util.write_file(path, repo_blob) diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py deleted file mode 100644 index 412431f2..00000000 --- a/cloudinit/cs_utils.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 CloudSigma -# -# Author: Kiril Vladimiroff -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -cepko implements easy-to-use communication with CloudSigma's VMs through -a virtual serial port without bothering with formatting the messages -properly nor parsing the output with the specific and sometimes -confusing shell tools for that purpose. - -Having the server definition accessible by the VM can ve useful in various -ways. For example it is possible to easily determine from within the VM, -which network interfaces are connected to public and which to private network. -Another use is to pass some data to initial VM setup scripts, like setting the -hostname to the VM name or passing ssh public keys through server meta. - -For more information take a look at the Server Context section of CloudSigma -API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html -""" -import json -import platform - -from cloudinit import serial - - -# these high timeouts are necessary as read may read a lot of data. -READ_TIMEOUT = 60 -WRITE_TIMEOUT = 10 - -SERIAL_PORT = '/dev/ttyS1' -if platform.system() == 'Windows': - SERIAL_PORT = 'COM2' - - -class Cepko(object): - """ - One instance of that object could be use for one or more - queries to the serial port. - """ - request_pattern = "<\n{}\n>" - - def get(self, key="", request_pattern=None): - if request_pattern is None: - request_pattern = self.request_pattern - return CepkoResult(request_pattern.format(key)) - - def all(self): - return self.get() - - def meta(self, key=""): - request_pattern = self.request_pattern.format("/meta/{}") - return self.get(key, request_pattern) - - def global_context(self, key=""): - request_pattern = self.request_pattern.format("/global_context/{}") - return self.get(key, request_pattern) - - -class CepkoResult(object): - """ - CepkoResult executes the request to the virtual serial port as soon - as the instance is initialized and stores the result in both raw and - marshalled format. - """ - def __init__(self, request): - self.request = request - self.raw_result = self._execute() - self.result = self._marshal(self.raw_result) - - def _execute(self): - connection = serial.Serial(port=SERIAL_PORT, - timeout=READ_TIMEOUT, - writeTimeout=WRITE_TIMEOUT) - connection.write(self.request.encode('ascii')) - return connection.readline().strip(b'\x04\n').decode('ascii') - - def _marshal(self, raw_result): - try: - return json.loads(raw_result) - except ValueError: - return raw_result - - def __len__(self): - return self.result.__len__() - - def __getitem__(self, key): - return self.result.__getitem__(key) - - def __contains__(self, item): - return self.result.__contains__(item) - - def __iter__(self): - return self.result.__iter__() diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py deleted file mode 100644 index 40af8802..00000000 --- a/cloudinit/distros/__init__.py +++ /dev/null @@ -1,980 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import six -from six import StringIO - -import abc -import os -import re -import stat - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import net -from cloudinit.net import eni -from cloudinit.net import network_state -from cloudinit import ssh_util -from cloudinit import type_utils -from cloudinit import util - -from cloudinit.distros.parsers import hosts - - -OSFAMILIES = { - 'debian': ['debian', 'ubuntu'], - 'redhat': ['fedora', 'rhel'], - 'gentoo': ['gentoo'], - 'freebsd': ['freebsd'], - 'suse': ['sles'], - 'arch': ['arch'], -} - -LOG = logging.getLogger(__name__) - - -@six.add_metaclass(abc.ABCMeta) -class Distro(object): - - usr_lib_exec = "/usr/lib" - hosts_fn = "/etc/hosts" - ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" - hostname_conf_fn = "/etc/hostname" - tz_zone_dir = "/usr/share/zoneinfo" - init_cmd = ['service'] # systemctl, service etc - - def __init__(self, name, cfg, paths): - self._paths = paths - self._cfg = cfg - self.name = name - - @abc.abstractmethod - def install_packages(self, pkglist): - raise NotImplementedError() - - @abc.abstractmethod - def _write_network(self, settings): - # In the future use the http://fedorahosted.org/netcf/ - # to write this blob out in a distro format - raise NotImplementedError() - - def _write_network_config(self, settings): - raise NotImplementedError() - - def _find_tz_file(self, tz): - tz_file = os.path.join(self.tz_zone_dir, str(tz)) - if not os.path.isfile(tz_file): - raise IOError(("Invalid timezone %s," - " no file found at %s") % (tz, tz_file)) - return tz_file - - def get_option(self, opt_name, default=None): - return self._cfg.get(opt_name, default) - - def set_hostname(self, hostname, fqdn=None): - writeable_hostname = self._select_hostname(hostname, fqdn) - self._write_hostname(writeable_hostname, self.hostname_conf_fn) - self._apply_hostname(writeable_hostname) - - def uses_systemd(self): - try: - res = os.lstat('/run/systemd/system') - return stat.S_ISDIR(res.st_mode) - except Exception: - return False - - @abc.abstractmethod - def package_command(self, cmd, args=None, pkgs=None): - raise NotImplementedError() - - @abc.abstractmethod - def update_package_sources(self): - raise NotImplementedError() - - def get_primary_arch(self): - arch = os.uname[4] - if arch in ("i386", "i486", "i586", "i686"): - return "i386" - return arch - - def _get_arch_package_mirror_info(self, arch=None): - mirror_info = self.get_option("package_mirrors", []) - if not arch: - arch = self.get_primary_arch() - return _get_arch_package_mirror_info(mirror_info, arch) - - def get_package_mirror_info(self, arch=None, data_source=None): - # This resolves the package_mirrors config option - # down to a single dict of {mirror_name: mirror_url} - arch_info = self._get_arch_package_mirror_info(arch) - return _get_package_mirror_info(data_source=data_source, - mirror_info=arch_info) - - def apply_network(self, settings, bring_up=True): - # this applies network where 'settings' is interfaces(5) style - # it is obsolete compared to apply_network_config - # Write it out - dev_names = self._write_network(settings) - # Now try to bring them up - if bring_up: - return self._bring_up_interfaces(dev_names) - return False - - def _apply_network_from_network_config(self, netconfig, bring_up=True): - distro = self.__class__ - LOG.warn("apply_network_config is not currently implemented " - "for distribution '%s'. Attempting to use apply_network", - distro) - header = '\n'.join([ - "# Converted from network_config for distro %s" % distro, - "# Implmentation of _write_network_config is needed." - ]) - ns = network_state.parse_net_config_data(netconfig) - contents = eni.network_state_to_eni( - ns, header=header, render_hwaddress=True) - return self.apply_network(contents, bring_up=bring_up) - - def apply_network_config(self, netconfig, bring_up=False): - # apply network config netconfig - # This method is preferred to apply_network which only takes - # a much less complete network config format (interfaces(5)). - try: - dev_names = self._write_network_config(netconfig) - except NotImplementedError: - # backwards compat until all distros have apply_network_config - return self._apply_network_from_network_config( - netconfig, bring_up=bring_up) - - # Now try to bring them up - if bring_up: - return self._bring_up_interfaces(dev_names) - return False - - def apply_network_config_names(self, netconfig): - net.apply_network_config_names(netconfig) - - @abc.abstractmethod - def apply_locale(self, locale, out_fn=None): - raise NotImplementedError() - - @abc.abstractmethod - def set_timezone(self, tz): - raise NotImplementedError() - - def _get_localhost_ip(self): - return "127.0.0.1" - - @abc.abstractmethod - def _read_hostname(self, filename, default=None): - raise NotImplementedError() - - @abc.abstractmethod - def _write_hostname(self, hostname, filename): - raise NotImplementedError() - - @abc.abstractmethod - def _read_system_hostname(self): - raise NotImplementedError() - - def _apply_hostname(self, hostname): - # This really only sets the hostname - # temporarily (until reboot so it should - # not be depended on). Use the write - # hostname functions for 'permanent' adjustments. - LOG.debug("Non-persistently setting the system hostname to %s", - hostname) - try: - util.subp(['hostname', hostname]) - except util.ProcessExecutionError: - util.logexc(LOG, "Failed to non-persistently adjust the system " - "hostname to %s", hostname) - - def _select_hostname(self, hostname, fqdn): - # Prefer the short hostname over the long - # fully qualified domain name - if not hostname: - return fqdn - return hostname - - @staticmethod - def expand_osfamily(family_list): - distros = [] - for family in family_list: - if family not in OSFAMILIES: - raise ValueError("No distibutions found for osfamily %s" - % (family)) - distros.extend(OSFAMILIES[family]) - return distros - - def update_hostname(self, hostname, fqdn, prev_hostname_fn): - applying_hostname = hostname - - # Determine what the actual written hostname should be - hostname = self._select_hostname(hostname, fqdn) - - # If the previous hostname file exists lets see if we - # can get a hostname from it - if prev_hostname_fn and os.path.exists(prev_hostname_fn): - prev_hostname = self._read_hostname(prev_hostname_fn) - else: - prev_hostname = None - - # Lets get where we should write the system hostname - # and what the system hostname is - (sys_fn, sys_hostname) = self._read_system_hostname() - update_files = [] - - # If there is no previous hostname or it differs - # from what we want, lets update it or create the - # file in the first place - if not prev_hostname or prev_hostname != hostname: - update_files.append(prev_hostname_fn) - - # If the system hostname is different than the previous - # one or the desired one lets update it as well - if ((not sys_hostname) or (sys_hostname == prev_hostname and - sys_hostname != hostname)): - update_files.append(sys_fn) - - # If something else has changed the hostname after we set it - # initially, we should not overwrite those changes (we should - # only be setting the hostname once per instance) - if (sys_hostname and prev_hostname and - sys_hostname != prev_hostname): - LOG.info("%s differs from %s, assuming user maintained hostname.", - prev_hostname_fn, sys_fn) - return - - # Remove duplicates (incase the previous config filename) - # is the same as the system config filename, don't bother - # doing it twice - update_files = set([f for f in update_files if f]) - LOG.debug("Attempting to update hostname to %s in %s files", - hostname, len(update_files)) - - for fn in update_files: - try: - self._write_hostname(hostname, fn) - except IOError: - util.logexc(LOG, "Failed to write hostname %s to %s", hostname, - fn) - - # If the system hostname file name was provided set the - # non-fqdn as the transient hostname. - if sys_fn in update_files: - self._apply_hostname(applying_hostname) - - def update_etc_hosts(self, hostname, fqdn): - header = '' - if os.path.exists(self.hosts_fn): - eh = hosts.HostsConf(util.load_file(self.hosts_fn)) - else: - eh = hosts.HostsConf('') - header = util.make_header(base="added") - local_ip = self._get_localhost_ip() - prev_info = eh.get_entry(local_ip) - need_change = False - if not prev_info: - eh.add_entry(local_ip, fqdn, hostname) - need_change = True - else: - need_change = True - for entry in prev_info: - entry_fqdn = None - entry_aliases = [] - if len(entry) >= 1: - entry_fqdn = entry[0] - if len(entry) >= 2: - entry_aliases = entry[1:] - if entry_fqdn is not None and entry_fqdn == fqdn: - if hostname in entry_aliases: - # Exists already, leave it be - need_change = False - if need_change: - # Doesn't exist, add that entry in... - new_entries = list(prev_info) - new_entries.append([fqdn, hostname]) - eh.del_entries(local_ip) - for entry in new_entries: - if len(entry) == 1: - eh.add_entry(local_ip, entry[0]) - elif len(entry) >= 2: - eh.add_entry(local_ip, *entry) - if need_change: - contents = StringIO() - if header: - contents.write("%s\n" % (header)) - contents.write("%s\n" % (eh)) - util.write_file(self.hosts_fn, contents.getvalue(), mode=0o644) - - def _bring_up_interface(self, device_name): - cmd = ['ifup', device_name] - LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) - try: - (_out, err) = util.subp(cmd) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", cmd, err) - return True - except util.ProcessExecutionError: - util.logexc(LOG, "Running interface command %s failed", cmd) - return False - - def _bring_up_interfaces(self, device_names): - am_failed = 0 - for d in device_names: - if not self._bring_up_interface(d): - am_failed += 1 - if am_failed == 0: - return True - return False - - def get_default_user(self): - return self.get_option('default_user') - - def add_user(self, name, **kwargs): - """ - Add a user to the system using standard GNU tools - """ - if util.is_user(name): - LOG.info("User %s already exists, skipping." % name) - return - - if 'create_groups' in kwargs: - create_groups = kwargs.pop('create_groups') - else: - create_groups = True - - adduser_cmd = ['useradd', name] - log_adduser_cmd = ['useradd', name] - - # Since we are creating users, we want to carefully validate the - # inputs. If something goes wrong, we can end up with a system - # that nobody can login to. - adduser_opts = { - "gecos": '--comment', - "homedir": '--home', - "primary_group": '--gid', - "uid": '--uid', - "groups": '--groups', - "passwd": '--password', - "shell": '--shell', - "expiredate": '--expiredate', - "inactive": '--inactive', - "selinux_user": '--selinux-user', - } - - adduser_flags = { - "no_user_group": '--no-user-group', - "system": '--system', - "no_log_init": '--no-log-init', - } - - redact_opts = ['passwd'] - - # support kwargs having groups=[list] or groups="g1,g2" - groups = kwargs.get('groups') - if groups: - if isinstance(groups, (list, tuple)): - # kwargs.items loop below wants a comma delimeted string - # that can go right through to the command. - kwargs['groups'] = ",".join(groups) - else: - groups = groups.split(",") - - primary_group = kwargs.get('primary_group') - if primary_group: - groups.append(primary_group) - - if create_groups and groups: - for group in groups: - if not util.is_group(group): - self.create_group(group) - LOG.debug("created group %s for user %s", name, group) - - # Check the values and create the command - for key, val in kwargs.items(): - - if key in adduser_opts and val and isinstance(val, str): - adduser_cmd.extend([adduser_opts[key], val]) - - # Redact certain fields from the logs - if key in redact_opts: - log_adduser_cmd.extend([adduser_opts[key], 'REDACTED']) - else: - log_adduser_cmd.extend([adduser_opts[key], val]) - - elif key in adduser_flags and val: - adduser_cmd.append(adduser_flags[key]) - log_adduser_cmd.append(adduser_flags[key]) - - # Don't create the home directory if directed so or if the user is a - # system user - if 'no_create_home' in kwargs or 'system' in kwargs: - adduser_cmd.append('-M') - log_adduser_cmd.append('-M') - else: - adduser_cmd.append('-m') - log_adduser_cmd.append('-m') - - # Run the command - LOG.debug("Adding user %s", name) - try: - util.subp(adduser_cmd, logstring=log_adduser_cmd) - except Exception as e: - util.logexc(LOG, "Failed to create user %s", name) - raise e - - def create_user(self, name, **kwargs): - """ - Creates users for the system using the GNU passwd tools. This - will work on an GNU system. This should be overriden on - distros where useradd is not desirable or not available. - """ - - # Add the user - self.add_user(name, **kwargs) - - # Set password if plain-text password provided and non-empty - if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: - self.set_passwd(name, kwargs['plain_text_passwd']) - - # Set password if hashed password is provided and non-empty - if 'hashed_passwd' in kwargs and kwargs['hashed_passwd']: - self.set_passwd(name, kwargs['hashed_passwd'], hashed=True) - - # Default locking down the account. 'lock_passwd' defaults to True. - # lock account unless lock_password is False. - if kwargs.get('lock_passwd', True): - self.lock_passwd(name) - - # Configure sudo access - if 'sudo' in kwargs: - self.write_sudo_rules(name, kwargs['sudo']) - - # Import SSH keys - if 'ssh_authorized_keys' in kwargs: - # Try to handle this in a smart manner. - keys = kwargs['ssh_authorized_keys'] - if isinstance(keys, six.string_types): - keys = [keys] - elif isinstance(keys, dict): - keys = list(keys.values()) - if keys is not None: - if not isinstance(keys, (tuple, list, set)): - LOG.warn("Invalid type '%s' detected for" - " 'ssh_authorized_keys', expected list," - " string, dict, or set.", type(keys)) - else: - keys = set(keys) or [] - ssh_util.setup_user_keys(keys, name, options=None) - - return True - - def lock_passwd(self, name): - """ - Lock the password of a user, i.e., disable password logins - """ - try: - # Need to use the short option name '-l' instead of '--lock' - # (which would be more descriptive) since SLES 11 doesn't know - # about long names. - util.subp(['passwd', '-l', name]) - except Exception as e: - util.logexc(LOG, 'Failed to disable password for user %s', name) - raise e - - def set_passwd(self, user, passwd, hashed=False): - pass_string = '%s:%s' % (user, passwd) - cmd = ['chpasswd'] - - if hashed: - # Need to use the short option name '-e' instead of '--encrypted' - # (which would be more descriptive) since SLES 11 doesn't know - # about long names. - cmd.append('-e') - - try: - util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) - except Exception as e: - util.logexc(LOG, "Failed to set password for %s", user) - raise e - - return True - - def ensure_sudo_dir(self, path, sudo_base='/etc/sudoers'): - # Ensure the dir is included and that - # it actually exists as a directory - sudoers_contents = '' - base_exists = False - if os.path.exists(sudo_base): - sudoers_contents = util.load_file(sudo_base) - base_exists = True - found_include = False - for line in sudoers_contents.splitlines(): - line = line.strip() - include_match = re.search(r"^#includedir\s+(.*)$", line) - if not include_match: - continue - included_dir = include_match.group(1).strip() - if not included_dir: - continue - included_dir = os.path.abspath(included_dir) - if included_dir == path: - found_include = True - break - if not found_include: - try: - if not base_exists: - lines = [('# See sudoers(5) for more information' - ' on "#include" directives:'), '', - util.make_header(base="added"), - "#includedir %s" % (path), ''] - sudoers_contents = "\n".join(lines) - util.write_file(sudo_base, sudoers_contents, 0o440) - else: - lines = ['', util.make_header(base="added"), - "#includedir %s" % (path), ''] - sudoers_contents = "\n".join(lines) - util.append_file(sudo_base, sudoers_contents) - LOG.debug("Added '#includedir %s' to %s" % (path, sudo_base)) - except IOError as e: - util.logexc(LOG, "Failed to write %s", sudo_base) - raise e - util.ensure_dir(path, 0o750) - - def write_sudo_rules(self, user, rules, sudo_file=None): - if not sudo_file: - sudo_file = self.ci_sudoers_fn - - lines = [ - '', - "# User rules for %s" % user, - ] - if isinstance(rules, (list, tuple)): - for rule in rules: - lines.append("%s %s" % (user, rule)) - elif isinstance(rules, six.string_types): - lines.append("%s %s" % (user, rules)) - else: - msg = "Can not create sudoers rule addition with type %r" - raise TypeError(msg % (type_utils.obj_name(rules))) - content = "\n".join(lines) - content += "\n" # trailing newline - - self.ensure_sudo_dir(os.path.dirname(sudo_file)) - if not os.path.exists(sudo_file): - contents = [ - util.make_header(), - content, - ] - try: - util.write_file(sudo_file, "\n".join(contents), 0o440) - except IOError as e: - util.logexc(LOG, "Failed to write sudoers file %s", sudo_file) - raise e - else: - try: - util.append_file(sudo_file, content) - except IOError as e: - util.logexc(LOG, "Failed to append sudoers file %s", sudo_file) - raise e - - def create_group(self, name, members=None): - group_add_cmd = ['groupadd', name] - if not members: - members = [] - - # Check if group exists, and then add it doesn't - if util.is_group(name): - LOG.warn("Skipping creation of existing group '%s'" % name) - else: - try: - util.subp(group_add_cmd) - LOG.info("Created new group %s" % name) - except Exception: - util.logexc(LOG, "Failed to create group %s", name) - - # Add members to the group, if so defined - if len(members) > 0: - for member in members: - if not util.is_user(member): - LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) - continue - - util.subp(['usermod', '-a', '-G', name, member]) - LOG.info("Added user '%s' to group '%s'" % (member, name)) - - -def _get_package_mirror_info(mirror_info, data_source=None, - mirror_filter=util.search_for_mirror): - # given a arch specific 'mirror_info' entry (from package_mirrors) - # search through the 'search' entries, and fallback appropriately - # return a dict with only {name: mirror} entries. - if not mirror_info: - mirror_info = {} - - # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) - # the region is us-east-1. so region = az[0:-1] - directions_re = '|'.join([ - 'central', 'east', 'north', 'northeast', 'northwest', - 'south', 'southeast', 'southwest', 'west']) - ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re) - - subst = {} - if data_source and data_source.availability_zone: - subst['availability_zone'] = data_source.availability_zone - - if re.match(ec2_az_re, data_source.availability_zone): - subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] - - if data_source and data_source.region: - subst['region'] = data_source.region - - results = {} - for (name, mirror) in mirror_info.get('failsafe', {}).items(): - results[name] = mirror - - for (name, searchlist) in mirror_info.get('search', {}).items(): - mirrors = [] - for tmpl in searchlist: - try: - mirrors.append(tmpl % subst) - except KeyError: - pass - - found = mirror_filter(mirrors) - if found: - results[name] = found - - LOG.debug("filtered distro mirror info: %s" % results) - - return results - - -def _get_arch_package_mirror_info(package_mirrors, arch): - # pull out the specific arch from a 'package_mirrors' config option - default = None - for item in package_mirrors: - arches = item.get("arches") - if arch in arches: - return item - if "default" in arches: - default = item - return default - - -# Normalizes a input group configuration -# which can be a comma seperated list of -# group names, or a list of group names -# or a python dictionary of group names -# to a list of members of that group. -# -# The output is a dictionary of group -# names => members of that group which -# is the standard form used in the rest -# of cloud-init -def _normalize_groups(grp_cfg): - if isinstance(grp_cfg, six.string_types): - grp_cfg = grp_cfg.strip().split(",") - if isinstance(grp_cfg, list): - c_grp_cfg = {} - for i in grp_cfg: - if isinstance(i, dict): - for k, v in i.items(): - if k not in c_grp_cfg: - if isinstance(v, list): - c_grp_cfg[k] = list(v) - elif isinstance(v, six.string_types): - c_grp_cfg[k] = [v] - else: - raise TypeError("Bad group member type %s" % - type_utils.obj_name(v)) - else: - if isinstance(v, list): - c_grp_cfg[k].extend(v) - elif isinstance(v, six.string_types): - c_grp_cfg[k].append(v) - else: - raise TypeError("Bad group member type %s" % - type_utils.obj_name(v)) - elif isinstance(i, six.string_types): - if i not in c_grp_cfg: - c_grp_cfg[i] = [] - else: - raise TypeError("Unknown group name type %s" % - type_utils.obj_name(i)) - grp_cfg = c_grp_cfg - groups = {} - if isinstance(grp_cfg, dict): - for (grp_name, grp_members) in grp_cfg.items(): - groups[grp_name] = util.uniq_merge_sorted(grp_members) - else: - raise TypeError(("Group config must be list, dict " - " or string types only and not %s") % - type_utils.obj_name(grp_cfg)) - return groups - - -# Normalizes a input group configuration -# which can be a comma seperated list of -# user names, or a list of string user names -# or a list of dictionaries with components -# that define the user config + 'name' (if -# a 'name' field does not exist then the -# default user is assumed to 'own' that -# configuration. -# -# The output is a dictionary of user -# names => user config which is the standard -# form used in the rest of cloud-init. Note -# the default user will have a special config -# entry 'default' which will be marked as true -# all other users will be marked as false. -def _normalize_users(u_cfg, def_user_cfg=None): - if isinstance(u_cfg, dict): - ad_ucfg = [] - for (k, v) in u_cfg.items(): - if isinstance(v, (bool, int, float) + six.string_types): - if util.is_true(v): - ad_ucfg.append(str(k)) - elif isinstance(v, dict): - v['name'] = k - ad_ucfg.append(v) - else: - raise TypeError(("Unmappable user value type %s" - " for key %s") % (type_utils.obj_name(v), k)) - u_cfg = ad_ucfg - elif isinstance(u_cfg, six.string_types): - u_cfg = util.uniq_merge_sorted(u_cfg) - - users = {} - for user_config in u_cfg: - if isinstance(user_config, (list,) + six.string_types): - for u in util.uniq_merge(user_config): - if u and u not in users: - users[u] = {} - elif isinstance(user_config, dict): - if 'name' in user_config: - n = user_config.pop('name') - prev_config = users.get(n) or {} - users[n] = util.mergemanydict([prev_config, - user_config]) - else: - # Assume the default user then - prev_config = users.get('default') or {} - users['default'] = util.mergemanydict([prev_config, - user_config]) - else: - raise TypeError(("User config must be dictionary/list " - " or string types only and not %s") % - type_utils.obj_name(user_config)) - - # Ensure user options are in the right python friendly format - if users: - c_users = {} - for (uname, uconfig) in users.items(): - c_uconfig = {} - for (k, v) in uconfig.items(): - k = k.replace('-', '_').strip() - if k: - c_uconfig[k] = v - c_users[uname] = c_uconfig - users = c_users - - # Fixup the default user into the real - # default user name and replace it... - def_user = None - if users and 'default' in users: - def_config = users.pop('default') - if def_user_cfg: - # Pickup what the default 'real name' is - # and any groups that are provided by the - # default config - def_user_cfg = def_user_cfg.copy() - def_user = def_user_cfg.pop('name') - def_groups = def_user_cfg.pop('groups', []) - # Pickup any config + groups for that user name - # that we may have previously extracted - parsed_config = users.pop(def_user, {}) - parsed_groups = parsed_config.get('groups', []) - # Now merge our extracted groups with - # anything the default config provided - users_groups = util.uniq_merge_sorted(parsed_groups, def_groups) - parsed_config['groups'] = ",".join(users_groups) - # The real config for the default user is the - # combination of the default user config provided - # by the distro, the default user config provided - # by the above merging for the user 'default' and - # then the parsed config from the user's 'real name' - # which does not have to be 'default' (but could be) - users[def_user] = util.mergemanydict([def_user_cfg, - def_config, - parsed_config]) - - # Ensure that only the default user that we - # found (if any) is actually marked as being - # the default user - if users: - for (uname, uconfig) in users.items(): - if def_user and uname == def_user: - uconfig['default'] = True - else: - uconfig['default'] = False - - return users - - -# Normalizes a set of user/users and group -# dictionary configuration into a useable -# format that the rest of cloud-init can -# understand using the default user -# provided by the input distrobution (if any) -# to allow for mapping of the 'default' user. -# -# Output is a dictionary of group names -> [member] (list) -# and a dictionary of user names -> user configuration (dict) -# -# If 'user' exists it will override -# the 'users'[0] entry (if a list) otherwise it will -# just become an entry in the returned dictionary (no override) -def normalize_users_groups(cfg, distro): - if not cfg: - cfg = {} - - users = {} - groups = {} - if 'groups' in cfg: - groups = _normalize_groups(cfg['groups']) - - # Handle the previous style of doing this where the first user - # overrides the concept of the default user if provided in the user: XYZ - # format. - old_user = {} - if 'user' in cfg and cfg['user']: - old_user = cfg['user'] - # Translate it into the format that is more useful - # going forward - if isinstance(old_user, six.string_types): - old_user = { - 'name': old_user, - } - if not isinstance(old_user, dict): - LOG.warn(("Format for 'user' key must be a string or " - "dictionary and not %s"), type_utils.obj_name(old_user)) - old_user = {} - - # If no old user format, then assume the distro - # provides what the 'default' user maps to, but notice - # that if this is provided, we won't automatically inject - # a 'default' user into the users list, while if a old user - # format is provided we will. - distro_user_config = {} - try: - distro_user_config = distro.get_default_user() - except NotImplementedError: - LOG.warn(("Distro has not implemented default user " - "access. No distribution provided default user" - " will be normalized.")) - - # Merge the old user (which may just be an empty dict when not - # present with the distro provided default user configuration so - # that the old user style picks up all the distribution specific - # attributes (if any) - default_user_config = util.mergemanydict([old_user, distro_user_config]) - - base_users = cfg.get('users', []) - if not isinstance(base_users, (list, dict) + six.string_types): - LOG.warn(("Format for 'users' key must be a comma separated string" - " or a dictionary or a list and not %s"), - type_utils.obj_name(base_users)) - base_users = [] - - if old_user: - # Ensure that when user: is provided that this user - # always gets added (as the default user) - if isinstance(base_users, list): - # Just add it on at the end... - base_users.append({'name': 'default'}) - elif isinstance(base_users, dict): - base_users['default'] = dict(base_users).get('default', True) - elif isinstance(base_users, six.string_types): - # Just append it on to be re-parsed later - base_users += ",default" - - users = _normalize_users(base_users, default_user_config) - return (users, groups) - - -# Given a user dictionary config it will -# extract the default user name and user config -# from that list and return that tuple or -# return (None, None) if no default user is -# found in the given input -def extract_default(users, default_name=None, default_config=None): - if not users: - users = {} - - def safe_find(entry): - config = entry[1] - if not config or 'default' not in config: - return False - else: - return config['default'] - - tmp_users = users.items() - tmp_users = dict(filter(safe_find, tmp_users)) - if not tmp_users: - return (default_name, default_config) - else: - name = list(tmp_users)[0] - config = tmp_users[name] - config.pop('default', None) - return (name, config) - - -def fetch(name): - locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) - if not locs: - raise ImportError("No distribution found for distro %s (searched %s)" - % (name, looked_locs)) - mod = importer.import_module(locs[0]) - cls = getattr(mod, 'Distro') - return cls - - -def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", - tz_local="/etc/localtime"): - util.write_file(tz_conf, str(tz).rstrip() + "\n") - # This ensures that the correct tz will be used for the system - if tz_local and tz_file: - # use a symlink if there exists a symlink or tz_local is not present - islink = os.path.islink(tz_local) - if islink or not os.path.exists(tz_local): - if islink: - util.del_file(tz_local) - os.symlink(tz_file, tz_local) - else: - util.copy(tz_file, tz_local) - return diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py deleted file mode 100644 index 66209f22..00000000 --- a/cloudinit/distros/arch.py +++ /dev/null @@ -1,201 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 Rackspace, US Inc. -# -# Author: Nate House -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import distros -from cloudinit import helpers -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit.settings import PER_INSTANCE - -LOG = logging.getLogger(__name__) - - -class Distro(distros.Distro): - locale_conf_fn = "/etc/locale.gen" - network_conf_dir = "/etc/netctl" - resolve_conf_fn = "/etc/resolv.conf" - init_cmd = ['systemctl'] # init scripts - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'arch' - cfg['ssh_svcname'] = 'sshd' - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) - # "" provides trailing newline during join - lines = [ - util.make_header(), - 'LANG="%s"' % (locale), - "", - ] - util.write_file(out_fn, "\n".join(lines)) - - def install_packages(self, pkglist): - self.update_package_sources() - self.package_command('', pkgs=pkglist) - - def _write_network(self, settings): - entries = net_util.translate_network(settings) - LOG.debug("Translated ubuntu style network settings %s into %s", - settings, entries) - dev_names = entries.keys() - # Format for netctl - for (dev, info) in entries.items(): - nameservers = [] - net_fn = self.network_conf_dir + dev - net_cfg = { - 'Connection': 'ethernet', - 'Interface': dev, - 'IP': info.get('bootproto'), - 'Address': "('%s/%s')" % (info.get('address'), - info.get('netmask')), - 'Gateway': info.get('gateway'), - 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') - } - util.write_file(net_fn, convert_netctl(net_cfg)) - if info.get('auto'): - self._enable_interface(dev) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - - if nameservers: - util.write_file(self.resolve_conf_fn, - convert_resolv_conf(nameservers)) - - return dev_names - - def _enable_interface(self, device_name): - cmd = ['netctl', 'reenable', device_name] - try: - (_out, err) = util.subp(cmd) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: - util.logexc(LOG, "Running interface command %s failed", cmd) - - def _bring_up_interface(self, device_name): - cmd = ['netctl', 'restart', device_name] - LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) - try: - (_out, err) = util.subp(cmd) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", cmd, err) - return True - except util.ProcessExecutionError: - util.logexc(LOG, "Running interface command %s failed", cmd) - return False - - def _bring_up_interfaces(self, device_names): - for d in device_names: - if not self._bring_up_interface(d): - return False - return True - - def _write_hostname(self, your_hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(your_hostname) - util.write_file(out_fn, conf, 0o644) - - def _read_system_hostname(self): - sys_hostname = self._read_hostname(self.hostname_conf_fn) - return (self.hostname_conf_fn, sys_hostname) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def set_timezone(self, tz): - distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['pacman'] - # Redirect output - cmd.append("-Sy") - cmd.append("--quiet") - cmd.append("--noconfirm") - - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - if command: - cmd.append(command) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ["-y"], freq=PER_INSTANCE) - - -def convert_netctl(settings): - """Returns a settings string formatted for netctl.""" - result = '' - if isinstance(settings, dict): - for k, v in settings.items(): - result = result + '%s=%s\n' % (k, v) - return result - - -def convert_resolv_conf(settings): - """Returns a settings string formatted for resolv.conf.""" - result = '' - if isinstance(settings, list): - for ns in settings: - result = result + 'nameserver %s\n' % ns - return result diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py deleted file mode 100644 index f9b3b92e..00000000 --- a/cloudinit/distros/debian.py +++ /dev/null @@ -1,236 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import distros -from cloudinit import helpers -from cloudinit import log as logging -from cloudinit.net import eni -from cloudinit.net.network_state import parse_net_config_data -from cloudinit import util - -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit.settings import PER_INSTANCE - -LOG = logging.getLogger(__name__) - -APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', - '--option=Dpkg::options::=--force-unsafe-io', - '--assume-yes', '--quiet') -APT_GET_WRAPPER = { - 'command': 'eatmydata', - 'enabled': 'auto', -} - -ENI_HEADER = """# This file is generated from information provided by -# the datasource. Changes to it will not persist across an instance. -# To disable cloud-init's network configuration capabilities, write a file -# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: -# network: {config: disabled} -""" - - -class Distro(distros.Distro): - hostname_conf_fn = "/etc/hostname" - locale_conf_fn = "/etc/default/locale" - network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg" - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'debian' - self._net_renderer = eni.Renderer({ - 'eni_path': self.network_conf_fn, - 'eni_header': ENI_HEADER, - 'links_path_prefix': None, - 'netrules_path': None, - }) - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', locale], capture=False) - util.subp(['update-locale', locale], capture=False) - # "" provides trailing newline during join - lines = [ - util.make_header(), - 'LANG="%s"' % (locale), - "", - ] - util.write_file(out_fn, "\n".join(lines)) - - def install_packages(self, pkglist): - self.update_package_sources() - self.package_command('install', pkgs=pkglist) - - def _write_network(self, settings): - util.write_file(self.network_conf_fn, settings) - return ['all'] - - def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) - _maybe_remove_legacy_eth0() - return [] - - def _bring_up_interfaces(self, device_names): - use_all = False - for d in device_names: - if d == 'all': - use_all = True - if use_all: - return distros.Distro._bring_up_interface(self, '--all') - else: - return distros.Distro._bring_up_interfaces(self, device_names) - - def _write_hostname(self, your_hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0o644) - - def _read_system_hostname(self): - sys_hostname = self._read_hostname(self.hostname_conf_fn) - return (self.hostname_conf_fn, sys_hostname) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def _get_localhost_ip(self): - # Note: http://www.leonardoborda.com/blog/127-0-1-1-ubuntu-debian/ - return "127.0.1.1" - - def set_timezone(self, tz): - distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - e = os.environ.copy() - # See: http://tiny.cc/kg91fw - # Or: http://tiny.cc/mh91fw - e['DEBIAN_FRONTEND'] = 'noninteractive' - - wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER) - cmd = _get_wrapper_prefix( - wcfg.get('command', APT_GET_WRAPPER['command']), - wcfg.get('enabled', APT_GET_WRAPPER['enabled'])) - - cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND))) - - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - subcmd = command - if command == "upgrade": - subcmd = self.get_option("apt_get_upgrade_subcommand", - "dist-upgrade") - - cmd.append(subcmd) - - pkglist = util.expand_package_list('%s=%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.log_time(logfunc=LOG.debug, - msg="apt-%s [%s]" % (command, ' '.join(cmd)), - func=util.subp, - args=(cmd,), kwargs={'env': e, 'capture': False}) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ["update"], freq=PER_INSTANCE) - - def get_primary_arch(self): - (arch, _err) = util.subp(['dpkg', '--print-architecture']) - return str(arch).strip() - - -def _get_wrapper_prefix(cmd, mode): - if isinstance(cmd, str): - cmd = [str(cmd)] - - if (util.is_true(mode) or - (str(mode).lower() == "auto" and cmd[0] and - util.which(cmd[0]))): - return cmd - else: - return [] - - -def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): - """Ubuntu cloud images previously included a 'eth0.cfg' that had - hard coded content. That file would interfere with the rendered - configuration if it was present. - - if the file does not exist do nothing. - If the file exists: - - with known content, remove it and warn - - with unknown content, leave it and warn - """ - - if not os.path.exists(path): - return - - bmsg = "Dynamic networking config may not apply." - try: - contents = util.load_file(path) - known_contents = ["auto eth0", "iface eth0 inet dhcp"] - lines = [f.strip() for f in contents.splitlines() - if not f.startswith("#")] - if lines == known_contents: - util.del_file(path) - msg = "removed %s with known contents" % path - else: - msg = (bmsg + " '%s' exists with user configured content." % path) - except Exception: - msg = bmsg + " %s exists, but could not be read." % path - - LOG.warn(msg) diff --git a/cloudinit/distros/fedora.py b/cloudinit/distros/fedora.py deleted file mode 100644 index c777845d..00000000 --- a/cloudinit/distros/fedora.py +++ /dev/null @@ -1,31 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.distros import rhel - -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - - -class Distro(rhel.Distro): - pass diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py deleted file mode 100644 index 91bf4a4e..00000000 --- a/cloudinit/distros/freebsd.py +++ /dev/null @@ -1,417 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 Harm Weites -# -# Author: Harm Weites -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import six -from six import StringIO - -import re - -from cloudinit import distros -from cloudinit import helpers -from cloudinit import log as logging -from cloudinit import ssh_util -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros.parsers.resolv_conf import ResolvConf - -from cloudinit.settings import PER_INSTANCE - -LOG = logging.getLogger(__name__) - - -class Distro(distros.Distro): - rc_conf_fn = "/etc/rc.conf" - login_conf_fn = '/etc/login.conf' - login_conf_fn_bak = '/etc/login.conf.orig' - resolv_conf_fn = '/etc/resolv.conf' - ci_sudoers_fn = '/usr/local/etc/sudoers.d/90-cloud-init-users' - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'freebsd' - - # Updates a key in /etc/rc.conf. - def updatercconf(self, key, value): - LOG.debug("Checking %s for: %s = %s", self.rc_conf_fn, key, value) - conf = self.loadrcconf() - config_changed = False - if key not in conf: - LOG.debug("Adding key in %s: %s = %s", self.rc_conf_fn, key, - value) - conf[key] = value - config_changed = True - else: - for item in conf.keys(): - if item == key and conf[item] != value: - conf[item] = value - LOG.debug("Changing key in %s: %s = %s", self.rc_conf_fn, - key, value) - config_changed = True - - if config_changed: - LOG.info("Writing %s", self.rc_conf_fn) - buf = StringIO() - for keyval in conf.items(): - buf.write('%s="%s"\n' % keyval) - util.write_file(self.rc_conf_fn, buf.getvalue()) - - # Load the contents of /etc/rc.conf and store all keys in a dict. Make sure - # quotes are ignored: - # hostname="bla" - def loadrcconf(self): - RE_MATCH = re.compile(r'^(\w+)\s*=\s*(.*)\s*') - conf = {} - lines = util.load_file(self.rc_conf_fn).splitlines() - for line in lines: - m = RE_MATCH.match(line) - if not m: - LOG.debug("Skipping line from /etc/rc.conf: %s", line) - continue - key = m.group(1).rstrip() - val = m.group(2).rstrip() - # Kill them quotes (not completely correct, aka won't handle - # quoted values, but should be ok ...) - if val[0] in ('"', "'"): - val = val[1:] - if val[-1] in ('"', "'"): - val = val[0:-1] - if len(val) == 0: - LOG.debug("Skipping empty value from /etc/rc.conf: %s", line) - continue - conf[key] = val - return conf - - def readrcconf(self, key): - conf = self.loadrcconf() - try: - val = conf[key] - except KeyError: - val = None - return val - - # NOVA will inject something like eth0, rewrite that to use the FreeBSD - # adapter. Since this adapter is based on the used driver, we need to - # figure out which interfaces are available. On KVM platforms this is - # vtnet0, where Xen would use xn0. - def getnetifname(self, dev): - LOG.debug("Translating network interface %s", dev) - if dev.startswith('lo'): - return dev - - n = re.search('\d+$', dev) - index = n.group(0) - - (out, err) = util.subp(['ifconfig', '-a']) - ifconfigoutput = [x for x in (out.strip()).splitlines() - if len(x.split()) > 0] - for line in ifconfigoutput: - m = re.match('^\w+', line) - if m: - if m.group(0).startswith('lo'): - continue - # Just settle with the first non-lo adapter we find, since it's - # rather unlikely there will be multiple nicdrivers involved. - bsddev = m.group(0) - break - - # Replace the index with the one we're after. - bsddev = re.sub('\d+$', index, bsddev) - LOG.debug("Using network interface %s", bsddev) - return bsddev - - def _read_system_hostname(self): - sys_hostname = self._read_hostname(filename=None) - return ('rc.conf', sys_hostname) - - def _read_hostname(self, filename, default=None): - hostname = None - try: - hostname = self.readrcconf('hostname') - except IOError: - pass - if not hostname: - return default - return hostname - - def _write_hostname(self, hostname, filename): - self.updatercconf('hostname', hostname) - - def create_group(self, name, members): - group_add_cmd = ['pw', '-n', name] - if util.is_group(name): - LOG.warn("Skipping creation of existing group '%s'", name) - else: - try: - util.subp(group_add_cmd) - LOG.info("Created new group %s", name) - except Exception as e: - util.logexc(LOG, "Failed to create group %s", name) - raise e - - if len(members) > 0: - for member in members: - if not util.is_user(member): - LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) - continue - try: - util.subp(['pw', 'usermod', '-n', name, '-G', member]) - LOG.info("Added user '%s' to group '%s'", member, name) - except Exception: - util.logexc(LOG, "Failed to add user '%s' to group '%s'", - member, name) - - def add_user(self, name, **kwargs): - if util.is_user(name): - LOG.info("User %s already exists, skipping.", name) - return False - - adduser_cmd = ['pw', 'useradd', '-n', name] - log_adduser_cmd = ['pw', 'useradd', '-n', name] - - adduser_opts = { - "homedir": '-d', - "gecos": '-c', - "primary_group": '-g', - "groups": '-G', - "passwd": '-h', - "shell": '-s', - "inactive": '-E', - } - adduser_flags = { - "no_user_group": '--no-user-group', - "system": '--system', - "no_log_init": '--no-log-init', - } - - redact_opts = ['passwd'] - - for key, val in kwargs.items(): - if (key in adduser_opts and val and - isinstance(val, six.string_types)): - adduser_cmd.extend([adduser_opts[key], val]) - - # Redact certain fields from the logs - if key in redact_opts: - log_adduser_cmd.extend([adduser_opts[key], 'REDACTED']) - else: - log_adduser_cmd.extend([adduser_opts[key], val]) - - elif key in adduser_flags and val: - adduser_cmd.append(adduser_flags[key]) - log_adduser_cmd.append(adduser_flags[key]) - - if 'no_create_home' in kwargs or 'system' in kwargs: - adduser_cmd.append('-d/nonexistent') - log_adduser_cmd.append('-d/nonexistent') - else: - adduser_cmd.append('-d/usr/home/%s' % name) - adduser_cmd.append('-m') - log_adduser_cmd.append('-d/usr/home/%s' % name) - log_adduser_cmd.append('-m') - - # Run the command - LOG.info("Adding user %s", name) - try: - util.subp(adduser_cmd, logstring=log_adduser_cmd) - except Exception as e: - util.logexc(LOG, "Failed to create user %s", name) - raise e - - def set_passwd(self, user, passwd, hashed=False): - cmd = ['pw', 'usermod', user] - - if hashed: - cmd.append('-H') - else: - cmd.append('-h') - - cmd.append('0') - - try: - util.subp(cmd, passwd, logstring="chpasswd for %s" % user) - except Exception as e: - util.logexc(LOG, "Failed to set password for %s", user) - raise e - - def lock_passwd(self, name): - try: - util.subp(['pw', 'usermod', name, '-h', '-']) - except Exception as e: - util.logexc(LOG, "Failed to lock user %s", name) - raise e - - def create_user(self, name, **kwargs): - self.add_user(name, **kwargs) - - # Set password if plain-text password provided and non-empty - if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: - self.set_passwd(name, kwargs['plain_text_passwd']) - - # Default locking down the account. 'lock_passwd' defaults to True. - # lock account unless lock_password is False. - if kwargs.get('lock_passwd', True): - self.lock_passwd(name) - - # Configure sudo access - if 'sudo' in kwargs: - self.write_sudo_rules(name, kwargs['sudo']) - - # Import SSH keys - if 'ssh_authorized_keys' in kwargs: - keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, options=None) - - def _write_network(self, settings): - entries = net_util.translate_network(settings) - nameservers = [] - searchdomains = [] - dev_names = entries.keys() - for (device, info) in entries.items(): - # Skip the loopback interface. - if device.startswith('lo'): - continue - - dev = self.getnetifname(device) - - LOG.info('Configuring interface %s', dev) - - if info.get('bootproto') == 'static': - LOG.debug('Configuring dev %s with %s / %s', dev, - info.get('address'), info.get('netmask')) - # Configure an ipv4 address. - ifconfig = (info.get('address') + ' netmask ' + - info.get('netmask')) - - # Configure the gateway. - self.updatercconf('defaultrouter', info.get('gateway')) - - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - if 'dns-search' in info: - searchdomains.extend(info['dns-search']) - else: - ifconfig = 'DHCP' - - self.updatercconf('ifconfig_' + dev, ifconfig) - - # Try to read the /etc/resolv.conf or just start from scratch if that - # fails. - try: - resolvconf = ResolvConf(util.load_file(self.resolv_conf_fn)) - resolvconf.parse() - except IOError: - util.logexc(LOG, "Failed to parse %s, use new empty file", - self.resolv_conf_fn) - resolvconf = ResolvConf('') - resolvconf.parse() - - # Add some nameservers - for server in nameservers: - try: - resolvconf.add_nameserver(server) - except ValueError: - util.logexc(LOG, "Failed to add nameserver %s", server) - - # And add any searchdomains. - for domain in searchdomains: - try: - resolvconf.add_search_domain(domain) - except ValueError: - util.logexc(LOG, "Failed to add search domain %s", domain) - util.write_file(self.resolv_conf_fn, str(resolvconf), 0o644) - - return dev_names - - def apply_locale(self, locale, out_fn=None): - # Adjust the locals value to the new value - newconf = StringIO() - for line in util.load_file(self.login_conf_fn).splitlines(): - newconf.write(re.sub(r'^default:', - r'default:lang=%s:' % locale, line)) - newconf.write("\n") - - # Make a backup of login.conf. - util.copy(self.login_conf_fn, self.login_conf_fn_bak) - - # And write the new login.conf. - util.write_file(self.login_conf_fn, newconf.getvalue()) - - try: - LOG.debug("Running cap_mkdb for %s", locale) - util.subp(['cap_mkdb', self.login_conf_fn]) - except util.ProcessExecutionError: - # cap_mkdb failed, so restore the backup. - util.logexc(LOG, "Failed to apply locale %s", locale) - try: - util.copy(self.login_conf_fn_bak, self.login_conf_fn) - except IOError: - util.logexc(LOG, "Failed to restore %s backup", - self.login_conf_fn) - - def _bring_up_interface(self, device_name): - if device_name.startswith('lo'): - return - dev = self.getnetifname(device_name) - cmd = ['/etc/rc.d/netif', 'start', dev] - LOG.debug("Attempting to bring up interface %s using command %s", - dev, cmd) - # This could return 1 when the interface has already been put UP by the - # OS. This is just fine. - (_out, err) = util.subp(cmd, rcs=[0, 1]) - if len(err): - LOG.warn("Error running %s: %s", cmd, err) - - def install_packages(self, pkglist): - self.update_package_sources() - self.package_command('install', pkgs=pkglist) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - e = os.environ.copy() - e['ASSUME_ALWAYS_YES'] = 'YES' - - cmd = ['pkg'] - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - if command: - cmd.append(command) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=e, capture=False) - - def set_timezone(self, tz): - distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ["update"], freq=PER_INSTANCE) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py deleted file mode 100644 index 6267dd6e..00000000 --- a/cloudinit/distros/gentoo.py +++ /dev/null @@ -1,160 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 Rackspace, US Inc. -# -# Author: Nate House -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import distros -from cloudinit import helpers -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit.settings import PER_INSTANCE - -LOG = logging.getLogger(__name__) - - -class Distro(distros.Distro): - locale_conf_fn = "/etc/locale.gen" - network_conf_fn = "/etc/conf.d/net" - init_cmd = [''] # init scripts - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'gentoo' - # Fix sshd restarts - cfg['ssh_svcname'] = '/etc/init.d/sshd' - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) - # "" provides trailing newline during join - lines = [ - util.make_header(), - 'LANG="%s"' % (locale), - "", - ] - util.write_file(out_fn, "\n".join(lines)) - - def install_packages(self, pkglist): - self.update_package_sources() - self.package_command('', pkgs=pkglist) - - def _write_network(self, settings): - util.write_file(self.network_conf_fn, settings) - return ['all'] - - def _bring_up_interface(self, device_name): - cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] - LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) - try: - (_out, err) = util.subp(cmd) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", cmd, err) - return True - except util.ProcessExecutionError: - util.logexc(LOG, "Running interface command %s failed", cmd) - return False - - def _bring_up_interfaces(self, device_names): - use_all = False - for d in device_names: - if d == 'all': - use_all = True - if use_all: - # Grab device names from init scripts - cmd = ['ls', '/etc/init.d/net.*'] - try: - (_out, err) = util.subp(cmd) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", cmd, - err) - except util.ProcessExecutionError: - util.logexc(LOG, "Running interface command %s failed", cmd) - return False - devices = [x.split('.')[2] for x in _out.split(' ')] - return distros.Distro._bring_up_interfaces(self, devices) - else: - return distros.Distro._bring_up_interfaces(self, device_names) - - def _write_hostname(self, your_hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(your_hostname) - util.write_file(out_fn, conf, 0o644) - - def _read_system_hostname(self): - sys_hostname = self._read_hostname(self.hostname_conf_fn) - return (self.hostname_conf_fn, sys_hostname) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def set_timezone(self, tz): - distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['emerge'] - # Redirect output - cmd.append("--quiet") - - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - if command: - cmd.append(command) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ["-u", "world"], freq=PER_INSTANCE) diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py deleted file mode 100644 index cadfa6b6..00000000 --- a/cloudinit/distros/net_util.py +++ /dev/null @@ -1,182 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -# This is a util function to translate debian based distro interface blobs as -# given in /etc/network/interfaces to an *somewhat* agnostic format for -# distributions that use other formats. -# -# TODO(harlowja) remove when we have python-netcf active... -# -# The format is the following: -# { -# : { -# # All optional (if not existent in original format) -# "netmask": , -# "broadcast": , -# "gateway": , -# "address": , -# "bootproto": "static"|"dhcp", -# "dns-search": , -# "hwaddress": , -# "auto": True (or non-existent), -# "dns-nameservers": [, ...], -# } -# } -# -# Things to note, comments are removed, if a ubuntu/debian interface is -# marked as auto then only then first segment (?) is retained, ie -# 'auto eth0 eth0:1' just marks eth0 as auto (not eth0:1). -# -# Example input: -# -# auto lo -# iface lo inet loopback -# -# auto eth0 -# iface eth0 inet static -# address 10.0.0.1 -# netmask 255.255.252.0 -# broadcast 10.0.0.255 -# gateway 10.0.0.2 -# dns-nameservers 98.0.0.1 98.0.0.2 -# -# Example output: -# { -# "lo": { -# "auto": true -# }, -# "eth0": { -# "auto": true, -# "dns-nameservers": [ -# "98.0.0.1", -# "98.0.0.2" -# ], -# "broadcast": "10.0.0.255", -# "netmask": "255.255.252.0", -# "bootproto": "static", -# "address": "10.0.0.1", -# "gateway": "10.0.0.2" -# } -# } - -def translate_network(settings): - # Get the standard cmd, args from the ubuntu format - entries = [] - for line in settings.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - split_up = line.split(None, 1) - if len(split_up) <= 1: - continue - entries.append(split_up) - # Figure out where each iface section is - ifaces = [] - consume = {} - for (cmd, args) in entries: - if cmd == 'iface': - if consume: - ifaces.append(consume) - consume = {} - consume[cmd] = args - else: - consume[cmd] = args - # Check if anything left over to consume - absorb = False - for (cmd, args) in consume.items(): - if cmd == 'iface': - absorb = True - if absorb: - ifaces.append(consume) - # Now translate - real_ifaces = {} - for info in ifaces: - if 'iface' not in info: - continue - iface_details = info['iface'].split(None) - # Check if current device *may* have an ipv6 IP - use_ipv6 = False - if 'inet6' in iface_details: - use_ipv6 = True - dev_name = None - if len(iface_details) >= 1: - dev = iface_details[0].strip().lower() - if dev: - dev_name = dev - if not dev_name: - continue - iface_info = {} - iface_info['ipv6'] = {} - if len(iface_details) >= 3: - proto_type = iface_details[2].strip().lower() - # Seems like this can be 'loopback' which we don't - # really care about - if proto_type in ['dhcp', 'static']: - iface_info['bootproto'] = proto_type - # These can just be copied over - if use_ipv6: - for k in ['address', 'gateway']: - if k in info: - val = info[k].strip().lower() - if val: - iface_info['ipv6'][k] = val - else: - for k in ['netmask', 'address', 'gateway', 'broadcast']: - if k in info: - val = info[k].strip().lower() - if val: - iface_info[k] = val - # Name server info provided?? - if 'dns-nameservers' in info: - iface_info['dns-nameservers'] = info['dns-nameservers'].split() - # Name server search info provided?? - if 'dns-search' in info: - iface_info['dns-search'] = info['dns-search'].split() - # Is any mac address spoofing going on?? - if 'hwaddress' in info: - hw_info = info['hwaddress'].lower().strip() - hw_split = hw_info.split(None, 1) - if len(hw_split) == 2 and hw_split[0].startswith('ether'): - hw_addr = hw_split[1] - if hw_addr: - iface_info['hwaddress'] = hw_addr - # If ipv6 is enabled, device will have multiple IPs, so we need to - # update the dictionary instead of overwriting it... - if dev_name in real_ifaces: - real_ifaces[dev_name].update(iface_info) - else: - real_ifaces[dev_name] = iface_info - # Check for those that should be started on boot via 'auto' - for (cmd, args) in entries: - args = args.split(None) - if not args: - continue - dev_name = args[0].strip().lower() - if cmd == 'auto': - # Seems like auto can be like 'auto eth0 eth0:1' so just get the - # first part out as the device name - if dev_name in real_ifaces: - real_ifaces[dev_name]['auto'] = True - if cmd == 'iface' and 'inet6' in args: - real_ifaces[dev_name]['inet6'] = True - return real_ifaces diff --git a/cloudinit/distros/parsers/__init__.py b/cloudinit/distros/parsers/__init__.py deleted file mode 100644 index 1c413eaa..00000000 --- a/cloudinit/distros/parsers/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -def chop_comment(text, comment_chars): - comment_locations = [text.find(c) for c in comment_chars] - comment_locations = [c for c in comment_locations if c != -1] - if not comment_locations: - return (text, '') - min_comment = min(comment_locations) - before_comment = text[0:min_comment] - comment = text[min_comment:] - return (before_comment, comment) diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py deleted file mode 100644 index efb185d4..00000000 --- a/cloudinit/distros/parsers/hostname.py +++ /dev/null @@ -1,88 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from six import StringIO - -from cloudinit.distros.parsers import chop_comment - - -# Parser that knows how to work with /etc/hostname format -class HostnameConf(object): - def __init__(self, text): - self._text = text - self._contents = None - - def parse(self): - if self._contents is None: - self._contents = self._parse(self._text) - - def __str__(self): - self.parse() - contents = StringIO() - for (line_type, components) in self._contents: - if line_type == 'blank': - contents.write("%s\n" % (components[0])) - elif line_type == 'all_comment': - contents.write("%s\n" % (components[0])) - elif line_type == 'hostname': - (hostname, tail) = components - contents.write("%s%s\n" % (hostname, tail)) - # Ensure trailing newline - contents = contents.getvalue() - if not contents.endswith("\n"): - contents += "\n" - return contents - - @property - def hostname(self): - self.parse() - for (line_type, components) in self._contents: - if line_type == 'hostname': - return components[0] - return None - - def set_hostname(self, your_hostname): - your_hostname = your_hostname.strip() - if not your_hostname: - return - self.parse() - replaced = False - for (line_type, components) in self._contents: - if line_type == 'hostname': - components[0] = str(your_hostname) - replaced = True - if not replaced: - self._contents.append(('hostname', [str(your_hostname), ''])) - - def _parse(self, contents): - entries = [] - hostnames_found = set() - for line in contents.splitlines(): - if not len(line.strip()): - entries.append(('blank', [line])) - continue - (head, tail) = chop_comment(line.strip(), '#') - if not len(head): - entries.append(('all_comment', [line])) - continue - entries.append(('hostname', [head, tail])) - hostnames_found.add(head) - if len(hostnames_found) > 1: - raise IOError("Multiple hostnames (%s) found!" - % (hostnames_found)) - return entries diff --git a/cloudinit/distros/parsers/hosts.py b/cloudinit/distros/parsers/hosts.py deleted file mode 100644 index 3c5498ee..00000000 --- a/cloudinit/distros/parsers/hosts.py +++ /dev/null @@ -1,92 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from six import StringIO - -from cloudinit.distros.parsers import chop_comment - - -# See: man hosts -# or http://unixhelp.ed.ac.uk/CGI/man-cgi?hosts -# or http://tinyurl.com/6lmox3 -class HostsConf(object): - def __init__(self, text): - self._text = text - self._contents = None - - def parse(self): - if self._contents is None: - self._contents = self._parse(self._text) - - def get_entry(self, ip): - self.parse() - options = [] - for (line_type, components) in self._contents: - if line_type == 'option': - (pieces, _tail) = components - if len(pieces) and pieces[0] == ip: - options.append(pieces[1:]) - return options - - def del_entries(self, ip): - self.parse() - n_entries = [] - for (line_type, components) in self._contents: - if line_type != 'option': - n_entries.append((line_type, components)) - continue - else: - (pieces, _tail) = components - if len(pieces) and pieces[0] == ip: - pass - elif len(pieces): - n_entries.append((line_type, list(components))) - self._contents = n_entries - - def add_entry(self, ip, canonical_hostname, *aliases): - self.parse() - self._contents.append(('option', - ([ip, canonical_hostname] + list(aliases), ''))) - - def _parse(self, contents): - entries = [] - for line in contents.splitlines(): - if not len(line.strip()): - entries.append(('blank', [line])) - continue - (head, tail) = chop_comment(line.strip(), '#') - if not len(head): - entries.append(('all_comment', [line])) - continue - entries.append(('option', [head.split(None), tail])) - return entries - - def __str__(self): - self.parse() - contents = StringIO() - for (line_type, components) in self._contents: - if line_type == 'blank': - contents.write("%s\n" % (components[0])) - elif line_type == 'all_comment': - contents.write("%s\n" % (components[0])) - elif line_type == 'option': - (pieces, tail) = components - pieces = [str(p) for p in pieces] - pieces = "\t".join(pieces) - contents.write("%s%s\n" % (pieces, tail)) - return contents.getvalue() diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py deleted file mode 100644 index 2ed13d9c..00000000 --- a/cloudinit/distros/parsers/resolv_conf.py +++ /dev/null @@ -1,169 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from six import StringIO - -from cloudinit import util - -from cloudinit.distros.parsers import chop_comment - - -# See: man resolv.conf -class ResolvConf(object): - def __init__(self, text): - self._text = text - self._contents = None - - def parse(self): - if self._contents is None: - self._contents = self._parse(self._text) - - @property - def nameservers(self): - self.parse() - return self._retr_option('nameserver') - - @property - def local_domain(self): - self.parse() - dm = self._retr_option('domain') - if dm: - return dm[0] - return None - - @property - def search_domains(self): - self.parse() - current_sds = self._retr_option('search') - flat_sds = [] - for sdlist in current_sds: - for sd in sdlist.split(None): - if sd: - flat_sds.append(sd) - return flat_sds - - def __str__(self): - self.parse() - contents = StringIO() - for (line_type, components) in self._contents: - if line_type == 'blank': - contents.write("\n") - elif line_type == 'all_comment': - contents.write("%s\n" % (components[0])) - elif line_type == 'option': - (cfg_opt, cfg_value, comment_tail) = components - line = "%s %s" % (cfg_opt, cfg_value) - if len(comment_tail): - line += comment_tail - contents.write("%s\n" % (line)) - return contents.getvalue() - - def _retr_option(self, opt_name): - found = [] - for (line_type, components) in self._contents: - if line_type == 'option': - (cfg_opt, cfg_value, _comment_tail) = components - if cfg_opt == opt_name: - found.append(cfg_value) - return found - - def add_nameserver(self, ns): - self.parse() - current_ns = self._retr_option('nameserver') - new_ns = list(current_ns) - new_ns.append(str(ns)) - new_ns = util.uniq_list(new_ns) - if len(new_ns) == len(current_ns): - return current_ns - if len(current_ns) >= 3: - # Hard restriction on only 3 name servers - raise ValueError(("Adding %r would go beyond the " - "'3' maximum name servers") % (ns)) - self._remove_option('nameserver') - for n in new_ns: - self._contents.append(('option', ['nameserver', n, ''])) - return new_ns - - def _remove_option(self, opt_name): - - def remove_opt(item): - line_type, components = item - if line_type != 'option': - return False - (cfg_opt, _cfg_value, _comment_tail) = components - if cfg_opt != opt_name: - return False - return True - - new_contents = [] - for c in self._contents: - if not remove_opt(c): - new_contents.append(c) - self._contents = new_contents - - def add_search_domain(self, search_domain): - flat_sds = self.search_domains - new_sds = list(flat_sds) - new_sds.append(str(search_domain)) - new_sds = util.uniq_list(new_sds) - if len(flat_sds) == len(new_sds): - return new_sds - if len(flat_sds) >= 6: - # Hard restriction on only 6 search domains - raise ValueError(("Adding %r would go beyond the " - "'6' maximum search domains") % (search_domain)) - s_list = " ".join(new_sds) - if len(s_list) > 256: - # Some hard limit on 256 chars total - raise ValueError(("Adding %r would go beyond the " - "256 maximum search list character limit") - % (search_domain)) - self._remove_option('search') - self._contents.append(('option', ['search', s_list, ''])) - return flat_sds - - @local_domain.setter - def local_domain(self, domain): - self.parse() - self._remove_option('domain') - self._contents.append(('option', ['domain', str(domain), ''])) - return domain - - def _parse(self, contents): - entries = [] - for (i, line) in enumerate(contents.splitlines()): - sline = line.strip() - if not sline: - entries.append(('blank', [line])) - continue - (head, tail) = chop_comment(line, ';#') - if not len(head.strip()): - entries.append(('all_comment', [line])) - continue - if not tail: - tail = '' - try: - (cfg_opt, cfg_values) = head.split(None, 1) - except (IndexError, ValueError): - raise IOError("Incorrectly formatted resolv.conf line %s" - % (i + 1)) - if cfg_opt not in ['nameserver', 'domain', - 'search', 'sortlist', 'options']: - raise IOError("Unexpected resolv.conf option %s" % (cfg_opt)) - entries.append(("option", [cfg_opt, cfg_values, tail])) - return entries diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py deleted file mode 100644 index 6157cf32..00000000 --- a/cloudinit/distros/parsers/sys_conf.py +++ /dev/null @@ -1,113 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import six -from six import StringIO - -import pipes -import re - -# This library is used to parse/write -# out the various sysconfig files edited (best attempt effort) -# -# It has to be slightly modified though -# to ensure that all values are quoted/unquoted correctly -# since these configs are usually sourced into -# bash scripts... -import configobj - -# See: http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html -# or look at the 'param_expand()' function in the subst.c file in the bash -# source tarball... -SHELL_VAR_RULE = r'[a-zA-Z_]+[a-zA-Z0-9_]*' -SHELL_VAR_REGEXES = [ - # Basic variables - re.compile(r"\$" + SHELL_VAR_RULE), - # Things like $?, $0, $-, $@ - re.compile(r"\$[0-9#\?\-@\*]"), - # Things like ${blah:1} - but this one - # gets very complex so just try the - # simple path - re.compile(r"\$\{.+\}"), -] - - -def _contains_shell_variable(text): - for r in SHELL_VAR_REGEXES: - if r.search(text): - return True - return False - - -class SysConf(configobj.ConfigObj): - def __init__(self, contents): - configobj.ConfigObj.__init__(self, contents, - interpolation=False, - write_empty_values=True) - - def __str__(self): - contents = self.write() - out_contents = StringIO() - if isinstance(contents, (list, tuple)): - out_contents.write("\n".join(contents)) - else: - out_contents.write(str(contents)) - return out_contents.getvalue() - - def _quote(self, value, multiline=False): - if not isinstance(value, six.string_types): - raise ValueError('Value "%s" is not a string' % (value)) - if len(value) == 0: - return '' - quot_func = None - if value[0] in ['"', "'"] and value[-1] in ['"', "'"]: - if len(value) == 1: - quot_func = (lambda x: self._get_single_quote(x) % x) - else: - # Quote whitespace if it isn't the start + end of a shell command - if value.strip().startswith("$(") and value.strip().endswith(")"): - pass - else: - if re.search(r"[\t\r\n ]", value): - if _contains_shell_variable(value): - # If it contains shell variables then we likely want to - # leave it alone since the pipes.quote function likes - # to use single quotes which won't get expanded... - if re.search(r"[\n\"']", value): - quot_func = (lambda x: - self._get_triple_quote(x) % x) - else: - quot_func = (lambda x: - self._get_single_quote(x) % x) - else: - quot_func = pipes.quote - if not quot_func: - return value - return quot_func(value) - - def _write_line(self, indent_string, entry, this_entry, comment): - # Ensure it is formatted fine for - # how these sysconfig scripts are used - val = self._decode_element(self._quote(this_entry)) - key = self._decode_element(self._quote(entry)) - cmnt = self._decode_element(comment) - return '%s%s%s%s%s' % (indent_string, - key, - self._a_to_u('='), - val, - cmnt) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py deleted file mode 100644 index 1aa42d75..00000000 --- a/cloudinit/distros/rhel.py +++ /dev/null @@ -1,230 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import distros -from cloudinit import helpers -from cloudinit import log as logging -from cloudinit.net.network_state import parse_net_config_data -from cloudinit.net import sysconfig -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros import rhel_util -from cloudinit.settings import PER_INSTANCE - -LOG = logging.getLogger(__name__) - - -def _make_sysconfig_bool(val): - if val: - return 'yes' - else: - return 'no' - - -class Distro(distros.Distro): - # See: http://tiny.cc/6r99fw - clock_conf_fn = "/etc/sysconfig/clock" - locale_conf_fn = '/etc/sysconfig/i18n' - systemd_locale_conf_fn = '/etc/locale.conf' - network_conf_fn = "/etc/sysconfig/network" - hostname_conf_fn = "/etc/sysconfig/network" - systemd_hostname_conf_fn = "/etc/hostname" - network_script_tpl = '/etc/sysconfig/network-scripts/ifcfg-%s' - resolve_conf_fn = "/etc/resolv.conf" - tz_local_fn = "/etc/localtime" - usr_lib_exec = "/usr/libexec" - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'redhat' - self._net_renderer = sysconfig.Renderer() - - def install_packages(self, pkglist): - self.package_command('install', pkgs=pkglist) - - def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) - return [] - - def _write_network(self, settings): - # TODO(harlowja) fix this... since this is the ubuntu format - entries = net_util.translate_network(settings) - LOG.debug("Translated ubuntu style network settings %s into %s", - settings, entries) - # Make the intermediate format as the rhel format... - nameservers = [] - searchservers = [] - dev_names = entries.keys() - use_ipv6 = False - for (dev, info) in entries.items(): - net_fn = self.network_script_tpl % (dev) - net_cfg = { - 'DEVICE': dev, - 'NETMASK': info.get('netmask'), - 'IPADDR': info.get('address'), - 'BOOTPROTO': info.get('bootproto'), - 'GATEWAY': info.get('gateway'), - 'BROADCAST': info.get('broadcast'), - 'MACADDR': info.get('hwaddress'), - 'ONBOOT': _make_sysconfig_bool(info.get('auto')), - } - if info.get('inet6'): - use_ipv6 = True - net_cfg.update({ - 'IPV6INIT': _make_sysconfig_bool(True), - 'IPV6ADDR': info.get('ipv6').get('address'), - 'IPV6_DEFAULTGW': info.get('ipv6').get('gateway'), - }) - rhel_util.update_sysconfig_file(net_fn, net_cfg) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - if 'dns-search' in info: - searchservers.extend(info['dns-search']) - if nameservers or searchservers: - rhel_util.update_resolve_conf_file(self.resolve_conf_fn, - nameservers, searchservers) - if dev_names: - net_cfg = { - 'NETWORKING': _make_sysconfig_bool(True), - } - # If IPv6 interface present, enable ipv6 networking - if use_ipv6: - net_cfg['NETWORKING_IPV6'] = _make_sysconfig_bool(True) - net_cfg['IPV6_AUTOCONF'] = _make_sysconfig_bool(False) - rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg) - return dev_names - - def apply_locale(self, locale, out_fn=None): - if self.uses_systemd(): - if not out_fn: - out_fn = self.systemd_locale_conf_fn - out_fn = self.systemd_locale_conf_fn - else: - if not out_fn: - out_fn = self.locale_conf_fn - locale_cfg = { - 'LANG': locale, - } - rhel_util.update_sysconfig_file(out_fn, locale_cfg) - - def _write_hostname(self, hostname, out_fn): - # systemd will never update previous-hostname for us, so - # we need to do it ourselves - if self.uses_systemd() and out_fn.endswith('/previous-hostname'): - util.write_file(out_fn, hostname) - elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) - else: - host_cfg = { - 'HOSTNAME': hostname, - } - rhel_util.update_sysconfig_file(out_fn, host_cfg) - - def _select_hostname(self, hostname, fqdn): - # See: http://bit.ly/TwitgL - # Should be fqdn if we can use it - if fqdn: - return fqdn - return hostname - - def _read_system_hostname(self): - if self.uses_systemd(): - host_fn = self.systemd_hostname_conf_fn - else: - host_fn = self.hostname_conf_fn - return (host_fn, self._read_hostname(host_fn)) - - def _read_hostname(self, filename, default=None): - if self.uses_systemd() and filename.endswith('/previous-hostname'): - return util.load_file(filename).strip() - elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) - if len(out): - return out - else: - return default - else: - (_exists, contents) = rhel_util.read_sysconfig_file(filename) - if 'HOSTNAME' in contents: - return contents['HOSTNAME'] - else: - return default - - def _bring_up_interfaces(self, device_names): - if device_names and 'all' in device_names: - raise RuntimeError(('Distro %s can not translate ' - 'the device name "all"') % (self.name)) - return distros.Distro._bring_up_interfaces(self, device_names) - - def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - if self.uses_systemd(): - # Currently, timedatectl complains if invoked during startup - # so for compatibility, create the link manually. - util.del_file(self.tz_local_fn) - util.sym_link(tz_file, self.tz_local_fn) - else: - # Adjust the sysconfig clock zone setting - clock_cfg = { - 'ZONE': str(tz), - } - rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['yum'] - # If enabled, then yum will be tolerant of errors on the command line - # with regard to packages. - # For example: if you request to install foo, bar and baz and baz is - # installed; yum won't error out complaining that baz is already - # installed. - cmd.append("-t") - # Determines whether or not yum prompts for confirmation - # of critical actions. We don't want to prompt... - cmd.append("-y") - - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - cmd.append(command) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ["makecache"], freq=PER_INSTANCE) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py deleted file mode 100644 index 903d7793..00000000 --- a/cloudinit/distros/rhel_util.py +++ /dev/null @@ -1,89 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -from cloudinit.distros.parsers.resolv_conf import ResolvConf -from cloudinit.distros.parsers.sys_conf import SysConf - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -# Helper function to update a RHEL/SUSE /etc/sysconfig/* file -def update_sysconfig_file(fn, adjustments, allow_empty=False): - if not adjustments: - return - (exists, contents) = read_sysconfig_file(fn) - updated_am = 0 - for (k, v) in adjustments.items(): - if v is None: - continue - v = str(v) - if len(v) == 0 and not allow_empty: - continue - contents[k] = v - updated_am += 1 - if updated_am: - lines = [ - str(contents), - ] - if not exists: - lines.insert(0, util.make_header()) - util.write_file(fn, "\n".join(lines) + "\n", 0o644) - - -# Helper function to read a RHEL/SUSE /etc/sysconfig/* file -def read_sysconfig_file(fn): - exists = False - try: - contents = util.load_file(fn).splitlines() - exists = True - except IOError: - contents = [] - return (exists, SysConf(contents)) - - -# Helper function to update RHEL/SUSE /etc/resolv.conf -def update_resolve_conf_file(fn, dns_servers, search_servers): - try: - r_conf = ResolvConf(util.load_file(fn)) - r_conf.parse() - except IOError: - util.logexc(LOG, "Failed at parsing %s reverting to an empty " - "instance", fn) - r_conf = ResolvConf('') - r_conf.parse() - if dns_servers: - for s in dns_servers: - try: - r_conf.add_nameserver(s) - except ValueError: - util.logexc(LOG, "Failed at adding nameserver %s", s) - if search_servers: - for s in search_servers: - try: - r_conf.add_search_domain(s) - except ValueError: - util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0o644) diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py deleted file mode 100644 index 620c974c..00000000 --- a/cloudinit/distros/sles.py +++ /dev/null @@ -1,179 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Juerg Haefliger -# -# Leaning very heavily on the RHEL and Debian implementation -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import distros - -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit import helpers -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros import rhel_util -from cloudinit.settings import PER_INSTANCE - -LOG = logging.getLogger(__name__) - - -class Distro(distros.Distro): - clock_conf_fn = '/etc/sysconfig/clock' - locale_conf_fn = '/etc/sysconfig/language' - network_conf_fn = '/etc/sysconfig/network' - hostname_conf_fn = '/etc/HOSTNAME' - network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' - resolve_conf_fn = '/etc/resolv.conf' - tz_local_fn = '/etc/localtime' - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'suse' - - def install_packages(self, pkglist): - self.package_command('install', args='-l', pkgs=pkglist) - - def _write_network(self, settings): - # Convert debian settings to ifcfg format - entries = net_util.translate_network(settings) - LOG.debug("Translated ubuntu style network settings %s into %s", - settings, entries) - # Make the intermediate format as the suse format... - nameservers = [] - searchservers = [] - dev_names = entries.keys() - for (dev, info) in entries.items(): - net_fn = self.network_script_tpl % (dev) - mode = info.get('auto') - if mode and mode.lower() == 'true': - mode = 'auto' - else: - mode = 'manual' - net_cfg = { - 'BOOTPROTO': info.get('bootproto'), - 'BROADCAST': info.get('broadcast'), - 'GATEWAY': info.get('gateway'), - 'IPADDR': info.get('address'), - 'LLADDR': info.get('hwaddress'), - 'NETMASK': info.get('netmask'), - 'STARTMODE': mode, - 'USERCONTROL': 'no' - } - if dev != 'lo': - net_cfg['ETHERDEVICE'] = dev - net_cfg['ETHTOOL_OPTIONS'] = '' - else: - net_cfg['FIREWALL'] = 'no' - rhel_util.update_sysconfig_file(net_fn, net_cfg, True) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - if 'dns-search' in info: - searchservers.extend(info['dns-search']) - if nameservers or searchservers: - rhel_util.update_resolve_conf_file(self.resolve_conf_fn, - nameservers, searchservers) - return dev_names - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - locale_cfg = { - 'RC_LANG': locale, - } - rhel_util.update_sysconfig_file(out_fn, locale_cfg) - - def _write_hostname(self, hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0o644) - - def _read_system_hostname(self): - host_fn = self.hostname_conf_fn - return (host_fn, self._read_hostname(host_fn)) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def _bring_up_interfaces(self, device_names): - if device_names and 'all' in device_names: - raise RuntimeError(('Distro %s can not translate ' - 'the device name "all"') % (self.name)) - return distros.Distro._bring_up_interfaces(self, device_names) - - def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Adjust the sysconfig clock zone setting - clock_cfg = { - 'TIMEZONE': str(tz), - } - rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['zypper'] - # No user interaction possible, enable non-interactive mode - cmd.append('--non-interactive') - - # Comand is the operation, such as install - cmd.append(command) - - # args are the arguments to the command, not global options - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ['refresh'], freq=PER_INSTANCE) diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py deleted file mode 100644 index c527f248..00000000 --- a/cloudinit/distros/ubuntu.py +++ /dev/null @@ -1,31 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.distros import debian -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - - -class Distro(debian.Distro): - pass diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py deleted file mode 100644 index 76dda042..00000000 --- a/cloudinit/ec2_utils.py +++ /dev/null @@ -1,201 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import functools -import json - -from cloudinit import log as logging -from cloudinit import url_helper -from cloudinit import util - -LOG = logging.getLogger(__name__) -SKIP_USERDATA_CODES = frozenset([url_helper.NOT_FOUND]) - - -class MetadataLeafDecoder(object): - """Decodes a leaf blob into something meaningful.""" - - def _maybe_json_object(self, text): - if not text: - return False - text = text.strip() - if text.startswith("{") and text.endswith("}"): - return True - return False - - def __call__(self, field, blob): - if not blob: - return blob - try: - blob = util.decode_binary(blob) - except UnicodeDecodeError: - return blob - if self._maybe_json_object(blob): - try: - # Assume it's json, unless it fails parsing... - return json.loads(blob) - except (ValueError, TypeError) as e: - LOG.warn("Field %s looked like a json object, but it was" - " not: %s", field, e) - if blob.find("\n") != -1: - return blob.splitlines() - return blob - - -# See: http://bit.ly/TyoUQs -# -class MetadataMaterializer(object): - def __init__(self, blob, base_url, caller, leaf_decoder=None): - self._blob = blob - self._md = None - self._base_url = base_url - self._caller = caller - if leaf_decoder is None: - self._leaf_decoder = MetadataLeafDecoder() - else: - self._leaf_decoder = leaf_decoder - - def _parse(self, blob): - leaves = {} - children = [] - blob = util.decode_binary(blob) - - if not blob: - return (leaves, children) - - def has_children(item): - if item.endswith("/"): - return True - else: - return False - - def get_name(item): - if item.endswith("/"): - return item.rstrip("/") - return item - - for field in blob.splitlines(): - field = field.strip() - field_name = get_name(field) - if not field or not field_name: - continue - if has_children(field): - if field_name not in children: - children.append(field_name) - else: - contents = field.split("=", 1) - resource = field_name - if len(contents) > 1: - # What a PITA... - (ident, sub_contents) = contents - ident = util.safe_int(ident) - if ident is not None: - resource = "%s/openssh-key" % (ident) - field_name = sub_contents - leaves[field_name] = resource - return (leaves, children) - - def materialize(self): - if self._md is not None: - return self._md - self._md = self._materialize(self._blob, self._base_url) - return self._md - - def _materialize(self, blob, base_url): - (leaves, children) = self._parse(blob) - child_contents = {} - for c in children: - child_url = url_helper.combine_url(base_url, c) - if not child_url.endswith("/"): - child_url += "/" - child_blob = self._caller(child_url) - child_contents[c] = self._materialize(child_blob, child_url) - leaf_contents = {} - for (field, resource) in leaves.items(): - leaf_url = url_helper.combine_url(base_url, resource) - leaf_blob = self._caller(leaf_url) - leaf_contents[field] = self._leaf_decoder(field, leaf_blob) - joined = {} - joined.update(child_contents) - for field in leaf_contents.keys(): - if field in joined: - LOG.warn("Duplicate key found in results from %s", base_url) - else: - joined[field] = leaf_contents[field] - return joined - - -def _skip_retry_on_codes(status_codes, _request_args, cause): - """Returns if a request should retry based on a given set of codes that - case retrying to be stopped/skipped. - """ - return cause.code in status_codes - - -def get_instance_userdata(api_version='latest', - metadata_address='http://169.254.169.254', - ssl_details=None, timeout=5, retries=5): - ud_url = url_helper.combine_url(metadata_address, api_version) - ud_url = url_helper.combine_url(ud_url, 'user-data') - user_data = '' - try: - # It is ok for userdata to not exist (thats why we are stopping if - # NOT_FOUND occurs) and just in that case returning an empty string. - exception_cb = functools.partial(_skip_retry_on_codes, - SKIP_USERDATA_CODES) - response = util.read_file_or_url(ud_url, - ssl_details=ssl_details, - timeout=timeout, - retries=retries, - exception_cb=exception_cb) - user_data = response.contents - except url_helper.UrlError as e: - if e.code not in SKIP_USERDATA_CODES: - util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - except Exception: - util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return user_data - - -def get_instance_metadata(api_version='latest', - metadata_address='http://169.254.169.254', - ssl_details=None, timeout=5, retries=5, - leaf_decoder=None): - md_url = url_helper.combine_url(metadata_address, api_version) - # Note, 'meta-data' explicitly has trailing /. - # this is required for CloudStack (LP: #1356855) - md_url = url_helper.combine_url(md_url, 'meta-data/') - caller = functools.partial(util.read_file_or_url, - ssl_details=ssl_details, timeout=timeout, - retries=retries) - - def mcaller(url): - return caller(url).contents - - try: - response = caller(md_url) - materializer = MetadataMaterializer(response.contents, - md_url, mcaller, - leaf_decoder=leaf_decoder) - md = materializer.materialize() - if not isinstance(md, (dict)): - md = {} - return md - except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", md_url) - return {} diff --git a/cloudinit/filters/__init__.py b/cloudinit/filters/__init__.py deleted file mode 100644 index da124641..00000000 --- a/cloudinit/filters/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/cloudinit/filters/launch_index.py b/cloudinit/filters/launch_index.py deleted file mode 100644 index baecdac9..00000000 --- a/cloudinit/filters/launch_index.py +++ /dev/null @@ -1,75 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import copy - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -class Filter(object): - def __init__(self, wanted_idx, allow_none=True): - self.wanted_idx = wanted_idx - self.allow_none = allow_none - - def _select(self, message): - msg_idx = message.get('Launch-Index', None) - if self.allow_none and msg_idx is None: - return True - msg_idx = util.safe_int(msg_idx) - if msg_idx != self.wanted_idx: - return False - return True - - def _do_filter(self, message): - # Don't use walk() here since we want to do the reforming of the - # messages ourselves and not flatten the message listings... - if not self._select(message): - return None - if message.is_multipart(): - # Recreate it and its child messages - prev_msgs = message.get_payload(decode=False) - new_msgs = [] - discarded = 0 - for m in prev_msgs: - m = self._do_filter(m) - if m is not None: - new_msgs.append(m) - else: - discarded += 1 - LOG.debug(("Discarding %s multipart messages " - "which do not match launch index %s"), - discarded, self.wanted_idx) - new_message = copy.copy(message) - new_message.set_payload(new_msgs) - new_message[ud.ATTACHMENT_FIELD] = str(len(new_msgs)) - return new_message - else: - return copy.copy(message) - - def apply(self, root_message): - if self.wanted_idx is None: - return root_message - return self._do_filter(root_message) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py deleted file mode 100644 index 6a76d785..00000000 --- a/cloudinit/gpg.py +++ /dev/null @@ -1,74 +0,0 @@ -"""gpg.py - Collection of gpg key related functions""" -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# -# Author: Scott Moser -# Author: Christian Ehrhardt -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -def export_armour(key): - """Export gpg key, armoured key gets returned""" - try: - (armour, _) = util.subp(["gpg", "--export", "--armour", key], - capture=True) - except util.ProcessExecutionError as error: - # debug, since it happens for any key not on the system initially - LOG.debug('Failed to export armoured key "%s": %s', key, error) - armour = None - return armour - - -def receive_key(key, keyserver): - """Receive gpg key from the specified keyserver""" - LOG.debug('Receive gpg key "%s"', key) - try: - util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key], - capture=True) - except util.ProcessExecutionError as error: - raise ValueError(('Failed to import key "%s" ' - 'from server "%s" - error %s') % - (key, keyserver, error)) - - -def delete_key(key): - """Delete the specified key from the local gpg ring""" - try: - util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], - capture=True) - except util.ProcessExecutionError as error: - LOG.warn('Failed delete key "%s": %s', key, error) - - -def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"): - """get gpg keyid from keyserver""" - armour = export_armour(keyid) - if not armour: - try: - receive_key(keyid, keyserver=keyserver) - armour = export_armour(keyid) - except ValueError: - LOG.exception('Failed to obtain gpg key %s', keyid) - raise - finally: - # delete just imported key to leave environment as it was before - delete_key(keyid) - - return armour diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py deleted file mode 100644 index b6c43ce8..00000000 --- a/cloudinit/handlers/__init__.py +++ /dev/null @@ -1,274 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import abc -import os -import six - -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import type_utils -from cloudinit import util - -LOG = logging.getLogger(__name__) - -# Used as the content type when a message is not multipart -# and it doesn't contain its own content-type -NOT_MULTIPART_TYPE = "text/x-not-multipart" - -# When none is assigned this gets used -OCTET_TYPE = 'application/octet-stream' - -# Special content types that signal the start and end of processing -CONTENT_END = "__end__" -CONTENT_START = "__begin__" -CONTENT_SIGNALS = [CONTENT_START, CONTENT_END] - -# Used when a part-handler type is encountered -# to allow for registration of new types. -PART_CONTENT_TYPES = ["text/part-handler"] -PART_HANDLER_FN_TMPL = 'part-handler-%03d' - -# For parts without filenames -PART_FN_TPL = 'part-%03d' - -# Different file beginnings to there content type -INCLUSION_TYPES_MAP = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', - '#cloud-config-jsonp': 'text/cloud-config-jsonp', -} - -# Sorted longest first -INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), - key=(lambda e: 0 - len(e))) - - -@six.add_metaclass(abc.ABCMeta) -class Handler(object): - - def __init__(self, frequency, version=2): - self.handler_version = version - self.frequency = frequency - - def __repr__(self): - return "%s: [%s]" % (type_utils.obj_name(self), self.list_types()) - - @abc.abstractmethod - def list_types(self): - raise NotImplementedError() - - @abc.abstractmethod - def handle_part(self, *args, **kwargs): - raise NotImplementedError() - - -def run_part(mod, data, filename, payload, frequency, headers): - mod_freq = mod.frequency - if not (mod_freq == PER_ALWAYS or - (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): - return - # Sanity checks on version (should be an int convertable) - try: - mod_ver = mod.handler_version - mod_ver = int(mod_ver) - except (TypeError, ValueError, AttributeError): - mod_ver = 1 - content_type = headers['Content-Type'] - try: - LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", - mod, content_type, filename, mod_ver, frequency) - if mod_ver == 3: - # Treat as v. 3 which does get a frequency + headers - mod.handle_part(data, content_type, filename, - payload, frequency, headers) - elif mod_ver == 2: - # Treat as v. 2 which does get a frequency - mod.handle_part(data, content_type, filename, - payload, frequency) - elif mod_ver == 1: - # Treat as v. 1 which gets no frequency - mod.handle_part(data, content_type, filename, payload) - else: - raise ValueError("Unknown module version %s" % (mod_ver)) - except Exception: - util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " - "frequency %s", mod, content_type, filename, mod_ver, - frequency) - - -def call_begin(mod, data, frequency): - # Create a fake header set - headers = { - 'Content-Type': CONTENT_START, - } - run_part(mod, data, None, None, frequency, headers) - - -def call_end(mod, data, frequency): - # Create a fake header set - headers = { - 'Content-Type': CONTENT_END, - } - run_part(mod, data, None, None, frequency, headers) - - -def walker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = PART_HANDLER_FN_TMPL % (curcount) - frequency = pdata['frequency'] - modfname = os.path.join(pdata['handlerdir'], "%s" % (modname)) - if not modfname.endswith(".py"): - modfname = "%s.py" % (modfname) - # TODO(harlowja): Check if path exists?? - util.write_file(modfname, payload, 0o600) - handlers = pdata['handlers'] - try: - mod = fixup_handler(importer.import_module(modname)) - call_begin(mod, pdata['data'], frequency) - # Only register and increment after the above have worked, so we don't - # register if it fails starting. - handlers.register(mod, initialized=True) - pdata['handlercount'] = curcount + 1 - except Exception: - util.logexc(LOG, "Failed at registering python file: %s (part " - "handler %s)", modfname, curcount) - - -def _extract_first_or_bytes(blob, size): - # Extract the first line or upto X symbols for text objects - # Extract first X bytes for binary objects - try: - if isinstance(blob, six.string_types): - start = blob.split("\n", 1)[0] - else: - # We want to avoid decoding the whole blob (it might be huge) - # By taking 4*size bytes we guarantee to decode size utf8 chars - start = blob[:4 * size].decode(errors='ignore').split("\n", 1)[0] - if len(start) >= size: - start = start[:size] - except UnicodeDecodeError: - # Bytes array doesn't contain text so return chunk of raw bytes - start = blob[0:size] - return start - - -def _escape_string(text): - try: - return text.encode("string_escape") - except (LookupError, TypeError): - try: - # Unicode (and Python 3's str) doesn't support string_escape... - return text.encode('unicode_escape') - except TypeError: - # Give up... - pass - except AttributeError: - # We're in Python3 and received blob as text - # No escaping is needed because bytes are printed - # as 'b\xAA\xBB' automatically in Python3 - pass - return text - - -def walker_callback(data, filename, payload, headers): - content_type = headers['Content-Type'] - if content_type in data.get('excluded'): - LOG.debug('content_type "%s" is excluded', content_type) - return - - if content_type in PART_CONTENT_TYPES: - walker_handle_handler(data, content_type, filename, payload) - return - handlers = data['handlers'] - if content_type in handlers: - run_part(handlers[content_type], data['data'], filename, - payload, data['frequency'], headers) - elif payload: - # Extract the first line or 24 bytes for displaying in the log - start = _extract_first_or_bytes(payload, 24) - details = "'%s...'" % (_escape_string(start)) - if content_type == NOT_MULTIPART_TYPE: - LOG.warning("Unhandled non-multipart (%s) userdata: %s", - content_type, details) - else: - LOG.warning("Unhandled unknown content-type (%s) userdata: %s", - content_type, details) - else: - LOG.debug("Empty payload of type %s", content_type) - - -# Callback is a function that will be called with -# (data, content_type, filename, payload) -def walk(msg, callback, data): - partnum = 0 - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = OCTET_TYPE - - filename = part.get_filename() - if not filename: - filename = PART_FN_TPL % (partnum) - - headers = dict(part) - LOG.debug(headers) - headers['Content-Type'] = ctype - payload = util.fully_decoded_payload(part) - callback(data, filename, payload, headers) - partnum = partnum + 1 - - -def fixup_handler(mod, def_freq=PER_INSTANCE): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - if not hasattr(mod, 'frequency'): - setattr(mod, 'frequency', def_freq) - else: - freq = mod.frequency - if freq and freq not in FREQUENCIES: - LOG.warn("Handler %s has an unknown frequency %s", mod, freq) - return mod - - -def type_from_starts_with(payload, default=None): - try: - payload_lc = util.decode_binary(payload).lower() - except UnicodeDecodeError: - return default - payload_lc = payload_lc.lstrip() - for text in INCLUSION_SRCH: - if payload_lc.startswith(text): - return INCLUSION_TYPES_MAP[text] - return default diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py deleted file mode 100644 index a4ea47ac..00000000 --- a/cloudinit/handlers/boot_hook.py +++ /dev/null @@ -1,70 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import handlers -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) -BOOTHOOK_PREFIX = "#cloud-boothook" - - -class BootHookPartHandler(handlers.Handler): - def __init__(self, paths, datasource, **_kwargs): - handlers.Handler.__init__(self, PER_ALWAYS) - self.boothook_dir = paths.get_ipath("boothooks") - self.instance_id = None - if datasource: - self.instance_id = datasource.get_instance_id() - - def list_types(self): - return [ - handlers.type_from_starts_with(BOOTHOOK_PREFIX), - ] - - def _write_part(self, payload, filename): - filename = util.clean_filename(filename) - filepath = os.path.join(self.boothook_dir, filename) - contents = util.strip_prefix_suffix(util.dos2unix(payload), - prefix=BOOTHOOK_PREFIX) - util.write_file(filepath, contents.lstrip(), 0o700) - return filepath - - def handle_part(self, data, ctype, filename, payload, frequency): - if ctype in handlers.CONTENT_SIGNALS: - return - - filepath = self._write_part(payload, filename) - try: - env = os.environ.copy() - if self.instance_id is not None: - env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError: - util.logexc(LOG, "Boothooks script %s execution error", filepath) - except Exception: - util.logexc(LOG, "Boothooks unknown error when running %s", - filepath) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py deleted file mode 100644 index cad4dc0f..00000000 --- a/cloudinit/handlers/cloud_config.py +++ /dev/null @@ -1,163 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import jsonpatch - -from cloudinit import handlers -from cloudinit import log as logging -from cloudinit import mergers -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - -MERGE_HEADER = 'Merge-Type' - -# Due to the way the loading of yaml configuration was done previously, -# where previously each cloud config part was appended to a larger yaml -# file and then finally that file was loaded as one big yaml file we need -# to mimic that behavior by altering the default strategy to be replacing -# keys of prior merges. -# -# -# For example -# #file 1 -# a: 3 -# #file 2 -# a: 22 -# #combined file (comments not included) -# a: 3 -# a: 22 -# -# This gets loaded into yaml with final result {'a': 22} -DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') -CLOUD_PREFIX = "#cloud-config" -JSONP_PREFIX = "#cloud-config-jsonp" - -# The file header -> content types this module will handle. -CC_TYPES = { - JSONP_PREFIX: handlers.type_from_starts_with(JSONP_PREFIX), - CLOUD_PREFIX: handlers.type_from_starts_with(CLOUD_PREFIX), -} - - -class CloudConfigPartHandler(handlers.Handler): - def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_ALWAYS, version=3) - self.cloud_buf = None - self.cloud_fn = paths.get_ipath("cloud_config") - if 'cloud_config_path' in _kwargs: - self.cloud_fn = paths.get_ipath(_kwargs["cloud_config_path"]) - self.file_names = [] - - def list_types(self): - return list(CC_TYPES.values()) - - def _write_cloud_config(self): - if not self.cloud_fn: - return - # Capture which files we merged from... - file_lines = [] - if self.file_names: - file_lines.append("# from %s files" % (len(self.file_names))) - for fn in self.file_names: - if not fn: - fn = '?' - file_lines.append("# %s" % (fn)) - file_lines.append("") - if self.cloud_buf is not None: - # Something was actually gathered.... - lines = [ - CLOUD_PREFIX, - '', - ] - lines.extend(file_lines) - lines.append(util.yaml_dumps(self.cloud_buf)) - else: - lines = [] - util.write_file(self.cloud_fn, "\n".join(lines), 0o600) - - def _extract_mergers(self, payload, headers): - merge_header_headers = '' - for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]: - tmp_h = headers.get(h, '') - if tmp_h: - merge_header_headers = tmp_h - break - # Select either the merge-type from the content - # or the merge type from the headers or default to our own set - # if neither exists (or is empty) from the later. - payload_yaml = util.load_yaml(payload) - mergers_yaml = mergers.dict_extract_mergers(payload_yaml) - mergers_header = mergers.string_extract_mergers(merge_header_headers) - all_mergers = [] - all_mergers.extend(mergers_yaml) - all_mergers.extend(mergers_header) - if not all_mergers: - all_mergers = DEF_MERGERS - return (payload_yaml, all_mergers) - - def _merge_patch(self, payload): - # JSON doesn't handle comments in this manner, so ensure that - # if we started with this 'type' that we remove it before - # attempting to load it as json (which the jsonpatch library will - # attempt to do). - payload = payload.lstrip() - payload = util.strip_prefix_suffix(payload, prefix=JSONP_PREFIX) - patch = jsonpatch.JsonPatch.from_string(payload) - LOG.debug("Merging by applying json patch %s", patch) - self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) - - def _merge_part(self, payload, headers): - (payload_yaml, my_mergers) = self._extract_mergers(payload, headers) - LOG.debug("Merging by applying %s", my_mergers) - merger = mergers.construct(my_mergers) - self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml) - - def _reset(self): - self.file_names = [] - self.cloud_buf = None - - def handle_part(self, data, ctype, filename, payload, frequency, headers): - if ctype == handlers.CONTENT_START: - self._reset() - return - if ctype == handlers.CONTENT_END: - self._write_cloud_config() - self._reset() - return - try: - # First time through, merge with an empty dict... - if self.cloud_buf is None or not self.file_names: - self.cloud_buf = {} - if ctype == CC_TYPES[JSONP_PREFIX]: - self._merge_patch(payload) - else: - self._merge_part(payload, headers) - # Ensure filename is ok to store - for i in ("\n", "\r", "\t"): - filename = filename.replace(i, " ") - self.file_names.append(filename.strip()) - except Exception: - util.logexc(LOG, "Failed at merging in cloud config part from %s", - filename) diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py deleted file mode 100644 index b5087693..00000000 --- a/cloudinit/handlers/shell_script.py +++ /dev/null @@ -1,55 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import handlers -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) -SHELL_PREFIX = "#!" - - -class ShellScriptPartHandler(handlers.Handler): - def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_ALWAYS) - self.script_dir = paths.get_ipath_cur('scripts') - if 'script_path' in _kwargs: - self.script_dir = paths.get_ipath_cur(_kwargs['script_path']) - - def list_types(self): - return [ - handlers.type_from_starts_with(SHELL_PREFIX), - ] - - def handle_part(self, data, ctype, filename, payload, frequency): - if ctype in handlers.CONTENT_SIGNALS: - # TODO(harlowja): maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - path = os.path.join(self.script_dir, filename) - util.write_file(path, payload, 0o700) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py deleted file mode 100644 index ab381e00..00000000 --- a/cloudinit/handlers/upstart_job.py +++ /dev/null @@ -1,119 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import os -import re - -from cloudinit import handlers -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.settings import (PER_INSTANCE) - -LOG = logging.getLogger(__name__) -UPSTART_PREFIX = "#upstart-job" - - -class UpstartJobPartHandler(handlers.Handler): - def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_INSTANCE) - self.upstart_dir = paths.upstart_conf_d - - def list_types(self): - return [ - handlers.type_from_starts_with(UPSTART_PREFIX), - ] - - def handle_part(self, data, ctype, filename, payload, frequency): - if ctype in handlers.CONTENT_SIGNALS: - return - - # See: https://bugs.launchpad.net/bugs/819507 - if frequency != PER_INSTANCE: - return - - if not self.upstart_dir: - return - - filename = util.clean_filename(filename) - (_name, ext) = os.path.splitext(filename) - if not ext: - ext = '' - ext = ext.lower() - if ext != ".conf": - filename = filename + ".conf" - - payload = util.dos2unix(payload) - path = os.path.join(self.upstart_dir, filename) - util.write_file(path, payload, 0o644) - - if SUITABLE_UPSTART: - util.subp(["initctl", "reload-configuration"], capture=False) - - -def _has_suitable_upstart(): - # (LP: #1124384) - # a bug in upstart means that invoking reload-configuration - # at this stage in boot causes havoc. So, try to determine if upstart - # is installed, and reloading configuration is OK. - if not os.path.exists("/sbin/initctl"): - return False - try: - (version_out, _err) = util.subp(["initctl", "version"]) - except Exception: - util.logexc(LOG, "initctl version failed") - return False - - # expecting 'initctl version' to output something like: init (upstart X.Y) - if re.match("upstart 1.[0-7][)]", version_out): - return False - if "upstart 0." in version_out: - return False - elif "upstart 1.8" in version_out: - if not os.path.exists("/usr/bin/dpkg-query"): - return False - try: - (dpkg_ver, _err) = util.subp(["dpkg-query", - "--showformat=${Version}", - "--show", "upstart"], rcs=[0, 1]) - except Exception: - util.logexc(LOG, "dpkg-query failed") - return False - - try: - good = "1.8-0ubuntu1.2" - util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) - return True - except util.ProcessExecutionError as e: - if e.exit_code is 1: - pass - else: - util.logexc(LOG, "dpkg --compare-versions failed [%s]", - e.exit_code) - except Exception as e: - util.logexc(LOG, "dpkg --compare-versions failed") - return False - else: - return True - -SUITABLE_UPSTART = _has_suitable_upstart() diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py deleted file mode 100644 index fb95babc..00000000 --- a/cloudinit/helpers.py +++ /dev/null @@ -1,460 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from time import time - -import contextlib -import os - -import six -from six.moves.configparser import ( - NoSectionError, NoOptionError, RawConfigParser) - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, - CFG_ENV_NAME) - -from cloudinit import log as logging -from cloudinit import type_utils -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -class LockFailure(Exception): - pass - - -class DummyLock(object): - pass - - -class DummySemaphores(object): - def __init__(self): - pass - - @contextlib.contextmanager - def lock(self, _name, _freq, _clear_on_fail=False): - yield DummyLock() - - def has_run(self, _name, _freq): - return False - - def clear(self, _name, _freq): - return True - - def clear_all(self): - pass - - -class FileLock(object): - def __init__(self, fn): - self.fn = fn - - def __str__(self): - return "<%s using file %r>" % (type_utils.obj_name(self), self.fn) - - -def canon_sem_name(name): - return name.replace("-", "_") - - -class FileSemaphores(object): - def __init__(self, sem_path): - self.sem_path = sem_path - - @contextlib.contextmanager - def lock(self, name, freq, clear_on_fail=False): - name = canon_sem_name(name) - try: - yield self._acquire(name, freq) - except Exception: - if clear_on_fail: - self.clear(name, freq) - raise - - def clear(self, name, freq): - name = canon_sem_name(name) - sem_file = self._get_path(name, freq) - try: - util.del_file(sem_file) - except (IOError, OSError): - util.logexc(LOG, "Failed deleting semaphore %s", sem_file) - return False - return True - - def clear_all(self): - try: - util.del_dir(self.sem_path) - except (IOError, OSError): - util.logexc(LOG, "Failed deleting semaphore directory %s", - self.sem_path) - - def _acquire(self, name, freq): - # Check again if its been already gotten - if self.has_run(name, freq): - return None - # This is a race condition since nothing atomic is happening - # here, but this should be ok due to the nature of when - # and where cloud-init runs... (file writing is not a lock...) - sem_file = self._get_path(name, freq) - contents = "%s: %s\n" % (os.getpid(), time()) - try: - util.write_file(sem_file, contents) - except (IOError, OSError): - util.logexc(LOG, "Failed writing semaphore file %s", sem_file) - return None - return FileLock(sem_file) - - def has_run(self, name, freq): - if not freq or freq == PER_ALWAYS: - return False - - cname = canon_sem_name(name) - sem_file = self._get_path(cname, freq) - # This isn't really a good atomic check - # but it suffices for where and when cloudinit runs - if os.path.exists(sem_file): - return True - - # this case could happen if the migrator module hadn't run yet - # but the item had run before we did canon_sem_name. - if cname != name and os.path.exists(self._get_path(name, freq)): - LOG.warn("%s has run without canonicalized name [%s].\n" - "likely the migrator has not yet run. " - "It will run next boot.\n" - "run manually with: cloud-init single --name=migrator" - % (name, cname)) - return True - - return False - - def _get_path(self, name, freq): - sem_path = self.sem_path - if not freq or freq == PER_INSTANCE: - return os.path.join(sem_path, name) - else: - return os.path.join(sem_path, "%s.%s" % (name, freq)) - - -class Runners(object): - def __init__(self, paths): - self.paths = paths - self.sems = {} - - def _get_sem(self, freq): - if freq == PER_ALWAYS or not freq: - return None - sem_path = None - if freq == PER_INSTANCE: - # This may not exist, - # so thats why we still check for none - # below if say the paths object - # doesn't have a datasource that can - # provide this instance path... - sem_path = self.paths.get_ipath("sem") - elif freq == PER_ONCE: - sem_path = self.paths.get_cpath("sem") - if not sem_path: - return None - if sem_path not in self.sems: - self.sems[sem_path] = FileSemaphores(sem_path) - return self.sems[sem_path] - - def run(self, name, functor, args, freq=None, clear_on_fail=False): - sem = self._get_sem(freq) - if not sem: - sem = DummySemaphores() - if not args: - args = [] - if sem.has_run(name, freq): - LOG.debug("%s already ran (freq=%s)", name, freq) - return (False, None) - with sem.lock(name, freq, clear_on_fail) as lk: - if not lk: - raise LockFailure("Failed to acquire lock for %s" % name) - else: - LOG.debug("Running %s using lock (%s)", name, lk) - if isinstance(args, (dict)): - results = functor(**args) - else: - results = functor(*args) - return (True, results) - - -class ConfigMerger(object): - def __init__(self, paths=None, datasource=None, - additional_fns=None, base_cfg=None, - include_vendor=True): - self._paths = paths - self._ds = datasource - self._fns = additional_fns - self._base_cfg = base_cfg - self._include_vendor = include_vendor - # Created on first use - self._cfg = None - - def _get_datasource_configs(self): - d_cfgs = [] - if self._ds: - try: - ds_cfg = self._ds.get_config_obj() - if ds_cfg and isinstance(ds_cfg, (dict)): - d_cfgs.append(ds_cfg) - except Exception: - util.logexc(LOG, "Failed loading of datasource config object " - "from %s", self._ds) - return d_cfgs - - def _get_env_configs(self): - e_cfgs = [] - if CFG_ENV_NAME in os.environ: - e_fn = os.environ[CFG_ENV_NAME] - try: - e_cfgs.append(util.read_conf(e_fn)) - except Exception: - util.logexc(LOG, 'Failed loading of env. config from %s', - e_fn) - return e_cfgs - - def _get_instance_configs(self): - i_cfgs = [] - # If cloud-config was written, pick it up as - # a configuration file to use when running... - if not self._paths: - return i_cfgs - - cc_paths = ['cloud_config'] - if self._include_vendor: - cc_paths.append('vendor_cloud_config') - - for cc_p in cc_paths: - cc_fn = self._paths.get_ipath_cur(cc_p) - if cc_fn and os.path.isfile(cc_fn): - try: - i_cfgs.append(util.read_conf(cc_fn)) - except Exception: - util.logexc(LOG, 'Failed loading of cloud-config from %s', - cc_fn) - return i_cfgs - - def _read_cfg(self): - # Input config files override - # env config files which - # override instance configs - # which override datasource - # configs which override - # base configuration - cfgs = [] - if self._fns: - for c_fn in self._fns: - try: - cfgs.append(util.read_conf(c_fn)) - except Exception: - util.logexc(LOG, "Failed loading of configuration from %s", - c_fn) - - cfgs.extend(self._get_env_configs()) - cfgs.extend(self._get_instance_configs()) - cfgs.extend(self._get_datasource_configs()) - if self._base_cfg: - cfgs.append(self._base_cfg) - return util.mergemanydict(cfgs) - - @property - def cfg(self): - # None check to avoid empty case causing re-reading - if self._cfg is None: - self._cfg = self._read_cfg() - return self._cfg - - -class ContentHandlers(object): - - def __init__(self): - self.registered = {} - self.initialized = [] - - def __contains__(self, item): - return self.is_registered(item) - - def __getitem__(self, key): - return self._get_handler(key) - - def is_registered(self, content_type): - return content_type in self.registered - - def register(self, mod, initialized=False, overwrite=True): - types = set() - for t in mod.list_types(): - if overwrite: - types.add(t) - else: - if not self.is_registered(t): - types.add(t) - for t in types: - self.registered[t] = mod - if initialized and mod not in self.initialized: - self.initialized.append(mod) - return types - - def _get_handler(self, content_type): - return self.registered[content_type] - - def items(self): - return list(self.registered.items()) - - -class Paths(object): - def __init__(self, path_cfgs, ds=None): - self.cfgs = path_cfgs - # Populate all the initial paths - self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') - self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') - self.instance_link = os.path.join(self.cloud_dir, 'instance') - self.boot_finished = os.path.join(self.instance_link, "boot-finished") - self.upstart_conf_d = path_cfgs.get('upstart_dir') - self.seed_dir = os.path.join(self.cloud_dir, 'seed') - # This one isn't joined, since it should just be read-only - template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/') - self.template_tpl = os.path.join(template_dir, '%s.tmpl') - self.lookups = { - "handlers": "handlers", - "scripts": "scripts", - "vendor_scripts": "scripts/vendor", - "sem": "sem", - "boothooks": "boothooks", - "userdata_raw": "user-data.txt", - "userdata": "user-data.txt.i", - "obj_pkl": "obj.pkl", - "cloud_config": "cloud-config.txt", - "vendor_cloud_config": "vendor-cloud-config.txt", - "data": "data", - "vendordata_raw": "vendor-data.txt", - "vendordata": "vendor-data.txt.i", - "instance_id": ".instance-id", - } - # Set when a datasource becomes active - self.datasource = ds - - # get_ipath_cur: get the current instance path for an item - def get_ipath_cur(self, name=None): - return self._get_path(self.instance_link, name) - - # get_cpath : get the "clouddir" (/var/lib/cloud/) - # for a name in dirmap - def get_cpath(self, name=None): - return self._get_path(self.cloud_dir, name) - - # _get_ipath : get the instance path for a name in pathmap - # (/var/lib/cloud/instances//) - def _get_ipath(self, name=None): - if not self.datasource: - return None - iid = self.datasource.get_instance_id() - if iid is None: - return None - path_safe_iid = str(iid).replace(os.sep, '_') - ipath = os.path.join(self.cloud_dir, 'instances', path_safe_iid) - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath - - # get_ipath : get the instance path for a name in pathmap - # (/var/lib/cloud/instances//) - # returns None + warns if no active datasource.... - def get_ipath(self, name=None): - ipath = self._get_ipath(name) - if not ipath: - LOG.warn(("No per instance data available, " - "is there an datasource/iid set?")) - return None - else: - return ipath - - def _get_path(self, base, name=None): - if name is None: - return base - return os.path.join(base, self.lookups[name]) - - def get_runpath(self, name=None): - return self._get_path(self.run_dir, name) - - -# This config parser will not throw when sections don't exist -# and you are setting values on those sections which is useful -# when writing to new options that may not have corresponding -# sections. Also it can default other values when doing gets -# so that if those sections/options do not exist you will -# get a default instead of an error. Another useful case where -# you can avoid catching exceptions that you typically don't -# care about... - -class DefaultingConfigParser(RawConfigParser): - DEF_INT = 0 - DEF_FLOAT = 0.0 - DEF_BOOLEAN = False - DEF_BASE = None - - def get(self, section, option): - value = self.DEF_BASE - try: - value = RawConfigParser.get(self, section, option) - except NoSectionError: - pass - except NoOptionError: - pass - return value - - def set(self, section, option, value=None): - if not self.has_section(section) and section.lower() != 'default': - self.add_section(section) - RawConfigParser.set(self, section, option, value) - - def remove_option(self, section, option): - if self.has_option(section, option): - RawConfigParser.remove_option(self, section, option) - - def getboolean(self, section, option): - if not self.has_option(section, option): - return self.DEF_BOOLEAN - return RawConfigParser.getboolean(self, section, option) - - def getfloat(self, section, option): - if not self.has_option(section, option): - return self.DEF_FLOAT - return RawConfigParser.getfloat(self, section, option) - - def getint(self, section, option): - if not self.has_option(section, option): - return self.DEF_INT - return RawConfigParser.getint(self, section, option) - - def stringify(self, header=None): - contents = '' - with six.StringIO() as outputstream: - self.write(outputstream) - outputstream.flush() - contents = outputstream.getvalue() - if header: - contents = "\n".join([header, contents]) - return contents diff --git a/cloudinit/importer.py b/cloudinit/importer.py deleted file mode 100644 index fb57253c..00000000 --- a/cloudinit/importer.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys - - -def import_module(module_name): - __import__(module_name) - return sys.modules[module_name] - - -def find_module(base_name, search_paths, required_attrs=None): - if not required_attrs: - required_attrs = [] - # NOTE(harlowja): translate the search paths to include the base name. - lookup_paths = [] - for path in search_paths: - real_path = [] - if path: - real_path.extend(path.split(".")) - real_path.append(base_name) - full_path = '.'.join(real_path) - lookup_paths.append(full_path) - found_paths = [] - for full_path in lookup_paths: - mod = None - try: - mod = import_module(full_path) - except ImportError: - pass - if not mod: - continue - found_attrs = 0 - for attr in required_attrs: - if hasattr(mod, attr): - found_attrs += 1 - if found_attrs == len(required_attrs): - found_paths.append(full_path) - return (found_paths, lookup_paths) diff --git a/cloudinit/log.py b/cloudinit/log.py deleted file mode 100644 index 3c79b9c9..00000000 --- a/cloudinit/log.py +++ /dev/null @@ -1,155 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -import logging.config -import logging.handlers - -import collections -import os -import sys - -import six -from six import StringIO - -# Logging levels for easy access -CRITICAL = logging.CRITICAL -FATAL = logging.FATAL -ERROR = logging.ERROR -WARNING = logging.WARNING -WARN = logging.WARN -INFO = logging.INFO -DEBUG = logging.DEBUG -NOTSET = logging.NOTSET - -# Default basic format -DEF_CON_FORMAT = '%(asctime)s - %(filename)s[%(levelname)s]: %(message)s' - - -def setupBasicLogging(level=DEBUG): - root = logging.getLogger() - console = logging.StreamHandler(sys.stderr) - console.setFormatter(logging.Formatter(DEF_CON_FORMAT)) - console.setLevel(level) - root.addHandler(console) - root.setLevel(level) - - -def flushLoggers(root): - if not root: - return - for h in root.handlers: - if isinstance(h, (logging.StreamHandler)): - try: - h.flush() - except IOError: - pass - flushLoggers(root.parent) - - -def setupLogging(cfg=None): - # See if the config provides any logging conf... - if not cfg: - cfg = {} - - log_cfgs = [] - log_cfg = cfg.get('logcfg') - if log_cfg and isinstance(log_cfg, six.string_types): - # If there is a 'logcfg' entry in the config, - # respect it, it is the old keyname - log_cfgs.append(str(log_cfg)) - elif "log_cfgs" in cfg: - for a_cfg in cfg['log_cfgs']: - if isinstance(a_cfg, six.string_types): - log_cfgs.append(a_cfg) - elif isinstance(a_cfg, (collections.Iterable)): - cfg_str = [str(c) for c in a_cfg] - log_cfgs.append('\n'.join(cfg_str)) - else: - log_cfgs.append(str(a_cfg)) - - # See if any of them actually load... - am_tried = 0 - for log_cfg in log_cfgs: - try: - am_tried += 1 - # Assume its just a string if not a filename - if log_cfg.startswith("/") and os.path.isfile(log_cfg): - # Leave it as a file and do not make it look like - # something that is a file (but is really a buffer that - # is acting as a file) - pass - else: - log_cfg = StringIO(log_cfg) - # Attempt to load its config - logging.config.fileConfig(log_cfg) - # The first one to work wins! - return - except Exception: - # We do not write any logs of this here, because the default - # configuration includes an attempt at using /dev/log, followed - # up by writing to a file. /dev/log will not exist in very early - # boot, so an exception on that is expected. - pass - - # If it didn't work, at least setup a basic logger (if desired) - basic_enabled = cfg.get('log_basic', True) - - sys.stderr.write(("WARN: no logging configured!" - " (tried %s configs)\n") % (am_tried)) - if basic_enabled: - sys.stderr.write("Setting up basic logging...\n") - setupBasicLogging() - - -def getLogger(name='cloudinit'): - return logging.getLogger(name) - - -# Fixes this annoyance... -# No handlers could be found for logger XXX annoying output... -try: - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass - - -def _resetLogger(log): - if not log: - return - handlers = list(log.handlers) - for h in handlers: - h.flush() - h.close() - log.removeHandler(h) - log.setLevel(NOTSET) - log.addHandler(NullHandler()) - - -def resetLogging(): - _resetLogger(logging.getLogger()) - _resetLogger(getLogger()) - - -resetLogging() diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py deleted file mode 100644 index e13f55ac..00000000 --- a/cloudinit/mergers/__init__.py +++ /dev/null @@ -1,166 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import re - -import six - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import type_utils - -NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") - -LOG = logging.getLogger(__name__) -DEF_MERGE_TYPE = "list()+dict()+str()" -MERGER_PREFIX = 'm_' -MERGER_ATTR = 'Merger' - - -class UnknownMerger(object): - # Named differently so auto-method finding - # doesn't pick this up if there is ever a type - # named "unknown" - def _handle_unknown(self, _meth_wanted, value, _merge_with): - return value - - # This merging will attempt to look for a '_on_X' method - # in our own object for a given object Y with type X, - # if found it will be called to perform the merge of a source - # object and a object to merge_with. - # - # If not found the merge will be given to a '_handle_unknown' - # function which can decide what to do wit the 2 values. - def merge(self, source, merge_with): - type_name = type_utils.obj_name(source) - type_name = type_name.lower() - method_name = "_on_%s" % (type_name) - meth = None - args = [source, merge_with] - if hasattr(self, method_name): - meth = getattr(self, method_name) - if not meth: - meth = self._handle_unknown - args.insert(0, method_name) - return meth(*args) - - -class LookupMerger(UnknownMerger): - def __init__(self, lookups=None): - UnknownMerger.__init__(self) - if lookups is None: - self._lookups = [] - else: - self._lookups = lookups - - def __str__(self): - return 'LookupMerger: (%s)' % (len(self._lookups)) - - # For items which can not be merged by the parent this object - # will lookup in a internally maintained set of objects and - # find which one of those objects can perform the merge. If - # any of the contained objects have the needed method, they - # will be called to perform the merge. - def _handle_unknown(self, meth_wanted, value, merge_with): - meth = None - for merger in self._lookups: - if hasattr(merger, meth_wanted): - # First one that has that method/attr gets to be - # the one that will be called - meth = getattr(merger, meth_wanted) - break - if not meth: - return UnknownMerger._handle_unknown(self, meth_wanted, - value, merge_with) - return meth(value, merge_with) - - -def dict_extract_mergers(config): - parsed_mergers = [] - raw_mergers = config.pop('merge_how', None) - if raw_mergers is None: - raw_mergers = config.pop('merge_type', None) - if raw_mergers is None: - return parsed_mergers - if isinstance(raw_mergers, six.string_types): - return string_extract_mergers(raw_mergers) - for m in raw_mergers: - if isinstance(m, (dict)): - name = m['name'] - name = name.replace("-", "_").strip() - opts = m['settings'] - else: - name = m[0] - if len(m) >= 2: - opts = m[1:] - else: - opts = [] - if name: - parsed_mergers.append((name, opts)) - return parsed_mergers - - -def string_extract_mergers(merge_how): - parsed_mergers = [] - for m_name in merge_how.split("+"): - # Canonicalize the name (so that it can be found - # even when users alter it in various ways) - m_name = m_name.lower().strip() - m_name = m_name.replace("-", "_") - if not m_name: - continue - match = NAME_MTCH.match(m_name) - if not match: - msg = ("Matcher identifer '%s' is not in the right format" % - (m_name)) - raise ValueError(msg) - (m_name, m_ops) = match.groups() - m_ops = m_ops.strip().split(",") - m_ops = [m.strip().lower() for m in m_ops if m.strip()] - parsed_mergers.append((m_name, m_ops)) - return parsed_mergers - - -def default_mergers(): - return tuple(string_extract_mergers(DEF_MERGE_TYPE)) - - -def construct(parsed_mergers): - mergers_to_be = [] - for (m_name, m_ops) in parsed_mergers: - if not m_name.startswith(MERGER_PREFIX): - m_name = MERGER_PREFIX + str(m_name) - merger_locs, looked_locs = importer.find_module(m_name, - [__name__], - [MERGER_ATTR]) - if not merger_locs: - msg = ("Could not find merger module named '%s' " - "with attribute '%s' (searched %s)") % (m_name, - MERGER_ATTR, - looked_locs) - raise ImportError(msg) - else: - mod = importer.import_module(merger_locs[0]) - mod_attr = getattr(mod, MERGER_ATTR) - mergers_to_be.append((mod_attr, m_ops)) - # Now form them... - mergers = [] - root = LookupMerger(mergers) - for (attr, opts) in mergers_to_be: - mergers.append(attr(root, opts)) - return root diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py deleted file mode 100644 index 87cf1a72..00000000 --- a/cloudinit/mergers/m_dict.py +++ /dev/null @@ -1,88 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import six - -DEF_MERGE_TYPE = 'no_replace' -MERGE_TYPES = ('replace', DEF_MERGE_TYPE,) - - -def _has_any(what, *keys): - for k in keys: - if k in what: - return True - return False - - -class Merger(object): - def __init__(self, merger, opts): - self._merger = merger - # Affects merging behavior... - self._method = DEF_MERGE_TYPE - for m in MERGE_TYPES: - if m in opts: - self._method = m - break - # Affect how recursive merging is done on other primitives. - self._recurse_str = 'recurse_str' in opts - self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') - self._allow_delete = 'allow_delete' in opts - # Backwards compat require this to be on. - self._recurse_dict = True - - def __str__(self): - s = ('DictMerger: (method=%s,recurse_str=%s,' - 'recurse_dict=%s,recurse_array=%s,allow_delete=%s)') - s = s % (self._method, self._recurse_str, - self._recurse_dict, self._recurse_array, self._allow_delete) - return s - - def _do_dict_replace(self, value, merge_with, do_replace): - - def merge_same_key(old_v, new_v): - if do_replace: - return new_v - if isinstance(new_v, (list, tuple)) and self._recurse_array: - return self._merger.merge(old_v, new_v) - if isinstance(new_v, six.string_types) and self._recurse_str: - return self._merger.merge(old_v, new_v) - if isinstance(new_v, (dict)) and self._recurse_dict: - return self._merger.merge(old_v, new_v) - # Otherwise leave it be... - return old_v - - for (k, v) in merge_with.items(): - if k in value: - if v is None and self._allow_delete: - value.pop(k) - else: - value[k] = merge_same_key(value[k], v) - else: - value[k] = v - return value - - def _on_dict(self, value, merge_with): - if not isinstance(merge_with, (dict)): - return value - if self._method == 'replace': - merged = self._do_dict_replace(dict(value), merge_with, True) - elif self._method == 'no_replace': - merged = self._do_dict_replace(dict(value), merge_with, False) - else: - raise NotImplementedError("Unknown merge type %s" % (self._method)) - return merged diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py deleted file mode 100644 index 81e5c580..00000000 --- a/cloudinit/mergers/m_list.py +++ /dev/null @@ -1,89 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import six - -DEF_MERGE_TYPE = 'replace' -MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE, 'no_replace') - - -def _has_any(what, *keys): - for k in keys: - if k in what: - return True - return False - - -class Merger(object): - def __init__(self, merger, opts): - self._merger = merger - # Affects merging behavior... - self._method = DEF_MERGE_TYPE - for m in MERGE_TYPES: - if m in opts: - self._method = m - break - # Affect how recursive merging is done on other primitives - self._recurse_str = _has_any(opts, 'recurse_str') - self._recurse_dict = _has_any(opts, 'recurse_dict') - self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') - - def __str__(self): - return ('ListMerger: (method=%s,recurse_str=%s,' - 'recurse_dict=%s,recurse_array=%s)') % (self._method, - self._recurse_str, - self._recurse_dict, - self._recurse_array) - - def _on_tuple(self, value, merge_with): - return tuple(self._on_list(list(value), merge_with)) - - def _on_list(self, value, merge_with): - if (self._method == 'replace' and - not isinstance(merge_with, (tuple, list))): - return merge_with - - # Ok we now know that what we are merging with is a list or tuple. - merged_list = [] - if self._method == 'prepend': - merged_list.extend(merge_with) - merged_list.extend(value) - return merged_list - elif self._method == 'append': - merged_list.extend(value) - merged_list.extend(merge_with) - return merged_list - - def merge_same_index(old_v, new_v): - if self._method == 'no_replace': - # Leave it be... - return old_v - if isinstance(new_v, (list, tuple)) and self._recurse_array: - return self._merger.merge(old_v, new_v) - if isinstance(new_v, six.string_types) and self._recurse_str: - return self._merger.merge(old_v, new_v) - if isinstance(new_v, (dict)) and self._recurse_dict: - return self._merger.merge(old_v, new_v) - return new_v - - # Ok now we are replacing same indexes - merged_list.extend(value) - common_len = min(len(merged_list), len(merge_with)) - for i in range(0, common_len): - merged_list[i] = merge_same_index(merged_list[i], merge_with[i]) - return merged_list diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py deleted file mode 100644 index b00c4bf3..00000000 --- a/cloudinit/mergers/m_str.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import six - - -class Merger(object): - def __init__(self, _merger, opts): - self._append = 'append' in opts - - def __str__(self): - return 'StringMerger: (append=%s)' % (self._append) - - # On encountering a unicode object to merge value with - # we will for now just proxy into the string method to let it handle it. - def _on_unicode(self, value, merge_with): - return self._on_str(value, merge_with) - - # On encountering a string object to merge with we will - # perform the following action, if appending we will - # merge them together, otherwise we will just return value. - def _on_str(self, value, merge_with): - if not isinstance(value, six.string_types): - return merge_with - if not self._append: - return merge_with - if isinstance(value, six.text_type): - return value + six.text_type(merge_with) - else: - return value + six.binary_type(merge_with) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py deleted file mode 100644 index 21cc602b..00000000 --- a/cloudinit/net/__init__.py +++ /dev/null @@ -1,371 +0,0 @@ -# Copyright (C) 2013-2014 Canonical Ltd. -# -# Author: Scott Moser -# Author: Blake Rouse -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -import errno -import logging -import os -import re - -from cloudinit import util - -LOG = logging.getLogger(__name__) -SYS_CLASS_NET = "/sys/class/net/" -DEFAULT_PRIMARY_INTERFACE = 'eth0' - - -def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path - - -def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): - try: - contents = util.load_file(sys_dev_path(devname, path)) - except (OSError, IOError) as e: - if getattr(e, 'errno', None) == errno.ENOENT: - if enoent is not None: - return enoent - raise - contents = contents.strip() - if translate is None: - return contents - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise - - -def is_up(devname): - # The linux kernel says to consider devices in 'unknown' - # operstate as up for the purposes of network configuration. See - # Documentation/networking/operstates.txt in the kernel source. - translate = {'up': True, 'unknown': True, 'down': False} - return read_sys_net(devname, "operstate", enoent=False, keyerror=False, - translate=translate) - - -def is_wireless(devname): - return os.path.exists(sys_dev_path(devname, "wireless")) - - -def is_connected(devname): - # is_connected isn't really as simple as that. 2 is - # 'physically connected'. 3 is 'not connected'. but a wlan interface will - # always show 3. - try: - iflink = read_sys_net(devname, "iflink", enoent=False) - if iflink == "2": - return True - if not is_wireless(devname): - return False - LOG.debug("'%s' is wireless, basing 'connected' on carrier", devname) - - return read_sys_net(devname, "carrier", enoent=False, keyerror=False, - translate={'0': False, '1': True}) - - except IOError as e: - if e.errno == errno.EINVAL: - return False - raise - - -def is_physical(devname): - return os.path.exists(sys_dev_path(devname, "device")) - - -def is_present(devname): - return os.path.exists(sys_dev_path(devname)) - - -def get_devicelist(): - return os.listdir(SYS_CLASS_NET) - - -class ParserError(Exception): - """Raised when a parser has issue parsing a file/content.""" - - -def is_disabled_cfg(cfg): - if not cfg or not isinstance(cfg, dict): - return False - return cfg.get('config') == "disabled" - - -def sys_netdev_info(name, field): - if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): - raise OSError("%s: interface does not exist in %s" % - (name, SYS_CLASS_NET)) - fname = os.path.join(SYS_CLASS_NET, name, field) - if not os.path.exists(fname): - raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) - data = util.load_file(fname) - if data[-1] == '\n': - data = data[:-1] - return data - - -def generate_fallback_config(): - """Determine which attached net dev is most likely to have a connection and - generate network state to run dhcp on that interface""" - # by default use eth0 as primary interface - nconf = {'config': [], 'version': 1} - - # get list of interfaces that could have connections - invalid_interfaces = set(['lo']) - potential_interfaces = set(get_devicelist()) - potential_interfaces = potential_interfaces.difference(invalid_interfaces) - # sort into interfaces with carrier, interfaces which could have carrier, - # and ignore interfaces that are definitely disconnected - connected = [] - possibly_connected = [] - for interface in potential_interfaces: - if interface.startswith("veth"): - continue - if os.path.exists(sys_dev_path(interface, "bridge")): - # skip any bridges - continue - try: - carrier = int(sys_netdev_info(interface, 'carrier')) - if carrier: - connected.append(interface) - continue - except OSError: - pass - # check if nic is dormant or down, as this may make a nick appear to - # not have a carrier even though it could acquire one when brought - # online by dhclient - try: - dormant = int(sys_netdev_info(interface, 'dormant')) - if dormant: - possibly_connected.append(interface) - continue - except OSError: - pass - try: - operstate = sys_netdev_info(interface, 'operstate') - if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: - possibly_connected.append(interface) - continue - except OSError: - pass - - # don't bother with interfaces that might not be connected if there are - # some that definitely are - if connected: - potential_interfaces = connected - else: - potential_interfaces = possibly_connected - # if there are no interfaces, give up - if not potential_interfaces: - return - # if eth0 exists use it above anything else, otherwise get the interface - # that looks 'first' - if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: - name = DEFAULT_PRIMARY_INTERFACE - else: - name = sorted(potential_interfaces)[0] - - mac = sys_netdev_info(name, 'address') - target_name = name - - nconf['config'].append( - {'type': 'physical', 'name': target_name, - 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]}) - return nconf - - -def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): - """read the network config and rename devices accordingly. - if strict_present is false, then do not raise exception if no devices - match. if strict_busy is false, then do not raise exception if the - device cannot be renamed because it is currently configured.""" - renames = [] - for ent in netcfg.get('config', {}): - if ent.get('type') != 'physical': - continue - mac = ent.get('mac_address') - name = ent.get('name') - if not mac: - continue - renames.append([mac, name]) - - return _rename_interfaces(renames) - - -def _get_current_rename_info(check_downable=True): - """Collect information necessary for rename_interfaces.""" - names = get_devicelist() - bymac = {} - for n in names: - bymac[get_interface_mac(n)] = { - 'name': n, 'up': is_up(n), 'downable': None} - - if check_downable: - nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") - ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', - 'scope', 'global'], capture=True) - ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) - - nics_with_addresses = set() - for bytes_out in (ipv6, ipv4): - nics_with_addresses.update(nmatch.findall(bytes_out)) - - for d in bymac.values(): - d['downable'] = (d['up'] is False or - d['name'] not in nics_with_addresses) - - return bymac - - -def _rename_interfaces(renames, strict_present=True, strict_busy=True, - current_info=None): - - if not len(renames): - LOG.debug("no interfaces to rename") - return - - if current_info is None: - current_info = _get_current_rename_info() - - cur_bymac = {} - for mac, data in current_info.items(): - cur = data.copy() - cur['mac'] = mac - cur_bymac[mac] = cur - - def update_byname(bymac): - return dict((data['name'], data) - for data in bymac.values()) - - def rename(cur, new): - util.subp(["ip", "link", "set", cur, "name", new], capture=True) - - def down(name): - util.subp(["ip", "link", "set", name, "down"], capture=True) - - def up(name): - util.subp(["ip", "link", "set", name, "up"], capture=True) - - ops = [] - errors = [] - ups = [] - cur_byname = update_byname(cur_bymac) - tmpname_fmt = "cirename%d" - tmpi = -1 - - for mac, new_name in renames: - cur = cur_bymac.get(mac, {}) - cur_name = cur.get('name') - cur_ops = [] - if cur_name == new_name: - # nothing to do - continue - - if not cur_name: - if strict_present: - errors.append( - "[nic not present] Cannot rename mac=%s to %s" - ", not available." % (mac, new_name)) - continue - - if cur['up']: - msg = "[busy] Error renaming mac=%s from %s to %s" - if not cur['downable']: - if strict_busy: - errors.append(msg % (mac, cur_name, new_name)) - continue - cur['up'] = False - cur_ops.append(("down", mac, new_name, (cur_name,))) - ups.append(("up", mac, new_name, (new_name,))) - - if new_name in cur_byname: - target = cur_byname[new_name] - if target['up']: - msg = "[busy-target] Error renaming mac=%s from %s to %s." - if not target['downable']: - if strict_busy: - errors.append(msg % (mac, cur_name, new_name)) - continue - else: - cur_ops.append(("down", mac, new_name, (new_name,))) - - tmp_name = None - while tmp_name is None or tmp_name in cur_byname: - tmpi += 1 - tmp_name = tmpname_fmt % tmpi - - cur_ops.append(("rename", mac, new_name, (new_name, tmp_name))) - target['name'] = tmp_name - cur_byname = update_byname(cur_bymac) - if target['up']: - ups.append(("up", mac, new_name, (tmp_name,))) - - cur_ops.append(("rename", mac, new_name, (cur['name'], new_name))) - cur['name'] = new_name - cur_byname = update_byname(cur_bymac) - ops += cur_ops - - opmap = {'rename': rename, 'down': down, 'up': up} - - if len(ops) + len(ups) == 0: - if len(errors): - LOG.debug("unable to do any work for renaming of %s", renames) - else: - LOG.debug("no work necessary for renaming of %s", renames) - else: - LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) - - for op, mac, new_name, params in ops + ups: - try: - opmap.get(op)(*params) - except Exception as e: - errors.append( - "[unknown] Error performing %s%s for %s, %s: %s" % - (op, params, mac, new_name, e)) - - if len(errors): - raise Exception('\n'.join(errors)) - - -def get_interface_mac(ifname): - """Returns the string value of an interface's MAC Address""" - return read_sys_net(ifname, "address", enoent=False) - - -def get_interfaces_by_mac(devs=None): - """Build a dictionary of tuples {mac: name}""" - if devs is None: - try: - devs = get_devicelist() - except OSError as e: - if e.errno == errno.ENOENT: - devs = [] - else: - raise - ret = {} - for name in devs: - mac = get_interface_mac(name) - # some devices may not have a mac (tun0) - if mac: - ret[mac] = name - return ret - -# vi: ts=4 expandtab syntax=python diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py deleted file mode 100644 index 822a020b..00000000 --- a/cloudinit/net/cmdline.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (C) 2013-2014 Canonical Ltd. -# -# Author: Scott Moser -# Author: Blake Rouse -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -import base64 -import glob -import gzip -import io -import shlex -import sys - -import six - -from . import get_devicelist -from . import sys_netdev_info - -from cloudinit import util - -PY26 = sys.version_info[0:2] == (2, 6) - - -def _shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in _shlex_split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - - -def _klibc_to_config_entry(content, mac_addrs=None): - """Convert a klibc writtent shell content file to a 'config' entry - When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate - /run/net-.cfg. - - The files are shell style syntax, and examples are in the tests - provided here. There is no good documentation on this unfortunately. - - DEVICE= is expected/required and PROTO should indicate if - this is 'static' or 'dhcp'. - """ - - if mac_addrs is None: - mac_addrs = {} - - data = _load_shell_content(content) - try: - name = data['DEVICE'] - except KeyError: - raise ValueError("no 'DEVICE' entry in data") - - # ipconfig on precise does not write PROTO - proto = data.get('PROTO') - if not proto: - if data.get('filename'): - proto = 'dhcp' - else: - proto = 'static' - - if proto not in ('static', 'dhcp'): - raise ValueError("Unexpected value for PROTO: %s" % proto) - - iface = { - 'type': 'physical', - 'name': name, - 'subnets': [], - } - - if name in mac_addrs: - iface['mac_address'] = mac_addrs[name] - - # originally believed there might be IPV6* values - for v, pre in (('ipv4', 'IPV4'),): - # if no IPV4ADDR or IPV6ADDR, then go on. - if pre + "ADDR" not in data: - continue - subnet = {'type': proto, 'control': 'manual'} - - # these fields go right on the subnet - for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): - if pre + key in data: - subnet[key.lower()] = data[pre + key] - - dns = [] - # handle IPV4DNS0 or IPV6DNS0 - for nskey in ('DNS0', 'DNS1'): - ns = data.get(pre + nskey) - # verify it has something other than 0.0.0.0 (or ipv6) - if ns and len(ns.strip(":.0")): - dns.append(data[pre + nskey]) - if dns: - subnet['dns_nameservers'] = dns - # add search to both ipv4 and ipv6, as it has no namespace - search = data.get('DOMAINSEARCH') - if search: - if ',' in search: - subnet['dns_search'] = search.split(",") - else: - subnet['dns_search'] = search.split() - - iface['subnets'].append(subnet) - - return name, iface - - -def config_from_klibc_net_cfg(files=None, mac_addrs=None): - if files is None: - files = glob.glob('/run/net*.conf') - - entries = [] - names = {} - for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), - mac_addrs=mac_addrs) - if name in names: - raise ValueError( - "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file)) - - names[name] = cfg_file - entries.append(entry) - return {'config': entries, 'version': 1} - - -def _decomp_gzip(blob, strict=True): - # decompress blob. raise exception if not compressed unless strict=False. - with io.BytesIO(blob) as iobuf: - gzfp = None - try: - gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) - return gzfp.read() - except IOError: - if strict: - raise - return blob - finally: - if gzfp: - gzfp.close() - - -def _b64dgz(b64str, gzipped="try"): - # decode a base64 string. If gzipped is true, transparently uncompresss - # if gzipped is 'try', then try gunzip, returning the original on fail. - try: - blob = base64.b64decode(b64str) - except TypeError: - raise ValueError("Invalid base64 text: %s" % b64str) - - if not gzipped: - return blob - - return _decomp_gzip(blob, strict=gzipped != "try") - - -def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - - if 'network-config=' in cmdline: - data64 = None - for tok in cmdline.split(): - if tok.startswith("network-config="): - data64 = tok.split("=", 1)[1] - if data64: - return util.load_yaml(_b64dgz(data64)) - - if 'ip=' not in cmdline: - return None - - if mac_addrs is None: - mac_addrs = dict((k, sys_netdev_info(k, 'address')) - for k in get_devicelist()) - - return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py deleted file mode 100644 index eff5b924..00000000 --- a/cloudinit/net/eni.py +++ /dev/null @@ -1,504 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import copy -import glob -import os -import re - -from . import ParserError - -from . import renderer - -from cloudinit import util - - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", -] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", -] - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", -] - - -# TODO: switch valid_map based on mode inet/inet6 -def _iface_add_subnet(iface, subnet): - content = [] - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content.append(" {0} {1}".format(key, value)) - - return sorted(content) - - -# TODO: switch to valid_map for attrs -def _iface_add_attrs(iface, index): - # If the index is non-zero, this is an alias interface. Alias interfaces - # represent additional interface addresses, and should not have additional - # attributes. (extra attributes here are almost always either incorrect, - # or are applied to the parent interface.) So if this is an alias, stop - # right here. - if index != 0: - return [] - content = [] - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - renames = {'mac_address': 'hwaddress'} - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if not value or key in ignore_map: - continue - if type(value) == list: - value = " ".join(value) - content.append(" {0} {1}".format(renames.get(key, key), value)) - - return sorted(content) - - -def _iface_start_entry(iface, index, render_hwaddress=False): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - lines = [ - "{cverb} {fullname}".format(**subst), - "iface {fullname} {inet} {mode}".format(**subst)] - if render_hwaddress and iface.get('mac_address'): - lines.append(" hwaddress {mac_address}".format(**subst)) - - return lines - - -def _parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - if split[1] == "ether": - val = split[2] - else: - val = split[1] - ifaces[currif]['hwaddress'] = val - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - _parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - -def convert_eni_data(eni_data): - # return a network config representation of what is in eni_data - ifaces = {} - _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) - return _ifaces_to_net_config_data(ifaces) - - -def _ifaces_to_net_config_data(ifaces): - """Return network config that represents the ifaces data provided. - ifaces = parse_deb_config("/etc/network/interfaces") - config = ifaces_to_net_config_data(ifaces) - state = parse_net_config_data(config).""" - devs = {} - for name, data in ifaces.items(): - # devname is 'eth0' for name='eth0:1' - devname = name.partition(":")[0] - if devname not in devs: - devs[devname] = {'type': 'physical', 'name': devname, - 'subnets': []} - # this isnt strictly correct, but some might specify - # hwaddress on a nic for matching / declaring name. - if 'hwaddress' in data: - devs[devname]['mac_address'] = data['hwaddress'] - subnet = {'_orig_eni_name': name, 'type': data['method']} - if data.get('auto'): - subnet['control'] = 'auto' - else: - subnet['control'] = 'manual' - - if data.get('method') == 'static': - subnet['address'] = data['address'] - - for copy_key in ('netmask', 'gateway', 'broadcast'): - if copy_key in data: - subnet[copy_key] = data[copy_key] - - if 'dns' in data: - for n in ('nameservers', 'search'): - if n in data['dns'] and data['dns'][n]: - subnet['dns_' + n] = data['dns'][n] - devs[devname]['subnets'].append(subnet) - - return {'version': 1, - 'config': [devs[d] for d in sorted(devs)]} - - -class Renderer(renderer.Renderer): - """Renders network information in a /etc/network/interfaces format.""" - - def __init__(self, config=None): - if not config: - config = {} - self.eni_path = config.get('eni_path', 'etc/network/interfaces') - self.eni_header = config.get('eni_header', None) - self.links_path_prefix = config.get( - 'links_path_prefix', 'etc/systemd/network/50-cloud-init-') - self.netrules_path = config.get( - 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') - - def _render_route(self, route, indent=""): - """When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = [] - up = indent + "post-up route add" - down = indent + "pre-down route del" - or_true = " || true" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content.append(up + default_gw + or_true) - content.append(down + default_gw + or_true) - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content.append(up + default_gw + or_true) - content.append(down + default_gw + or_true) - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content.append(up + route_line + or_true) - content.append(down + route_line + or_true) - return content - - def _render_iface(self, iface, render_hwaddress=False): - sections = [] - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - subnet_inet = 'inet' - if iface['mode'].endswith('6'): - # This is a request for DHCPv6. - subnet_inet += '6' - elif iface['mode'] == 'static' and ":" in subnet['address']: - # This is a static IPv6 address. - subnet_inet += '6' - iface['inet'] = subnet_inet - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - lines = list( - _iface_start_entry( - iface, index, render_hwaddress=render_hwaddress) + - _iface_add_subnet(iface, subnet) + - _iface_add_attrs(iface, index) - ) - for route in subnet.get('routes', []): - lines.extend(self._render_route(route, indent=" ")) - - if len(subnets) > 1 and index == 0: - tmpl = " post-up ifup %s:%s\n" - for i in range(1, len(subnets)): - lines.append(tmpl % (iface['name'], i)) - - sections.append(lines) - else: - # ifenslave docs say to auto the slave devices - lines = [] - if 'bond-master' in iface: - lines.append("auto {name}".format(**iface)) - lines.append("iface {name} {inet} {mode}".format(**iface)) - lines.extend(_iface_add_attrs(iface, index=0)) - sections.append(lines) - return sections - - def _render_interfaces(self, network_state, render_hwaddress=False): - '''Given state, emit etc/network/interfaces content.''' - - # handle 'lo' specifically as we need to insert the global dns entries - # there (as that is the only interface that will be always up). - lo = {'name': 'lo', 'type': 'physical', 'inet': 'inet', - 'subnets': [{'type': 'loopback', 'control': 'auto'}]} - for iface in network_state.iter_interfaces(): - if iface.get('name') == "lo": - lo = copy.deepcopy(iface) - - nameservers = network_state.dns_nameservers - if nameservers: - lo['subnets'][0]["dns_nameservers"] = (" ".join(nameservers)) - - searchdomains = network_state.dns_searchdomains - if searchdomains: - lo['subnets'][0]["dns_search"] = (" ".join(searchdomains)) - - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - - sections = [] - sections.extend(self._render_iface(lo)) - for iface in sorted(network_state.iter_interfaces(), - key=lambda k: (order[k['type']], k['name'])): - - if iface.get('name') == "lo": - continue - sections.extend( - self._render_iface(iface, render_hwaddress=render_hwaddress)) - - for route in network_state.iter_routes(): - sections.append(self._render_route(route)) - - return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" - - def render_network_state(self, target, network_state): - fpeni = os.path.join(target, self.eni_path) - util.ensure_dir(os.path.dirname(fpeni)) - header = self.eni_header if self.eni_header else "" - util.write_file(fpeni, header + self._render_interfaces(network_state)) - - if self.netrules_path: - netrules = os.path.join(target, self.netrules_path) - util.ensure_dir(os.path.dirname(netrules)) - util.write_file(netrules, - self._render_persistent_net(network_state)) - - if self.links_path_prefix: - self._render_systemd_links(target, network_state, - links_prefix=self.links_path_prefix) - - def _render_systemd_links(self, target, network_state, links_prefix): - fp_prefix = os.path.join(target, links_prefix) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - for iface in network_state.iter_interfaces(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - content = "\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ]) - util.write_file(fname, content) - - -def network_state_to_eni(network_state, header=None, render_hwaddress=False): - # render the provided network state, return a string of equivalent eni - eni_path = 'etc/network/interfaces' - renderer = Renderer({ - 'eni_path': eni_path, - 'eni_header': header, - 'links_path_prefix': None, - 'netrules_path': None, - }) - if not header: - header = "" - if not header.endswith("\n"): - header += "\n" - contents = renderer._render_interfaces( - network_state, render_hwaddress=render_hwaddress) - return header + contents diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py deleted file mode 100644 index 8ca5106f..00000000 --- a/cloudinit/net/network_state.py +++ /dev/null @@ -1,454 +0,0 @@ -# Copyright (C) 2013-2014 Canonical Ltd. -# -# Author: Ryan Harper -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -import copy -import functools -import logging - -import six - -from cloudinit import util - -LOG = logging.getLogger(__name__) - -NETWORK_STATE_VERSION = 1 -NETWORK_STATE_REQUIRED_KEYS = { - 1: ['version', 'config', 'network_state'], -} - - -def parse_net_config_data(net_config, skip_broken=True): - """Parses the config, returns NetworkState object - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - nsi = NetworkStateInterpreter(version=net_config.get('version'), - config=net_config.get('config')) - nsi.parse_config(skip_broken=skip_broken) - state = nsi.network_state - return state - - -def parse_net_config(path, skip_broken=True): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network'), - skip_broken=skip_broken) - return ns - - -def from_state_file(state_file): - state = util.read_conf(state_file) - nsi = NetworkStateInterpreter() - nsi.load(state) - return nsi - - -def diff_keys(expected, actual): - missing = set(expected) - for key in actual: - missing.discard(key) - return missing - - -class InvalidCommand(Exception): - pass - - -def ensure_command_keys(required_keys): - - def wrapper(func): - - @functools.wraps(func) - def decorator(self, command, *args, **kwargs): - if required_keys: - missing_keys = diff_keys(required_keys, command) - if missing_keys: - raise InvalidCommand("Command missing %s of required" - " keys %s" % (missing_keys, - required_keys)) - return func(self, command, *args, **kwargs) - - return decorator - - return wrapper - - -class CommandHandlerMeta(type): - """Metaclass that dynamically creates a 'command_handlers' attribute. - - This will scan the to-be-created class for methods that start with - 'handle_' and on finding those will populate a class attribute mapping - so that those methods can be quickly located and called. - """ - def __new__(cls, name, parents, dct): - command_handlers = {} - for attr_name, attr in dct.items(): - if callable(attr) and attr_name.startswith('handle_'): - handles_what = attr_name[len('handle_'):] - if handles_what: - command_handlers[handles_what] = attr - dct['command_handlers'] = command_handlers - return super(CommandHandlerMeta, cls).__new__(cls, name, - parents, dct) - - -class NetworkState(object): - - def __init__(self, network_state, version=NETWORK_STATE_VERSION): - self._network_state = copy.deepcopy(network_state) - self._version = version - - @property - def version(self): - return self._version - - def iter_routes(self, filter_func=None): - for route in self._network_state.get('routes', []): - if filter_func is not None: - if filter_func(route): - yield route - else: - yield route - - @property - def dns_nameservers(self): - try: - return self._network_state['dns']['nameservers'] - except KeyError: - return [] - - @property - def dns_searchdomains(self): - try: - return self._network_state['dns']['search'] - except KeyError: - return [] - - def iter_interfaces(self, filter_func=None): - ifaces = self._network_state.get('interfaces', {}) - for iface in six.itervalues(ifaces): - if filter_func is None: - yield iface - else: - if filter_func(iface): - yield iface - - -@six.add_metaclass(CommandHandlerMeta) -class NetworkStateInterpreter(object): - - initial_network_state = { - 'interfaces': {}, - 'routes': [], - 'dns': { - 'nameservers': [], - 'search': [], - } - } - - def __init__(self, version=NETWORK_STATE_VERSION, config=None): - self._version = version - self._config = config - self._network_state = copy.deepcopy(self.initial_network_state) - self._parsed = False - - @property - def network_state(self): - return NetworkState(self._network_state, version=self._version) - - def dump(self): - state = { - 'version': self._version, - 'config': self._config, - 'network_state': self._network_state, - } - return util.yaml_dumps(state) - - def load(self, state): - if 'version' not in state: - LOG.error('Invalid state, missing version field') - raise ValueError('Invalid state, missing version field') - - required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] - missing_keys = diff_keys(required_keys, state) - if missing_keys: - msg = 'Invalid state, missing keys: %s' % (missing_keys) - LOG.error(msg) - raise ValueError(msg) - - # v1 - direct attr mapping, except version - for key in [k for k in required_keys if k not in ['version']]: - setattr(self, key, state[key]) - - def dump_network_state(self): - return util.yaml_dumps(self._network_state) - - def parse_config(self, skip_broken=True): - # rebuild network state - for command in self._config: - command_type = command['type'] - try: - handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) - try: - handler(self, command) - except InvalidCommand: - if not skip_broken: - raise - else: - LOG.warn("Skipping invalid command: %s", command, - exc_info=True) - LOG.debug(self.dump_network_state()) - - @ensure_command_keys(['name']) - def handle_physical(self, command): - ''' - command = { - 'type': 'physical', - 'mac_address': 'c0:d6:9f:2c:e8:80', - 'name': 'eth0', - 'subnets': [ - {'type': 'dhcp4'} - ] - } - ''' - - interfaces = self._network_state.get('interfaces', {}) - iface = interfaces.get(command['name'], {}) - for param, val in command.get('params', {}).items(): - iface.update({param: val}) - - # convert subnet ipv6 netmask to cidr as needed - subnets = command.get('subnets') - if subnets: - for subnet in subnets: - if subnet['type'] == 'static': - if 'netmask' in subnet and ':' in subnet['address']: - subnet['netmask'] = mask2cidr(subnet['netmask']) - for route in subnet.get('routes', []): - if 'netmask' in route: - route['netmask'] = mask2cidr(route['netmask']) - iface.update({ - 'name': command.get('name'), - 'type': command.get('type'), - 'mac_address': command.get('mac_address'), - 'inet': 'inet', - 'mode': 'manual', - 'mtu': command.get('mtu'), - 'address': None, - 'gateway': None, - 'subnets': subnets, - }) - self._network_state['interfaces'].update({command.get('name'): iface}) - self.dump_network_state() - - @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) - def handle_vlan(self, command): - ''' - auto eth0.222 - iface eth0.222 inet static - address 10.10.10.1 - netmask 255.255.255.0 - hwaddress ether BC:76:4E:06:96:B3 - vlan-raw-device eth0 - ''' - interfaces = self._network_state.get('interfaces', {}) - self.handle_physical(command) - iface = interfaces.get(command.get('name'), {}) - iface['vlan-raw-device'] = command.get('vlan_link') - iface['vlan_id'] = command.get('vlan_id') - interfaces.update({iface['name']: iface}) - - @ensure_command_keys(['name', 'bond_interfaces', 'params']) - def handle_bond(self, command): - ''' - #/etc/network/interfaces - auto eth0 - iface eth0 inet manual - bond-master bond0 - bond-mode 802.3ad - - auto eth1 - iface eth1 inet manual - bond-master bond0 - bond-mode 802.3ad - - auto bond0 - iface bond0 inet static - address 192.168.0.10 - gateway 192.168.0.1 - netmask 255.255.255.0 - bond-slaves none - bond-mode 802.3ad - bond-miimon 100 - bond-downdelay 200 - bond-updelay 200 - bond-lacp-rate 4 - ''' - - self.handle_physical(command) - interfaces = self._network_state.get('interfaces') - iface = interfaces.get(command.get('name'), {}) - for param, val in command.get('params').items(): - iface.update({param: val}) - iface.update({'bond-slaves': 'none'}) - self._network_state['interfaces'].update({iface['name']: iface}) - - # handle bond slaves - for ifname in command.get('bond_interfaces'): - if ifname not in interfaces: - cmd = { - 'name': ifname, - 'type': 'bond', - } - # inject placeholder - self.handle_physical(cmd) - - interfaces = self._network_state.get('interfaces', {}) - bond_if = interfaces.get(ifname) - bond_if['bond-master'] = command.get('name') - # copy in bond config into slave - for param, val in command.get('params').items(): - bond_if.update({param: val}) - self._network_state['interfaces'].update({ifname: bond_if}) - - @ensure_command_keys(['name', 'bridge_interfaces', 'params']) - def handle_bridge(self, command): - ''' - auto br0 - iface br0 inet static - address 10.10.10.1 - netmask 255.255.255.0 - bridge_ports eth0 eth1 - bridge_stp off - bridge_fd 0 - bridge_maxwait 0 - - bridge_params = [ - "bridge_ports", - "bridge_ageing", - "bridge_bridgeprio", - "bridge_fd", - "bridge_gcint", - "bridge_hello", - "bridge_hw", - "bridge_maxage", - "bridge_maxwait", - "bridge_pathcost", - "bridge_portprio", - "bridge_stp", - "bridge_waitport", - ] - ''' - - # find one of the bridge port ifaces to get mac_addr - # handle bridge_slaves - interfaces = self._network_state.get('interfaces', {}) - for ifname in command.get('bridge_interfaces'): - if ifname in interfaces: - continue - - cmd = { - 'name': ifname, - } - # inject placeholder - self.handle_physical(cmd) - - interfaces = self._network_state.get('interfaces', {}) - self.handle_physical(command) - iface = interfaces.get(command.get('name'), {}) - iface['bridge_ports'] = command['bridge_interfaces'] - for param, val in command.get('params').items(): - iface.update({param: val}) - - interfaces.update({iface['name']: iface}) - - @ensure_command_keys(['address']) - def handle_nameserver(self, command): - dns = self._network_state.get('dns') - if 'address' in command: - addrs = command['address'] - if not type(addrs) == list: - addrs = [addrs] - for addr in addrs: - dns['nameservers'].append(addr) - if 'search' in command: - paths = command['search'] - if not isinstance(paths, list): - paths = [paths] - for path in paths: - dns['search'].append(path) - - @ensure_command_keys(['destination']) - def handle_route(self, command): - routes = self._network_state.get('routes', []) - network, cidr = command['destination'].split("/") - netmask = cidr2mask(int(cidr)) - route = { - 'network': network, - 'netmask': netmask, - 'gateway': command.get('gateway'), - 'metric': command.get('metric'), - } - routes.append(route) - - -def cidr2mask(cidr): - mask = [0, 0, 0, 0] - for i in list(range(0, cidr)): - idx = int(i / 8) - mask[idx] = mask[idx] + (1 << (7 - i % 8)) - return ".".join([str(x) for x in mask]) - - -def ipv4mask2cidr(mask): - if '.' not in mask: - return mask - return sum([bin(int(x)).count('1') for x in mask.split('.')]) - - -def ipv6mask2cidr(mask): - if ':' not in mask: - return mask - - bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00, - 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc, - 0xfffe, 0xffff] - cidr = 0 - for word in mask.split(':'): - if not word or int(word, 16) == 0: - break - cidr += bitCount.index(int(word, 16)) - - return cidr - - -def mask2cidr(mask): - if ':' in mask: - return ipv6mask2cidr(mask) - elif '.' in mask: - return ipv4mask2cidr(mask) - else: - return mask diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py deleted file mode 100644 index 310cbe0d..00000000 --- a/cloudinit/net/renderer.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 2013-2014 Canonical Ltd. -# -# Author: Scott Moser -# Author: Blake Rouse -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -import six - -from .udev import generate_udev_rule - - -def filter_by_type(match_type): - return lambda iface: match_type == iface['type'] - - -def filter_by_name(match_name): - return lambda iface: match_name == iface['name'] - - -filter_by_physical = filter_by_type('physical') - - -class Renderer(object): - - @staticmethod - def _render_persistent_net(network_state): - """Given state, emit udev rules to map mac to ifname.""" - # TODO(harlowja): this seems shared between eni renderer and - # this, so move it to a shared location. - content = six.StringIO() - for iface in network_state.iter_interfaces(filter_by_physical): - # for physical interfaces write out a persist net udev rule - if 'name' in iface and iface.get('mac_address'): - content.write(generate_udev_rule(iface['name'], - iface['mac_address'])) - return content.getvalue() diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py deleted file mode 100644 index c53acf71..00000000 --- a/cloudinit/net/sysconfig.py +++ /dev/null @@ -1,400 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re - -import six - -from cloudinit.distros.parsers import resolv_conf -from cloudinit import util - -from . import renderer - - -def _make_header(sep='#'): - lines = [ - "Created by cloud-init on instance boot automatically, do not edit.", - "", - ] - for i in range(0, len(lines)): - if lines[i]: - lines[i] = sep + " " + lines[i] - else: - lines[i] = sep - return "\n".join(lines) - - -def _is_default_route(route): - if route['network'] == '::' and route['netmask'] == 0: - return True - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - return True - return False - - -def _quote_value(value): - if re.search(r"\s", value): - # This doesn't handle complex cases... - if value.startswith('"') and value.endswith('"'): - return value - else: - return '"%s"' % value - else: - return value - - -class ConfigMap(object): - """Sysconfig like dictionary object.""" - - # Why does redhat prefer yes/no to true/false?? - _bool_map = { - True: 'yes', - False: 'no', - } - - def __init__(self): - self._conf = {} - - def __setitem__(self, key, value): - self._conf[key] = value - - def drop(self, key): - self._conf.pop(key, None) - - def __len__(self): - return len(self._conf) - - def to_string(self): - buf = six.StringIO() - buf.write(_make_header()) - if self._conf: - buf.write("\n") - for key in sorted(self._conf.keys()): - value = self._conf[key] - if isinstance(value, bool): - value = self._bool_map[value] - if not isinstance(value, six.string_types): - value = str(value) - buf.write("%s=%s\n" % (key, _quote_value(value))) - return buf.getvalue() - - -class Route(ConfigMap): - """Represents a route configuration.""" - - route_fn_tpl = '%(base)s/network-scripts/route-%(name)s' - - def __init__(self, route_name, base_sysconf_dir): - super(Route, self).__init__() - self.last_idx = 1 - self.has_set_default = False - self._route_name = route_name - self._base_sysconf_dir = base_sysconf_dir - - def copy(self): - r = Route(self._route_name, self._base_sysconf_dir) - r._conf = self._conf.copy() - r.last_idx = self.last_idx - r.has_set_default = self.has_set_default - return r - - @property - def path(self): - return self.route_fn_tpl % ({'base': self._base_sysconf_dir, - 'name': self._route_name}) - - -class NetInterface(ConfigMap): - """Represents a sysconfig/networking-script (and its config + children).""" - - iface_fn_tpl = '%(base)s/network-scripts/ifcfg-%(name)s' - - iface_types = { - 'ethernet': 'Ethernet', - 'bond': 'Bond', - 'bridge': 'Bridge', - } - - def __init__(self, iface_name, base_sysconf_dir, kind='ethernet'): - super(NetInterface, self).__init__() - self.children = [] - self.routes = Route(iface_name, base_sysconf_dir) - self._kind = kind - self._iface_name = iface_name - self._conf['DEVICE'] = iface_name - self._conf['TYPE'] = self.iface_types[kind] - self._base_sysconf_dir = base_sysconf_dir - - @property - def name(self): - return self._iface_name - - @name.setter - def name(self, iface_name): - self._iface_name = iface_name - self._conf['DEVICE'] = iface_name - - @property - def kind(self): - return self._kind - - @kind.setter - def kind(self, kind): - self._kind = kind - self._conf['TYPE'] = self.iface_types[kind] - - @property - def path(self): - return self.iface_fn_tpl % ({'base': self._base_sysconf_dir, - 'name': self.name}) - - def copy(self, copy_children=False, copy_routes=False): - c = NetInterface(self.name, self._base_sysconf_dir, kind=self._kind) - c._conf = self._conf.copy() - if copy_children: - c.children = list(self.children) - if copy_routes: - c.routes = self.routes.copy() - return c - - -class Renderer(renderer.Renderer): - """Renders network information in a /etc/sysconfig format.""" - - # See: https://access.redhat.com/documentation/en-US/\ - # Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/\ - # s1-networkscripts-interfaces.html (or other docs for - # details about this) - - iface_defaults = tuple([ - ('ONBOOT', True), - ('USERCTL', False), - ('NM_CONTROLLED', False), - ('BOOTPROTO', 'none'), - ]) - - # If these keys exist, then there values will be used to form - # a BONDING_OPTS grouping; otherwise no grouping will be set. - bond_tpl_opts = tuple([ - ('bond_mode', "mode=%s"), - ('bond_xmit_hash_policy', "xmit_hash_policy=%s"), - ('bond_miimon', "miimon=%s"), - ]) - - bridge_opts_keys = tuple([ - ('bridge_stp', 'STP'), - ('bridge_ageing', 'AGEING'), - ('bridge_bridgeprio', 'PRIO'), - ]) - - def __init__(self, config=None): - if not config: - config = {} - self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') - self.netrules_path = config.get( - 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') - self.dns_path = config.get('dns_path', 'etc/resolv.conf') - - @classmethod - def _render_iface_shared(cls, iface, iface_cfg): - for k, v in cls.iface_defaults: - iface_cfg[k] = v - for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: - old_value = iface.get(old_key) - if old_value is not None: - iface_cfg[new_key] = old_value - - @classmethod - def _render_subnet(cls, iface_cfg, route_cfg, subnet): - subnet_type = subnet.get('type') - if subnet_type == 'dhcp6': - iface_cfg['DHCPV6C'] = True - iface_cfg['IPV6INIT'] = True - iface_cfg['BOOTPROTO'] = 'dhcp' - elif subnet_type in ['dhcp4', 'dhcp']: - iface_cfg['BOOTPROTO'] = 'dhcp' - elif subnet_type == 'static': - iface_cfg['BOOTPROTO'] = 'static' - if subnet.get('ipv6'): - iface_cfg['IPV6ADDR'] = subnet['address'] - iface_cfg['IPV6INIT'] = True - else: - iface_cfg['IPADDR'] = subnet['address'] - else: - raise ValueError("Unknown subnet type '%s' found" - " for interface '%s'" % (subnet_type, - iface_cfg.name)) - if 'netmask' in subnet: - iface_cfg['NETMASK'] = subnet['netmask'] - for route in subnet.get('routes', []): - if _is_default_route(route): - if route_cfg.has_set_default: - raise ValueError("Duplicate declaration of default" - " route found for interface '%s'" - % (iface_cfg.name)) - # NOTE(harlowja): ipv6 and ipv4 default gateways - gw_key = 'GATEWAY0' - nm_key = 'NETMASK0' - addr_key = 'ADDRESS0' - # The owning interface provides the default route. - # - # TODO(harlowja): add validation that no other iface has - # also provided the default route? - iface_cfg['DEFROUTE'] = True - if 'gateway' in route: - iface_cfg['GATEWAY'] = route['gateway'] - route_cfg.has_set_default = True - else: - gw_key = 'GATEWAY%s' % route_cfg.last_idx - nm_key = 'NETMASK%s' % route_cfg.last_idx - addr_key = 'ADDRESS%s' % route_cfg.last_idx - route_cfg.last_idx += 1 - for (old_key, new_key) in [('gateway', gw_key), - ('netmask', nm_key), - ('network', addr_key)]: - if old_key in route: - route_cfg[new_key] = route[old_key] - - @classmethod - def _render_bonding_opts(cls, iface_cfg, iface): - bond_opts = [] - for (bond_key, value_tpl) in cls.bond_tpl_opts: - # Seems like either dash or underscore is possible? - bond_keys = [bond_key, bond_key.replace("_", "-")] - for bond_key in bond_keys: - if bond_key in iface: - bond_value = iface[bond_key] - if isinstance(bond_value, (tuple, list)): - bond_value = " ".join(bond_value) - bond_opts.append(value_tpl % (bond_value)) - break - if bond_opts: - iface_cfg['BONDING_OPTS'] = " ".join(bond_opts) - - @classmethod - def _render_physical_interfaces(cls, network_state, iface_contents): - physical_filter = renderer.filter_by_physical - for iface in network_state.iter_interfaces(physical_filter): - iface_name = iface['name'] - iface_subnets = iface.get("subnets", []) - iface_cfg = iface_contents[iface_name] - route_cfg = iface_cfg.routes - if len(iface_subnets) == 1: - cls._render_subnet(iface_cfg, route_cfg, iface_subnets[0]) - elif len(iface_subnets) > 1: - for i, iface_subnet in enumerate(iface_subnets, - start=len(iface.children)): - iface_sub_cfg = iface_cfg.copy() - iface_sub_cfg.name = "%s:%s" % (iface_name, i) - iface.children.append(iface_sub_cfg) - cls._render_subnet(iface_sub_cfg, route_cfg, iface_subnet) - - @classmethod - def _render_bond_interfaces(cls, network_state, iface_contents): - bond_filter = renderer.filter_by_type('bond') - for iface in network_state.iter_interfaces(bond_filter): - iface_name = iface['name'] - iface_cfg = iface_contents[iface_name] - cls._render_bonding_opts(iface_cfg, iface) - iface_master_name = iface['bond-master'] - iface_cfg['MASTER'] = iface_master_name - iface_cfg['SLAVE'] = True - # Ensure that the master interface (and any of its children) - # are actually marked as being bond types... - master_cfg = iface_contents[iface_master_name] - master_cfgs = [master_cfg] - master_cfgs.extend(master_cfg.children) - for master_cfg in master_cfgs: - master_cfg['BONDING_MASTER'] = True - master_cfg.kind = 'bond' - - @staticmethod - def _render_vlan_interfaces(network_state, iface_contents): - vlan_filter = renderer.filter_by_type('vlan') - for iface in network_state.iter_interfaces(vlan_filter): - iface_name = iface['name'] - iface_cfg = iface_contents[iface_name] - iface_cfg['VLAN'] = True - iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] - - @staticmethod - def _render_dns(network_state, existing_dns_path=None): - content = resolv_conf.ResolvConf("") - if existing_dns_path and os.path.isfile(existing_dns_path): - content = resolv_conf.ResolvConf(util.load_file(existing_dns_path)) - for nameserver in network_state.dns_nameservers: - content.add_nameserver(nameserver) - for searchdomain in network_state.dns_searchdomains: - content.add_search_domain(searchdomain) - return "\n".join([_make_header(';'), str(content)]) - - @classmethod - def _render_bridge_interfaces(cls, network_state, iface_contents): - bridge_filter = renderer.filter_by_type('bridge') - for iface in network_state.iter_interfaces(bridge_filter): - iface_name = iface['name'] - iface_cfg = iface_contents[iface_name] - iface_cfg.kind = 'bridge' - for old_key, new_key in cls.bridge_opts_keys: - if old_key in iface: - iface_cfg[new_key] = iface[old_key] - # Is this the right key to get all the connected interfaces? - for bridged_iface_name in iface.get('bridge_ports', []): - # Ensure all bridged interfaces are correctly tagged - # as being bridged to this interface. - bridged_cfg = iface_contents[bridged_iface_name] - bridged_cfgs = [bridged_cfg] - bridged_cfgs.extend(bridged_cfg.children) - for bridge_cfg in bridged_cfgs: - bridge_cfg['BRIDGE'] = iface_name - - @classmethod - def _render_sysconfig(cls, base_sysconf_dir, network_state): - '''Given state, return /etc/sysconfig files + contents''' - iface_contents = {} - for iface in network_state.iter_interfaces(): - iface_name = iface['name'] - iface_cfg = NetInterface(iface_name, base_sysconf_dir) - cls._render_iface_shared(iface, iface_cfg) - iface_contents[iface_name] = iface_cfg - cls._render_physical_interfaces(network_state, iface_contents) - cls._render_bond_interfaces(network_state, iface_contents) - cls._render_vlan_interfaces(network_state, iface_contents) - cls._render_bridge_interfaces(network_state, iface_contents) - contents = {} - for iface_name, iface_cfg in iface_contents.items(): - if iface_cfg or iface_cfg.children: - contents[iface_cfg.path] = iface_cfg.to_string() - for iface_cfg in iface_cfg.children: - if iface_cfg: - contents[iface_cfg.path] = iface_cfg.to_string() - if iface_cfg.routes: - contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() - return contents - - def render_network_state(self, target, network_state): - base_sysconf_dir = os.path.join(target, self.sysconf_dir) - for path, data in self._render_sysconfig(base_sysconf_dir, - network_state).items(): - util.write_file(path, data) - if self.dns_path: - dns_path = os.path.join(target, self.dns_path) - resolv_content = self._render_dns(network_state, - existing_dns_path=dns_path) - util.write_file(dns_path, resolv_content) - if self.netrules_path: - netrules_content = self._render_persistent_net(network_state) - netrules_path = os.path.join(target, self.netrules_path) - util.write_file(netrules_path, netrules_content) diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py deleted file mode 100644 index 09188295..00000000 --- a/cloudinit/net/udev.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (C) 2015 Canonical Ltd. -# -# Author: Ryan Harper -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - - -def compose_udev_equality(key, value): - """Return a udev comparison clause, like `ACTION=="add"`.""" - assert key == key.upper() - return '%s=="%s"' % (key, value) - - -def compose_udev_attr_equality(attribute, value): - """Return a udev attribute comparison clause, like `ATTR{type}=="1"`.""" - assert attribute == attribute.lower() - return 'ATTR{%s}=="%s"' % (attribute, value) - - -def compose_udev_setting(key, value): - """Return a udev assignment clause, like `NAME="eth0"`.""" - assert key == key.upper() - return '%s="%s"' % (key, value) - - -def generate_udev_rule(interface, mac): - """Return a udev rule to set the name of network interface with `mac`. - - The rule ends up as a single line looking something like: - - SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", - ATTR{address}="ff:ee:dd:cc:bb:aa", NAME="eth0" - """ - rule = ', '.join([ - compose_udev_equality('SUBSYSTEM', 'net'), - compose_udev_equality('ACTION', 'add'), - compose_udev_equality('DRIVERS', '?*'), - compose_udev_attr_equality('address', mac), - compose_udev_setting('NAME', interface), - ]) - return '%s\n' % rule - -# vi: ts=4 expandtab syntax=python diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py deleted file mode 100644 index d8698a5d..00000000 --- a/cloudinit/netinfo.py +++ /dev/null @@ -1,249 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import re - -from cloudinit import log as logging -from cloudinit import util - -from prettytable import PrettyTable - -LOG = logging.getLogger() - - -def netdev_info(empty=""): - fields = ("hwaddr", "addr", "bcast", "mask") - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"]) - devs = {} - for line in str(ifcfg_out).splitlines(): - if len(line) == 0: - continue - if line[0] not in ("\t", " "): - curdev = line.split()[0] - devs[curdev] = {"up": False} - for field in fields: - devs[curdev][field] = "" - toks = line.lower().strip().split() - if toks[0] == "up": - devs[curdev]['up'] = True - # If the output of ifconfig doesn't contain the required info in the - # obvious place, use a regex filter to be sure. - elif len(toks) > 1: - if re.search(r"flags=\d+= 0: - return "%s[%s]" % (r['gateway'], r['iface']) - return None - - -def netdev_pformat(): - lines = [] - try: - netdev = netdev_info(empty=".") - except Exception: - lines.append(util.center("Net device info failed", '!', 80)) - else: - fields = ['Device', 'Up', 'Address', 'Mask', 'Scope', 'Hw-Address'] - tbl = PrettyTable(fields) - for (dev, d) in netdev.items(): - tbl.add_row([dev, d["up"], d["addr"], d["mask"], ".", d["hwaddr"]]) - if d.get('addr6'): - tbl.add_row([dev, d["up"], - d["addr6"], ".", d.get("scope6"), d["hwaddr"]]) - netdev_s = tbl.get_string() - max_len = len(max(netdev_s.splitlines(), key=len)) - header = util.center("Net device info", "+", max_len) - lines.extend([header, netdev_s]) - return "\n".join(lines) - - -def route_pformat(): - lines = [] - try: - routes = route_info() - except Exception as e: - lines.append(util.center('Route info failed', '!', 80)) - util.logexc(LOG, "Route info failed: %s" % e) - else: - if routes.get('ipv4'): - fields_v4 = ['Route', 'Destination', 'Gateway', - 'Genmask', 'Interface', 'Flags'] - tbl_v4 = PrettyTable(fields_v4) - for (n, r) in enumerate(routes.get('ipv4')): - route_id = str(n) - tbl_v4.add_row([route_id, r['destination'], - r['gateway'], r['genmask'], - r['iface'], r['flags']]) - route_s = tbl_v4.get_string() - max_len = len(max(route_s.splitlines(), key=len)) - header = util.center("Route IPv4 info", "+", max_len) - lines.extend([header, route_s]) - if routes.get('ipv6'): - fields_v6 = ['Route', 'Proto', 'Recv-Q', 'Send-Q', - 'Local Address', 'Foreign Address', 'State'] - tbl_v6 = PrettyTable(fields_v6) - for (n, r) in enumerate(routes.get('ipv6')): - route_id = str(n) - tbl_v6.add_row([route_id, r['proto'], - r['recv-q'], r['send-q'], - r['local address'], r['foreign address'], - r['state']]) - route_s = tbl_v6.get_string() - max_len = len(max(route_s.splitlines(), key=len)) - header = util.center("Route IPv6 info", "+", max_len) - lines.extend([header, route_s]) - return "\n".join(lines) - - -def debug_info(prefix='ci-info: '): - lines = [] - netdev_lines = netdev_pformat().splitlines() - if prefix: - for line in netdev_lines: - lines.append("%s%s" % (prefix, line)) - else: - lines.extend(netdev_lines) - route_lines = route_pformat().splitlines() - if prefix: - for line in route_lines: - lines.append("%s%s" % (prefix, line)) - else: - lines.extend(route_lines) - return "\n".join(lines) diff --git a/cloudinit/patcher.py b/cloudinit/patcher.py deleted file mode 100644 index f6609d6f..00000000 --- a/cloudinit/patcher.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import imp -import logging -import sys - -# Default fallback format -FALL_FORMAT = ('FALLBACK: %(asctime)s - %(filename)s[%(levelname)s]: ' + - '%(message)s') - - -class QuietStreamHandler(logging.StreamHandler): - def handleError(self, record): - pass - - -def _patch_logging(): - # Replace 'handleError' with one that will be more - # tolerant of errors in that it can avoid - # re-notifying on exceptions and when errors - # do occur, it can at least try to write to - # sys.stderr using a fallback logger - fallback_handler = QuietStreamHandler(sys.stderr) - fallback_handler.setFormatter(logging.Formatter(FALL_FORMAT)) - - def handleError(self, record): - try: - fallback_handler.handle(record) - fallback_handler.flush() - except IOError: - pass - setattr(logging.Handler, 'handleError', handleError) - - -def patch(): - imp.acquire_lock() - try: - _patch_logging() - finally: - imp.release_lock() diff --git a/cloudinit/registry.py b/cloudinit/registry.py deleted file mode 100644 index 04368ddf..00000000 --- a/cloudinit/registry.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2015 Canonical Ltd. -# This file is part of cloud-init. See LICENCE file for license information. -# -# vi: ts=4 expandtab -import copy - - -class DictRegistry(object): - """A simple registry for a mapping of objects.""" - - def __init__(self): - self.reset() - - def reset(self): - self._items = {} - - def register_item(self, key, item): - """Add item to the registry.""" - if key in self._items: - raise ValueError( - 'Item already registered with key {0}'.format(key)) - self._items[key] = item - - def unregister_item(self, key, force=True): - """Remove item from the registry.""" - if key in self._items: - del self._items[key] - elif not force: - raise KeyError("%s: key not present to unregister" % key) - - @property - def registered_items(self): - """All the items that have been registered. - - This cannot be used to modify the contents of the registry. - """ - return copy.copy(self._items) diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py deleted file mode 100644 index 6b41ae61..00000000 --- a/cloudinit/reporting/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2015 Canonical Ltd. -# This file is part of cloud-init. See LICENCE file for license information. -# -""" -cloud-init reporting framework - -The reporting framework is intended to allow all parts of cloud-init to -report events in a structured manner. -""" - -from ..registry import DictRegistry -from .handlers import available_handlers - -DEFAULT_CONFIG = { - 'logging': {'type': 'log'}, -} - - -def update_configuration(config): - """Update the instanciated_handler_registry. - - :param config: - The dictionary containing changes to apply. If a key is given - with a False-ish value, the registered handler matching that name - will be unregistered. - """ - for handler_name, handler_config in config.items(): - if not handler_config: - instantiated_handler_registry.unregister_item( - handler_name, force=True) - continue - handler_config = handler_config.copy() - cls = available_handlers.registered_items[handler_config.pop('type')] - instantiated_handler_registry.unregister_item(handler_name) - instance = cls(**handler_config) - instantiated_handler_registry.register_item(handler_name, instance) - - -instantiated_handler_registry = DictRegistry() -update_configuration(DEFAULT_CONFIG) - -# vi: ts=4 expandtab diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py deleted file mode 100644 index df2b9b4a..00000000 --- a/cloudinit/reporting/events.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright 2015 Canonical Ltd. -# This file is part of cloud-init. See LICENCE file for license information. -# -""" -events for reporting. - -The events here are designed to be used with reporting. -They can be published to registered handlers with report_event. -""" -import base64 -import os.path -import time - -from . import instantiated_handler_registry - -FINISH_EVENT_TYPE = 'finish' -START_EVENT_TYPE = 'start' - -DEFAULT_EVENT_ORIGIN = 'cloudinit' - - -class _nameset(set): - def __getattr__(self, name): - if name in self: - return name - raise AttributeError("%s not a valid value" % name) - - -status = _nameset(("SUCCESS", "WARN", "FAIL")) - - -class ReportingEvent(object): - """Encapsulation of event formatting.""" - - def __init__(self, event_type, name, description, - origin=DEFAULT_EVENT_ORIGIN, timestamp=None): - self.event_type = event_type - self.name = name - self.description = description - self.origin = origin - if timestamp is None: - timestamp = time.time() - self.timestamp = timestamp - - def as_string(self): - """The event represented as a string.""" - return '{0}: {1}: {2}'.format( - self.event_type, self.name, self.description) - - def as_dict(self): - """The event represented as a dictionary.""" - return {'name': self.name, 'description': self.description, - 'event_type': self.event_type, 'origin': self.origin, - 'timestamp': self.timestamp} - - -class FinishReportingEvent(ReportingEvent): - - def __init__(self, name, description, result=status.SUCCESS, - post_files=None): - super(FinishReportingEvent, self).__init__( - FINISH_EVENT_TYPE, name, description) - self.result = result - if post_files is None: - post_files = [] - self.post_files = post_files - if result not in status: - raise ValueError("Invalid result: %s" % result) - - def as_string(self): - return '{0}: {1}: {2}: {3}'.format( - self.event_type, self.name, self.result, self.description) - - def as_dict(self): - """The event represented as json friendly.""" - data = super(FinishReportingEvent, self).as_dict() - data['result'] = self.result - if self.post_files: - data['files'] = _collect_file_info(self.post_files) - return data - - -def report_event(event): - """Report an event to all registered event handlers. - - This should generally be called via one of the other functions in - the reporting module. - - :param event_type: - The type of the event; this should be a constant from the - reporting module. - """ - for _, handler in instantiated_handler_registry.registered_items.items(): - handler.publish_event(event) - - -def report_finish_event(event_name, event_description, - result=status.SUCCESS, post_files=None): - """Report a "finish" event. - - See :py:func:`.report_event` for parameter details. - """ - event = FinishReportingEvent(event_name, event_description, result, - post_files=post_files) - return report_event(event) - - -def report_start_event(event_name, event_description): - """Report a "start" event. - - :param event_name: - The name of the event; this should be a topic which events would - share (e.g. it will be the same for start and finish events). - - :param event_description: - A human-readable description of the event that has occurred. - """ - event = ReportingEvent(START_EVENT_TYPE, event_name, event_description) - return report_event(event) - - -class ReportEventStack(object): - """Context Manager for using :py:func:`report_event` - - This enables calling :py:func:`report_start_event` and - :py:func:`report_finish_event` through a context manager. - - :param name: - the name of the event - - :param description: - the event's description, passed on to :py:func:`report_start_event` - - :param message: - the description to use for the finish event. defaults to - :param:description. - - :param parent: - :type parent: :py:class:ReportEventStack or None - The parent of this event. The parent is populated with - results of all its children. The name used in reporting - is / - - :param reporting_enabled: - Indicates if reporting events should be generated. - If not provided, defaults to the parent's value, or True if no parent - is provided. - - :param result_on_exception: - The result value to set if an exception is caught. default - value is FAIL. - """ - def __init__(self, name, description, message=None, parent=None, - reporting_enabled=None, result_on_exception=status.FAIL, - post_files=None): - self.parent = parent - self.name = name - self.description = description - self.message = message - self.result_on_exception = result_on_exception - self.result = status.SUCCESS - if post_files is None: - post_files = [] - self.post_files = post_files - - # use parents reporting value if not provided - if reporting_enabled is None: - if parent: - reporting_enabled = parent.reporting_enabled - else: - reporting_enabled = True - self.reporting_enabled = reporting_enabled - - if parent: - self.fullname = '/'.join((parent.fullname, name,)) - else: - self.fullname = self.name - self.children = {} - - def __repr__(self): - return ("ReportEventStack(%s, %s, reporting_enabled=%s)" % - (self.name, self.description, self.reporting_enabled)) - - def __enter__(self): - self.result = status.SUCCESS - if self.reporting_enabled: - report_start_event(self.fullname, self.description) - if self.parent: - self.parent.children[self.name] = (None, None) - return self - - def _childrens_finish_info(self): - for cand_result in (status.FAIL, status.WARN): - for name, (value, msg) in self.children.items(): - if value == cand_result: - return (value, self.message) - return (self.result, self.message) - - @property - def result(self): - return self._result - - @result.setter - def result(self, value): - if value not in status: - raise ValueError("'%s' not a valid result" % value) - self._result = value - - @property - def message(self): - if self._message is not None: - return self._message - return self.description - - @message.setter - def message(self, value): - self._message = value - - def _finish_info(self, exc): - # return tuple of description, and value - if exc: - return (self.result_on_exception, self.message) - return self._childrens_finish_info() - - def __exit__(self, exc_type, exc_value, traceback): - (result, msg) = self._finish_info(exc_value) - if self.parent: - self.parent.children[self.name] = (result, msg) - if self.reporting_enabled: - report_finish_event(self.fullname, msg, result, - post_files=self.post_files) - - -def _collect_file_info(files): - if not files: - return None - ret = [] - for fname in files: - if not os.path.isfile(fname): - content = None - else: - with open(fname, "rb") as fp: - content = base64.b64encode(fp.read()).decode() - ret.append({'path': fname, 'content': content, - 'encoding': 'base64'}) - return ret - -# vi: ts=4 expandtab syntax=python diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py deleted file mode 100644 index dff20ecb..00000000 --- a/cloudinit/reporting/handlers.py +++ /dev/null @@ -1,91 +0,0 @@ -# vi: ts=4 expandtab - -import abc -import json -import six - -from cloudinit import log as logging -from cloudinit.registry import DictRegistry -from cloudinit import (url_helper, util) - - -LOG = logging.getLogger(__name__) - - -@six.add_metaclass(abc.ABCMeta) -class ReportingHandler(object): - """Base class for report handlers. - - Implement :meth:`~publish_event` for controlling what - the handler does with an event. - """ - - @abc.abstractmethod - def publish_event(self, event): - """Publish an event.""" - - -class LogHandler(ReportingHandler): - """Publishes events to the cloud-init log at the ``DEBUG`` log level.""" - - def __init__(self, level="DEBUG"): - super(LogHandler, self).__init__() - if isinstance(level, int): - pass - else: - input_level = level - try: - level = getattr(logging, level.upper()) - except Exception: - LOG.warn("invalid level '%s', using WARN", input_level) - level = logging.WARN - self.level = level - - def publish_event(self, event): - logger = logging.getLogger( - '.'.join(['cloudinit', 'reporting', event.event_type, event.name])) - logger.log(self.level, event.as_string()) - - -class PrintHandler(ReportingHandler): - """Print the event as a string.""" - - def publish_event(self, event): - print(event.as_string()) - - -class WebHookHandler(ReportingHandler): - def __init__(self, endpoint, consumer_key=None, token_key=None, - token_secret=None, consumer_secret=None, timeout=None, - retries=None): - super(WebHookHandler, self).__init__() - - if any([consumer_key, token_key, token_secret, consumer_secret]): - self.oauth_helper = url_helper.OauthUrlHelper( - consumer_key=consumer_key, token_key=token_key, - token_secret=token_secret, consumer_secret=consumer_secret) - else: - self.oauth_helper = None - self.endpoint = endpoint - self.timeout = timeout - self.retries = retries - self.ssl_details = util.fetch_ssl_details() - - def publish_event(self, event): - if self.oauth_helper: - readurl = self.oauth_helper.readurl - else: - readurl = url_helper.readurl - try: - return readurl( - self.endpoint, data=json.dumps(event.as_dict()), - timeout=self.timeout, - retries=self.retries, ssl_details=self.ssl_details) - except Exception: - LOG.warn("failed posting event: %s" % event.as_string()) - - -available_handlers = DictRegistry() -available_handlers.register_item('log', LogHandler) -available_handlers.register_item('print', PrintHandler) -available_handlers.register_item('webhook', WebHookHandler) diff --git a/cloudinit/safeyaml.py b/cloudinit/safeyaml.py deleted file mode 100644 index eba5d056..00000000 --- a/cloudinit/safeyaml.py +++ /dev/null @@ -1,32 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import yaml - - -class _CustomSafeLoader(yaml.SafeLoader): - def construct_python_unicode(self, node): - return self.construct_scalar(node) - -_CustomSafeLoader.add_constructor( - u'tag:yaml.org,2002:python/unicode', - _CustomSafeLoader.construct_python_unicode) - - -def load(blob): - return(yaml.load(blob, Loader=_CustomSafeLoader)) diff --git a/cloudinit/serial.py b/cloudinit/serial.py deleted file mode 100644 index af45c13e..00000000 --- a/cloudinit/serial.py +++ /dev/null @@ -1,50 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -from __future__ import absolute_import - -try: - from serial import Serial -except ImportError: - # For older versions of python (ie 2.6) pyserial may not exist and/or - # work and/or be installed, so make a dummy/fake serial that blows up - # when used... - class Serial(object): - def __init__(self, *args, **kwargs): - pass - - @staticmethod - def isOpen(): - return False - - @staticmethod - def write(data): - raise IOError("Unable to perform serial `write` operation," - " pyserial not installed.") - - @staticmethod - def readline(): - raise IOError("Unable to perform serial `readline` operation," - " pyserial not installed.") - - @staticmethod - def flush(): - raise IOError("Unable to perform serial `flush` operation," - " pyserial not installed.") - - @staticmethod - def read(size=1): - raise IOError("Unable to perform serial `read` operation," - " pyserial not installed.") diff --git a/cloudinit/settings.py b/cloudinit/settings.py deleted file mode 100644 index 8c258ea1..00000000 --- a/cloudinit/settings.py +++ /dev/null @@ -1,68 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Set and read for determining the cloud config file location -CFG_ENV_NAME = "CLOUD_CFG" - -# This is expected to be a yaml formatted file -CLOUD_CONFIG = '/etc/cloud/cloud.cfg' - -# What u get if no config is provided -CFG_BUILTIN = { - 'datasource_list': [ - 'NoCloud', - 'ConfigDrive', - 'OpenNebula', - 'Azure', - 'AltCloud', - 'OVF', - 'MAAS', - 'GCE', - 'OpenStack', - 'Ec2', - 'CloudSigma', - 'CloudStack', - 'SmartOS', - 'Bigstep', - # At the end to act as a 'catch' when none of the above work... - 'None', - ], - 'def_log_file': '/var/log/cloud-init.log', - 'log_cfgs': [], - 'syslog_fix_perms': ['syslog:adm', 'root:adm'], - 'system_info': { - 'paths': { - 'cloud_dir': '/var/lib/cloud', - 'templates_dir': '/etc/cloud/templates/', - }, - 'distro': 'ubuntu', - }, - 'vendor_data': {'enabled': True, 'prefix': []}, -} - -# Valid frequencies of handlers/modules -PER_INSTANCE = "once-per-instance" -PER_ALWAYS = "always" -PER_ONCE = "once" - -# Used to sanity check incoming handlers/modules frequencies -FREQUENCIES = [PER_INSTANCE, PER_ALWAYS, PER_ONCE] diff --git a/cloudinit/signal_handler.py b/cloudinit/signal_handler.py deleted file mode 100644 index 0d95f506..00000000 --- a/cloudinit/signal_handler.py +++ /dev/null @@ -1,71 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import inspect -import signal -import sys - -from six import StringIO - -from cloudinit import log as logging -from cloudinit import util -from cloudinit import version as vr - -LOG = logging.getLogger(__name__) - - -BACK_FRAME_TRACE_DEPTH = 3 -EXIT_FOR = { - signal.SIGINT: ('Cloud-init %(version)s received SIGINT, exiting...', 1), - signal.SIGTERM: ('Cloud-init %(version)s received SIGTERM, exiting...', 1), - # Can't be caught... - # signal.SIGKILL: ('Cloud-init killed, exiting...', 1), - signal.SIGABRT: ('Cloud-init %(version)s received SIGABRT, exiting...', 1), -} - - -def _pprint_frame(frame, depth, max_depth, contents): - if depth > max_depth or not frame: - return - frame_info = inspect.getframeinfo(frame) - prefix = " " * (depth * 2) - contents.write("%sFilename: %s\n" % (prefix, frame_info.filename)) - contents.write("%sFunction: %s\n" % (prefix, frame_info.function)) - contents.write("%sLine number: %s\n" % (prefix, frame_info.lineno)) - _pprint_frame(frame.f_back, depth + 1, max_depth, contents) - - -def _handle_exit(signum, frame): - (msg, rc) = EXIT_FOR[signum] - msg = msg % ({'version': vr.version()}) - contents = StringIO() - contents.write("%s\n" % (msg)) - _pprint_frame(frame, 1, BACK_FRAME_TRACE_DEPTH, contents) - util.multi_log(contents.getvalue(), - console=True, stderr=False, log=LOG) - sys.exit(rc) - - -def attach_handlers(): - sigs_attached = 0 - for signum in EXIT_FOR.keys(): - signal.signal(signum, _handle_exit) - sigs_attached += len(EXIT_FOR) - return sigs_attached diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py deleted file mode 100644 index a3529609..00000000 --- a/cloudinit/sources/DataSourceAltCloud.py +++ /dev/null @@ -1,292 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joe VLcek -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -''' -This file contains code used to gather the user data passed to an -instance on RHEVm and vSphere. -''' - -import errno -import os -import os.path - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util - -from cloudinit.util import ProcessExecutionError - -LOG = logging.getLogger(__name__) - -# Needed file paths -CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info' - -# Shell command lists -CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy'] -CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle', '--timeout=5'] - -META_DATA_NOT_SUPPORTED = { - 'block-device-mapping': {}, - 'instance-id': 455, - 'local-hostname': 'localhost', - 'placement': {}, -} - - -def read_user_data_callback(mount_dir): - ''' - Description: - This callback will be applied by util.mount_cb() on the mounted - file. - - Deltacloud file name contains deltacloud. Those not using - Deltacloud but instead instrumenting the injection, could - drop deltacloud from the file name. - - Input: - mount_dir - Mount directory - - Returns: - User Data - - ''' - - deltacloud_user_data_file = mount_dir + '/deltacloud-user-data.txt' - user_data_file = mount_dir + '/user-data.txt' - - # First try deltacloud_user_data_file. On failure try user_data_file. - try: - user_data = util.load_file(deltacloud_user_data_file).strip() - except IOError: - try: - user_data = util.load_file(user_data_file).strip() - except IOError: - util.logexc(LOG, 'Failed accessing user data file.') - return None - - return user_data - - -class DataSourceAltCloud(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed = None - self.supported_seed_starts = ("/", "file://") - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def get_cloud_type(self): - ''' - Description: - Get the type for the cloud back end this instance is running on - by examining the string returned by reading the dmi data. - - Input: - None - - Returns: - One of the following strings: - 'RHEV', 'VSPHERE' or 'UNKNOWN' - - ''' - - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmi data is not available on ARM processors - LOG.debug("Disabling AltCloud datasource on arm (LP: #1243287)") - return 'UNKNOWN' - - system_name = util.read_dmi_data("system-product-name") - if not system_name: - return 'UNKNOWN' - - sys_name = system_name.upper() - - if sys_name.startswith('RHEV'): - return 'RHEV' - - if sys_name.startswith('VMWARE'): - return 'VSPHERE' - - return 'UNKNOWN' - - def get_data(self): - ''' - Description: - User Data is passed to the launching instance which - is used to perform instance configuration. - - Cloud providers expose the user data differently. - It is necessary to determine which cloud provider - the current instance is running on to determine - how to access the user data. Images built with - image factory will contain a CLOUD_INFO_FILE which - contains a string identifying the cloud provider. - - Images not built with Imagefactory will try to - determine what the cloud provider is based on system - information. - ''' - - LOG.debug('Invoked get_data()') - - if os.path.exists(CLOUD_INFO_FILE): - try: - cloud_type = util.load_file(CLOUD_INFO_FILE).strip().upper() - except IOError: - util.logexc(LOG, 'Unable to access cloud info file at %s.', - CLOUD_INFO_FILE) - return False - else: - cloud_type = self.get_cloud_type() - - LOG.debug('cloud_type: ' + str(cloud_type)) - - if 'RHEV' in cloud_type: - if self.user_data_rhevm(): - return True - elif 'VSPHERE' in cloud_type: - if self.user_data_vsphere(): - return True - else: - # there was no recognized alternate cloud type - # indicating this handler should not be used. - return False - - # No user data found - util.logexc(LOG, 'Failed accessing user data.') - return False - - def user_data_rhevm(self): - ''' - RHEVM specific userdata read - - If on RHEV-M the user data will be contained on the - floppy device in file - To access it: - modprobe floppy - - Leverage util.mount_cb to: - mkdir - mount /dev/fd0 - The call back passed to util.mount_cb will do: - read / - ''' - - return_str = None - - # modprobe floppy - try: - cmd = CMD_PROBE_FLOPPY - (cmd_out, _err) = util.subp(cmd) - LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) - return False - except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) - return False - - floppy_dev = '/dev/fd0' - - # udevadm settle for floppy device - try: - cmd = CMD_UDEVADM_SETTLE - cmd.append('--exit-if-exists=' + floppy_dev) - (cmd_out, _err) = util.subp(cmd) - LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) - return False - except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) - return False - - try: - return_str = util.mount_cb(floppy_dev, read_user_data_callback) - except OSError as err: - if err.errno != errno.ENOENT: - raise - except util.MountFailedError: - util.logexc(LOG, "Failed to mount %s when looking for user data", - floppy_dev) - - self.userdata_raw = return_str - self.metadata = META_DATA_NOT_SUPPORTED - - if return_str: - return True - else: - return False - - def user_data_vsphere(self): - ''' - vSphere specific userdata read - - If on vSphere the user data will be contained on the - cdrom device in file - To access it: - Leverage util.mount_cb to: - mkdir - mount /dev/fd0 - The call back passed to util.mount_cb will do: - read / - ''' - - return_str = None - cdrom_list = util.find_devs_with('LABEL=CDROM') - for cdrom_dev in cdrom_list: - try: - return_str = util.mount_cb(cdrom_dev, read_user_data_callback) - if return_str: - break - except OSError as err: - if err.errno != errno.ENOENT: - raise - except util.MountFailedError: - util.logexc(LOG, "Failed to mount %s when looking for user " - "data", cdrom_dev) - - self.userdata_raw = return_str - self.metadata = META_DATA_NOT_SUPPORTED - - if return_str: - return True - else: - return False - -# Used to match classes to dependencies -# Source DataSourceAltCloud does not really depend on networking. -# In the future 'dsmode' like behavior can be added to offer user -# the ability to run before networking. -datasources = [ - (DataSourceAltCloud, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py deleted file mode 100644 index 8c7e8673..00000000 --- a/cloudinit/sources/DataSourceAzure.py +++ /dev/null @@ -1,651 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 -import contextlib -import crypt -import fnmatch -import os -import os.path -import time -import xml.etree.ElementTree as ET - -from xml.dom import minidom - -from cloudinit.sources.helpers.azure import get_metadata_from_fabric - -from cloudinit import log as logging -from cloudinit.settings import PER_ALWAYS -from cloudinit import sources -from cloudinit import util - -LOG = logging.getLogger(__name__) - -DS_NAME = 'Azure' -DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} -AGENT_START = ['service', 'walinuxagent', 'start'] -BOUNCE_COMMAND = [ - 'sh', '-xc', - "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x" -] - -BUILTIN_DS_CONFIG = { - 'agent_command': AGENT_START, - 'data_dir': "/var/lib/waagent", - 'set_hostname': True, - 'hostname_bounce': { - 'interface': 'eth0', - 'policy': True, - 'command': BOUNCE_COMMAND, - 'hostname_command': 'hostname', - }, - 'disk_aliases': {'ephemeral0': '/dev/sdb'}, -} - -BUILTIN_CLOUD_CONFIG = { - 'disk_setup': { - 'ephemeral0': {'table_type': 'gpt', - 'layout': [100], - 'overwrite': True}, - }, - 'fs_setup': [{'filesystem': 'ext4', - 'device': 'ephemeral0.1', - 'replace_fs': 'ntfs'}], -} - -DS_CFG_PATH = ['datasource', DS_NAME] -DEF_EPHEMERAL_LABEL = 'Temporary Storage' - -# The redacted password fails to meet password complexity requirements -# so we can safely use this to mask/redact the password in the ovf-env.xml -DEF_PASSWD_REDACTION = 'REDACTED' - - -def get_hostname(hostname_command='hostname'): - return util.subp(hostname_command, capture=True)[0].strip() - - -def set_hostname(hostname, hostname_command='hostname'): - util.subp([hostname_command, hostname]) - - -@contextlib.contextmanager -def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): - """ - Set a temporary hostname, restoring the previous hostname on exit. - - Will have the value of the previous hostname when used as a context - manager, or None if the hostname was not changed. - """ - policy = cfg['hostname_bounce']['policy'] - previous_hostname = get_hostname(hostname_command) - if (not util.is_true(cfg.get('set_hostname')) or - util.is_false(policy) or - (previous_hostname == temp_hostname and policy != 'force')): - yield None - return - set_hostname(temp_hostname, hostname_command) - try: - yield previous_hostname - finally: - set_hostname(previous_hostname, hostname_command) - - -class DataSourceAzureNet(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed_dir = os.path.join(paths.seed_dir, 'azure') - self.cfg = {} - self.seed = None - self.ds_cfg = util.mergemanydict([ - util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), - BUILTIN_DS_CONFIG]) - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def get_metadata_from_agent(self): - temp_hostname = self.metadata.get('local-hostname') - hostname_command = self.ds_cfg['hostname_bounce']['hostname_command'] - with temporary_hostname(temp_hostname, self.ds_cfg, - hostname_command=hostname_command) \ - as previous_hostname: - if (previous_hostname is not None and - util.is_true(self.ds_cfg.get('set_hostname'))): - cfg = self.ds_cfg['hostname_bounce'] - try: - perform_hostname_bounce(hostname=temp_hostname, - cfg=cfg, - prev_hostname=previous_hostname) - except Exception as e: - LOG.warn("Failed publishing hostname: %s", e) - util.logexc(LOG, "handling set_hostname failed") - - try: - invoke_agent(self.ds_cfg['agent_command']) - except util.ProcessExecutionError: - # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", - self.ds_cfg['agent_command']) - - ddir = self.ds_cfg['data_dir'] - - fp_files = [] - key_value = None - for pk in self.cfg.get('_pubkeys', []): - if pk.get('value', None): - key_value = pk['value'] - LOG.debug("ssh authentication: using value from fabric") - else: - bname = str(pk['fingerprint'] + ".crt") - fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: " - "using fingerprint from fabirc") - - missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", - func=wait_for_files, - args=(fp_files,)) - if len(missing): - LOG.warn("Did not find files, but going on: %s", missing) - - metadata = {} - metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files) - return metadata - - def get_data(self): - # azure removes/ejects the cdrom containing the ovf-env.xml - # file on reboot. So, in order to successfully reboot we - # need to look in the datadir and consider that valid - ddir = self.ds_cfg['data_dir'] - - candidates = [self.seed_dir] - candidates.extend(list_possible_azure_ds_devs()) - if ddir: - candidates.append(ddir) - - found = None - - for cdev in candidates: - try: - if cdev.startswith("/dev/"): - ret = util.mount_cb(cdev, load_azure_ds_dir) - else: - ret = load_azure_ds_dir(cdev) - - except NonAzureDataSource: - continue - except BrokenAzureDataSource as exc: - raise exc - except util.MountFailedError: - LOG.warn("%s was not mountable", cdev) - continue - - (md, self.userdata_raw, cfg, files) = ret - self.seed = cdev - self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) - self.cfg = util.mergemanydict([cfg, BUILTIN_CLOUD_CONFIG]) - found = cdev - - LOG.debug("found datasource in %s", cdev) - break - - if not found: - return False - - if found == ddir: - LOG.debug("using files cached in %s", ddir) - - # azure / hyper-v provides random data here - seed = util.load_file("/sys/firmware/acpi/tables/OEM0", - quiet=True, decode=False) - if seed: - self.metadata['random_seed'] = seed - - # now update ds_cfg to reflect contents pass in config - user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) - self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) - - # walinux agent writes files world readable, but expects - # the directory to be protected. - write_files(ddir, files, dirmode=0o700) - - if self.ds_cfg['agent_command'] == '__builtin__': - metadata_func = get_metadata_from_fabric - else: - metadata_func = self.get_metadata_from_agent - try: - fabric_data = metadata_func() - except Exception as exc: - LOG.info("Error communicating with Azure fabric; assume we aren't" - " on Azure.", exc_info=True) - return False - - self.metadata['instance-id'] = util.read_dmi_data('system-uuid') - self.metadata.update(fabric_data) - - found_ephemeral = find_fabric_formatted_ephemeral_disk() - if found_ephemeral: - self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral - LOG.debug("using detected ephemeral0 of %s", found_ephemeral) - - cc_modules_override = support_new_ephemeral(self.sys_cfg) - if cc_modules_override: - self.cfg['cloud_config_modules'] = cc_modules_override - - return True - - def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) - - def get_config_obj(self): - return self.cfg - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still valid - return sources.instance_id_matches_system_uuid(self.get_instance_id()) - - -def count_files(mp): - return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*')) - - -def find_fabric_formatted_ephemeral_part(): - """ - Locate the first fabric formatted ephemeral device. - """ - potential_locations = ['/dev/disk/cloud/azure_resource-part1', - '/dev/disk/azure/resource-part1'] - device_location = None - for potential_location in potential_locations: - if os.path.exists(potential_location): - device_location = potential_location - break - if device_location is None: - return None - ntfs_devices = util.find_devs_with("TYPE=ntfs") - real_device = os.path.realpath(device_location) - if real_device in ntfs_devices: - return device_location - return None - - -def find_fabric_formatted_ephemeral_disk(): - """ - Get the ephemeral disk. - """ - part_dev = find_fabric_formatted_ephemeral_part() - if part_dev: - return part_dev.split('-')[0] - return None - - -def support_new_ephemeral(cfg): - """ - Windows Azure makes ephemeral devices ephemeral to boot; a ephemeral device - may be presented as a fresh device, or not. - - Since the knowledge of when a disk is supposed to be plowed under is - specific to Windows Azure, the logic resides here in the datasource. When a - new ephemeral device is detected, cloud-init overrides the default - frequency for both disk-setup and mounts for the current boot only. - """ - device = find_fabric_formatted_ephemeral_part() - if not device: - LOG.debug("no default fabric formated ephemeral0.1 found") - return None - LOG.debug("fabric formated ephemeral0.1 device at %s", device) - - file_count = 0 - try: - file_count = util.mount_cb(device, count_files) - except Exception: - return None - LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count) - - if file_count >= 1: - LOG.debug("fabric prepared ephemeral0.1 will be preserved") - return None - else: - # if device was already mounted, then we need to unmount it - # race conditions could allow for a check-then-unmount - # to have a false positive. so just unmount and then check. - try: - util.subp(['umount', device]) - except util.ProcessExecutionError as e: - if device in util.mounts(): - LOG.warn("Failed to unmount %s, will not reformat.", device) - LOG.debug("Failed umount: %s", e) - return None - - LOG.debug("cloud-init will format ephemeral0.1 this boot.") - LOG.debug("setting disk_setup and mounts modules 'always' for this boot") - - cc_modules = cfg.get('cloud_config_modules') - if not cc_modules: - return None - - mod_list = [] - for mod in cc_modules: - if mod in ("disk_setup", "mounts"): - mod_list.append([mod, PER_ALWAYS]) - LOG.debug("set module '%s' to 'always' for this boot", mod) - else: - mod_list.append(mod) - return mod_list - - -def perform_hostname_bounce(hostname, cfg, prev_hostname): - # set the hostname to 'hostname' if it is not already set to that. - # then, if policy is not off, bounce the interface using command - command = cfg['command'] - interface = cfg['interface'] - policy = cfg['policy'] - - msg = ("hostname=%s policy=%s interface=%s" % - (hostname, policy, interface)) - env = os.environ.copy() - env['interface'] = interface - env['hostname'] = hostname - env['old_hostname'] = prev_hostname - - if command == "builtin": - command = BOUNCE_COMMAND - - LOG.debug("pubhname: publishing hostname [%s]", msg) - shell = not isinstance(command, (list, tuple)) - # capture=False, see comments in bug 1202758 and bug 1206164. - util.log_time(logfunc=LOG.debug, msg="publishing hostname", - get_uptime=True, func=util.subp, - kwargs={'args': command, 'shell': shell, 'capture': False, - 'env': env}) - - -def crtfile_to_pubkey(fname, data=None): - pipeline = ('openssl x509 -noout -pubkey < "$0" |' - 'ssh-keygen -i -m PKCS8 -f /dev/stdin') - (out, _err) = util.subp(['sh', '-c', pipeline, fname], - capture=True, data=data) - return out.rstrip() - - -def pubkeys_from_crt_files(flist): - pubkeys = [] - errors = [] - for fname in flist: - try: - pubkeys.append(crtfile_to_pubkey(fname)) - except util.ProcessExecutionError: - errors.append(fname) - - if errors: - LOG.warn("failed to convert the crt files to pubkey: %s", errors) - - return pubkeys - - -def wait_for_files(flist, maxwait=60, naplen=.5): - need = set(flist) - waited = 0 - while waited < maxwait: - need -= set([f for f in need if os.path.exists(f)]) - if len(need) == 0: - return [] - time.sleep(naplen) - waited += naplen - return need - - -def write_files(datadir, files, dirmode=None): - - def _redact_password(cnt, fname): - """Azure provides the UserPassword in plain text. So we redact it""" - try: - root = ET.fromstring(cnt) - for elem in root.iter(): - if ('UserPassword' in elem.tag and - elem.text != DEF_PASSWD_REDACTION): - elem.text = DEF_PASSWD_REDACTION - return ET.tostring(root) - except Exception: - LOG.critical("failed to redact userpassword in %s", fname) - return cnt - - if not datadir: - return - if not files: - files = {} - util.ensure_dir(datadir, dirmode) - for (name, content) in files.items(): - fname = os.path.join(datadir, name) - if 'ovf-env.xml' in name: - content = _redact_password(content, fname) - util.write_file(filename=fname, content=content, mode=0o600) - - -def invoke_agent(cmd): - # this is a function itself to simplify patching it for test - if cmd: - LOG.debug("invoking agent: %s", cmd) - util.subp(cmd, shell=(not isinstance(cmd, list))) - else: - LOG.debug("not invoking agent") - - -def find_child(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return ret - - -def load_azure_ovf_pubkeys(sshnode): - # This parses a 'SSH' node formatted like below, and returns - # an array of dicts. - # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', - # 'path': 'where/to/go'}] - # - # - # ABC/ABC - # ... - # - results = find_child(sshnode, lambda n: n.localName == "PublicKeys") - if len(results) == 0: - return [] - if len(results) > 1: - raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" % - len(results)) - - pubkeys_node = results[0] - pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey") - - if len(pubkeys) == 0: - return [] - - found = [] - text_node = minidom.Document.TEXT_NODE - - for pk_node in pubkeys: - if not pk_node.hasChildNodes(): - continue - - cur = {'fingerprint': "", 'path': "", 'value': ""} - for child in pk_node.childNodes: - if child.nodeType == text_node or not child.localName: - continue - - name = child.localName.lower() - - if name not in cur.keys(): - continue - - if (len(child.childNodes) != 1 or - child.childNodes[0].nodeType != text_node): - continue - - cur[name] = child.childNodes[0].wholeText.strip() - found.append(cur) - - return found - - -def read_azure_ovf(contents): - try: - dom = minidom.parseString(contents) - except Exception as e: - raise BrokenAzureDataSource("invalid xml: %s" % e) - - results = find_child(dom.documentElement, - lambda n: n.localName == "ProvisioningSection") - - if len(results) == 0: - raise NonAzureDataSource("No ProvisioningSection") - if len(results) > 1: - raise BrokenAzureDataSource("found '%d' ProvisioningSection items" % - len(results)) - provSection = results[0] - - lpcs_nodes = find_child(provSection, - lambda n: - n.localName == "LinuxProvisioningConfigurationSet") - - if len(results) == 0: - raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") - if len(results) > 1: - raise BrokenAzureDataSource("found '%d' %ss" % - ("LinuxProvisioningConfigurationSet", - len(results))) - lpcs = lpcs_nodes[0] - - if not lpcs.hasChildNodes(): - raise BrokenAzureDataSource("no child nodes of configuration set") - - md_props = 'seedfrom' - md = {'azure_data': {}} - cfg = {} - ud = "" - password = None - username = None - - for child in lpcs.childNodes: - if child.nodeType == dom.TEXT_NODE or not child.localName: - continue - - name = child.localName.lower() - - simple = False - value = "" - if (len(child.childNodes) == 1 and - child.childNodes[0].nodeType == dom.TEXT_NODE): - simple = True - value = child.childNodes[0].wholeText - - attrs = dict([(k, v) for k, v in child.attributes.items()]) - - # we accept either UserData or CustomData. If both are present - # then behavior is undefined. - if name == "userdata" or name == "customdata": - if attrs.get('encoding') in (None, "base64"): - ud = base64.b64decode(''.join(value.split())) - else: - ud = value - elif name == "username": - username = value - elif name == "userpassword": - password = value - elif name == "hostname": - md['local-hostname'] = value - elif name == "dscfg": - if attrs.get('encoding') in (None, "base64"): - dscfg = base64.b64decode(''.join(value.split())) - else: - dscfg = value - cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})} - elif name == "ssh": - cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) - elif name == "disablesshpasswordauthentication": - cfg['ssh_pwauth'] = util.is_false(value) - elif simple: - if name in md_props: - md[name] = value - else: - md['azure_data'][name] = value - - defuser = {} - if username: - defuser['name'] = username - if password and DEF_PASSWD_REDACTION != password: - defuser['passwd'] = encrypt_pass(password) - defuser['lock_passwd'] = False - - if defuser: - cfg['system_info'] = {'default_user': defuser} - - if 'ssh_pwauth' not in cfg and password: - cfg['ssh_pwauth'] = True - - return (md, ud, cfg) - - -def encrypt_pass(password, salt_id="$6$"): - return crypt.crypt(password, salt_id + util.rand_str(strlen=16)) - - -def list_possible_azure_ds_devs(): - # return a sorted list of devices that might have a azure datasource - devlist = [] - for fstype in ("iso9660", "udf"): - devlist.extend(util.find_devs_with("TYPE=%s" % fstype)) - - devlist.sort(reverse=True) - return devlist - - -def load_azure_ds_dir(source_dir): - ovf_file = os.path.join(source_dir, "ovf-env.xml") - - if not os.path.isfile(ovf_file): - raise NonAzureDataSource("No ovf-env file found") - - with open(ovf_file, "rb") as fp: - contents = fp.read() - - md, ud, cfg = read_azure_ovf(contents) - return (md, ud, cfg, {'ovf-env.xml': contents}) - - -class BrokenAzureDataSource(Exception): - pass - - -class NonAzureDataSource(Exception): - pass - - -# Used to match classes to dependencies -datasources = [ - (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py deleted file mode 100644 index f80956a5..00000000 --- a/cloudinit/sources/DataSourceBigstep.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (C) 2015-2016 Bigstep Cloud Ltd. -# -# Author: Alexandru Sirbu -# - -import errno -import json - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -class DataSourceBigstep(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = {} - self.vendordata_raw = "" - self.userdata_raw = "" - - def get_data(self, apply_filter=False): - url = get_url_from_file() - if url is None: - return False - response = url_helper.readurl(url) - decoded = json.loads(response.contents) - self.metadata = decoded["metadata"] - self.vendordata_raw = decoded["vendordata_raw"] - self.userdata_raw = decoded["userdata_raw"] - return True - - -def get_url_from_file(): - try: - content = util.load_file("/var/lib/cloud/data/seed/bigstep/url") - except IOError as e: - # If the file doesn't exist, then the server probably isn't a Bigstep - # instance; otherwise, another problem exists which needs investigation - if e.errno == errno.ENOENT: - return None - else: - raise - return content - -# Used to match classes to dependencies -datasources = [ - (DataSourceBigstep, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py deleted file mode 100644 index d1f806d6..00000000 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ /dev/null @@ -1,132 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 CloudSigma -# -# Author: Kiril Vladimiroff -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from base64 import b64decode -import os -import re - -from cloudinit.cs_utils import Cepko - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -class DataSourceCloudSigma(sources.DataSource): - """ - Uses cepko in order to gather the server context from the VM. - - For more information about CloudSigma's Server Context: - http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html - """ - def __init__(self, sys_cfg, distro, paths): - self.cepko = Cepko() - self.ssh_public_key = '' - sources.DataSource.__init__(self, sys_cfg, distro, paths) - - def is_running_in_cloudsigma(self): - """ - Uses dmi data to detect if this instance of cloud-init is running - in the CloudSigma's infrastructure. - """ - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmi data on ARM processors - LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)") - return False - - LOG.debug("determining hypervisor product name via dmi data") - sys_product_name = util.read_dmi_data("system-product-name") - if not sys_product_name: - LOG.debug("system-product-name not available in dmi data") - return False - else: - LOG.debug("detected hypervisor as %s", sys_product_name) - return 'cloudsigma' in sys_product_name.lower() - - LOG.warn("failed to query dmi data for system product name") - return False - - def get_data(self): - """ - Metadata is the whole server context and /meta/cloud-config is used - as userdata. - """ - dsmode = None - if not self.is_running_in_cloudsigma(): - return False - - try: - server_context = self.cepko.all().result - server_meta = server_context['meta'] - except Exception: - # TODO: check for explicit "config on", and then warn - # but since no explicit config is available now, just debug. - LOG.debug("CloudSigma: Unable to read from serial port") - return False - - self.dsmode = self._determine_dsmode( - [server_meta.get('cloudinit-dsmode')]) - if dsmode == sources.DSMODE_DISABLED: - return False - - base64_fields = server_meta.get('base64_fields', '').split(',') - self.userdata_raw = server_meta.get('cloudinit-user-data', "") - if 'cloudinit-user-data' in base64_fields: - self.userdata_raw = b64decode(self.userdata_raw) - if 'cloudinit' in server_context.get('vendor_data', {}): - self.vendordata_raw = server_context["vendor_data"]["cloudinit"] - - self.metadata = server_context - self.ssh_public_key = server_meta['ssh_public_key'] - - return True - - def get_hostname(self, fqdn=False, resolve_ip=False): - """ - Cleans up and uses the server's name if the latter is set. Otherwise - the first part from uuid is being used. - """ - if re.match(r'^[A-Za-z0-9 -_\.]+$', self.metadata['name']): - return self.metadata['name'][:61] - else: - return self.metadata['uuid'].split('-')[0] - - def get_public_ssh_keys(self): - return [self.ssh_public_key] - - def get_instance_id(self): - return self.metadata['uuid'] - - -# Legacy: Must be present in case we load an old pkl object -DataSourceCloudSigmaNet = DataSourceCloudSigma - -# Used to match classes to dependencies. Since this datasource uses the serial -# port network is not really required, so it's okay to load without it, too. -datasources = [ - (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), -] - - -def get_datasource_list(depends): - """ - Return a list of data sources that match this set of dependencies - """ - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py deleted file mode 100644 index 4de1f563..00000000 --- a/cloudinit/sources/DataSourceCloudStack.py +++ /dev/null @@ -1,253 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Cosmin Luta -# Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2012 Gerard Dethier -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Cosmin Luta -# Author: Scott Moser -# Author: Joshua Harlow -# Author: Gerard Dethier -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -from socket import inet_ntoa -from struct import pack -import time - -from cloudinit import ec2_utils as ec2 -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper as uhelp -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -class CloudStackPasswordServerClient(object): - """ - Implements password fetching from the CloudStack password server. - - http://cloudstack-administration.readthedocs.org/ - en/latest/templates.html#adding-password-management-to-your-templates - has documentation about the system. This implementation is following that - found at - https://github.com/shankerbalan/cloudstack-scripts/ - blob/master/cloud-set-guest-password-debian - """ - - def __init__(self, virtual_router_address): - self.virtual_router_address = virtual_router_address - - def _do_request(self, domu_request): - # The password server was in the past, a broken HTTP server, but is now - # fixed. wget handles this seamlessly, so it's easier to shell out to - # that rather than write our own handling code. - output, _ = util.subp([ - 'wget', '--quiet', '--tries', '3', '--timeout', '20', - '--output-document', '-', '--header', - 'DomU_Request: {0}'.format(domu_request), - '{0}:8080'.format(self.virtual_router_address) - ]) - return output.strip() - - def get_password(self): - password = self._do_request('send_my_password') - if password in ['', 'saved_password']: - return None - if password == 'bad_request': - raise RuntimeError('Error when attempting to fetch root password.') - self._do_request('saved_password') - return password - - -class DataSourceCloudStack(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed_dir = os.path.join(paths.seed_dir, 'cs') - # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ - self.api_ver = 'latest' - self.vr_addr = get_vr_address() - if not self.vr_addr: - raise RuntimeError("No virtual router found!") - self.metadata_address = "http://%s/" % (self.vr_addr,) - self.cfg = {} - - def _get_url_settings(self): - mcfg = self.ds_cfg - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(LOG, "Failed to get timeout, using %s", timeout) - - return (max_wait, timeout) - - def wait_for_metadata_service(self): - (max_wait, timeout) = self._get_url_settings() - - urls = [uhelp.combine_url(self.metadata_address, - 'latest/meta-data/instance-id')] - start_time = time.time() - url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) - - if url: - LOG.debug("Using metadata source: '%s'", url) - else: - LOG.critical(("Giving up on waiting for the metadata from %s" - " after %s seconds"), - urls, int(time.time() - start_time)) - - return bool(url) - - def get_config_obj(self): - return self.cfg - - def get_data(self): - seed_ret = {} - if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): - self.userdata_raw = seed_ret['user-data'] - self.metadata = seed_ret['meta-data'] - LOG.debug("Using seeded cloudstack data from: %s", self.seed_dir) - return True - try: - if not self.wait_for_metadata_service(): - return False - start_time = time.time() - self.userdata_raw = ec2.get_instance_userdata( - self.api_ver, self.metadata_address) - self.metadata = ec2.get_instance_metadata(self.api_ver, - self.metadata_address) - LOG.debug("Crawl of metadata service took %s seconds", - int(time.time() - start_time)) - password_client = CloudStackPasswordServerClient(self.vr_addr) - try: - set_password = password_client.get_password() - except Exception: - util.logexc(LOG, - 'Failed to fetch password from virtual router %s', - self.vr_addr) - else: - if set_password: - self.cfg = { - 'ssh_pwauth': True, - 'password': set_password, - 'chpasswd': { - 'expire': False, - }, - } - return True - except Exception: - util.logexc(LOG, 'Failed fetching from metadata service %s', - self.metadata_address) - return False - - def get_instance_id(self): - return self.metadata['instance-id'] - - @property - def availability_zone(self): - return self.metadata['availability-zone'] - - -def get_default_gateway(): - # Returns the default gateway ip address in the dotted format. - lines = util.load_file("/proc/net/route").splitlines() - for line in lines: - items = line.split("\t") - if items[1] == "00000000": - # Found the default route, get the gateway - gw = inet_ntoa(pack(" latest_mtime: - latest_mtime = mtime - latest_file = abs_path - return latest_file - - -def get_vr_address(): - # Get the address of the virtual router via dhcp leases - # see http://bit.ly/T76eKC for documentation on the virtual router. - # If no virtual router is detected, fallback on default gateway. - lease_file = get_latest_lease() - if not lease_file: - LOG.debug("No lease file found, using default gateway") - return get_default_gateway() - - latest_address = None - with open(lease_file, "r") as fd: - for line in fd: - if "dhcp-server-identifier" in line: - words = line.strip(" ;\r\n").split(" ") - if len(words) > 2: - dhcp = words[2] - LOG.debug("Found DHCP identifier %s", dhcp) - latest_address = dhcp - if not latest_address: - # No virtual router found, fallback on default gateway - LOG.debug("No DHCP found, using default gateway") - return get_default_gateway() - return latest_address - - -# Used to match classes to dependencies -datasources = [ - (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py deleted file mode 100644 index 91d6ff13..00000000 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ /dev/null @@ -1,278 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util - -from cloudinit.net import eni - -from cloudinit.sources.helpers import openstack - -LOG = logging.getLogger(__name__) - -# Various defaults/constants... -DEFAULT_IID = "iid-dsconfigdrive" -DEFAULT_MODE = 'pass' -DEFAULT_METADATA = { - "instance-id": DEFAULT_IID, -} -FS_TYPES = ('vfat', 'iso9660') -LABEL_TYPES = ('config-2',) -POSSIBLE_MOUNTS = ('sr', 'cd') -OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS - for i in range(0, 2))) - - -class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) - self.source = None - self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') - self.version = None - self.ec2_metadata = None - self._network_config = None - self.network_json = None - self.network_eni = None - self.known_macs = None - self.files = {} - - def __str__(self): - root = sources.DataSource.__str__(self) - mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) - mstr += "[source=%s]" % (self.source) - return mstr - - def get_data(self): - found = None - md = {} - results = {} - if os.path.isdir(self.seed_dir): - try: - results = read_config_drive(self.seed_dir) - found = self.seed_dir - except openstack.NonReadable: - util.logexc(LOG, "Failed reading config drive from %s", - self.seed_dir) - if not found: - for dev in find_candidate_devs(): - try: - # Set mtype if freebsd and turn off sync - if dev.startswith("/dev/cd"): - mtype = "cd9660" - sync = False - else: - mtype = None - sync = True - results = util.mount_cb(dev, read_config_drive, - mtype=mtype, sync=sync) - found = dev - except openstack.NonReadable: - pass - except util.MountFailedError: - pass - except openstack.BrokenMetadata: - util.logexc(LOG, "Broken config drive: %s", dev) - if found: - break - if not found: - return False - - md = results.get('metadata', {}) - md = util.mergemanydict([md, DEFAULT_METADATA]) - - self.dsmode = self._determine_dsmode( - [results.get('dsmode'), self.ds_cfg.get('dsmode'), - sources.DSMODE_PASS if results['version'] == 1 else None]) - - if self.dsmode == sources.DSMODE_DISABLED: - return False - - prev_iid = get_previous_iid(self.paths) - cur_iid = md['instance-id'] - if prev_iid != cur_iid: - # better would be to handle this centrally, allowing - # the datasource to do something on new instance id - # note, networking is only rendered here if dsmode is DSMODE_PASS - # which means "DISABLED, but render files and networking" - on_first_boot(results, distro=self.distro, - network=self.dsmode == sources.DSMODE_PASS) - - # This is legacy and sneaky. If dsmode is 'pass' then do not claim - # the datasource was used, even though we did run on_first_boot above. - if self.dsmode == sources.DSMODE_PASS: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - self.dsmode) - return False - - self.source = found - self.metadata = md - self.ec2_metadata = results.get('ec2-metadata') - self.userdata_raw = results.get('userdata') - self.version = results['version'] - self.files.update(results.get('files', {})) - - vd = results.get('vendordata') - self.vendordata_pure = vd - try: - self.vendordata_raw = openstack.convert_vendordata_json(vd) - except ValueError as e: - LOG.warn("Invalid content in vendor-data: %s", e) - self.vendordata_raw = None - - # network_config is an /etc/network/interfaces formated file and is - # obsolete compared to networkdata (from network_data.json) but both - # might be present. - self.network_eni = results.get("network_config") - self.network_json = results.get('networkdata') - return True - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still valid - return sources.instance_id_matches_system_uuid(self.get_instance_id()) - - @property - def network_config(self): - if self._network_config is None: - if self.network_json is not None: - LOG.debug("network config provided via network_json") - self._network_config = openstack.convert_net_json( - self.network_json, known_macs=self.known_macs) - elif self.network_eni is not None: - self._network_config = eni.convert_eni_data(self.network_eni) - LOG.debug("network config provided via converted eni data") - else: - LOG.debug("no network configuration available") - return self._network_config - - -def read_config_drive(source_dir): - reader = openstack.ConfigDriveReader(source_dir) - finders = [ - (reader.read_v2, [], {}), - (reader.read_v1, [], {}), - ] - excps = [] - for (functor, args, kwargs) in finders: - try: - return functor(*args, **kwargs) - except openstack.NonReadable as e: - excps.append(e) - raise excps[-1] - - -def get_previous_iid(paths): - # interestingly, for this purpose the "previous" instance-id is the current - # instance-id. cloud-init hasn't moved them over yet as this datasource - # hasn't declared itself found. - fname = os.path.join(paths.get_cpath('data'), 'instance-id') - try: - return util.load_file(fname).rstrip("\n") - except IOError: - return None - - -def on_first_boot(data, distro=None, network=True): - """Performs any first-boot actions using data read from a config-drive.""" - if not isinstance(data, dict): - raise TypeError("Config-drive data expected to be a dict; not %s" - % (type(data))) - if network: - net_conf = data.get("network_config", '') - if net_conf and distro: - LOG.warn("Updating network interfaces from config drive") - distro.apply_network(net_conf) - write_injected_files(data.get('files')) - - -def write_injected_files(files): - if files: - LOG.debug("Writing %s injected files", len(files)) - for (filename, content) in files.items(): - if not filename.startswith(os.sep): - filename = os.sep + filename - try: - util.write_file(filename, content, mode=0o660) - except IOError: - util.logexc(LOG, "Failed writing file: %s", filename) - - -def find_candidate_devs(probe_optical=True): - """Return a list of devices that may contain the config drive. - - The returned list is sorted by search order where the first item has - should be searched first (highest priority) - - config drive v1: - Per documentation, this is "associated as the last available disk on the - instance", and should be VFAT. - Currently, we do not restrict search list to "last available disk" - - config drive v2: - Disk should be: - * either vfat or iso9660 formated - * labeled with 'config-2' - """ - # query optical drive to get it in blkid cache for 2.6 kernels - if probe_optical: - for device in OPTICAL_DEVICES: - try: - util.find_devs_with(path=device) - except util.ProcessExecutionError: - pass - - by_fstype = [] - for fs_type in FS_TYPES: - by_fstype.extend(util.find_devs_with("TYPE=%s" % (fs_type))) - - by_label = [] - for label in LABEL_TYPES: - by_label.extend(util.find_devs_with("LABEL=%s" % (label))) - - # give preference to "last available disk" (vdb over vda) - # note, this is not a perfect rendition of that. - by_fstype.sort(reverse=True) - by_label.sort(reverse=True) - - # combine list of items by putting by-label items first - # followed by fstype items, but with dupes removed - candidates = (by_label + [d for d in by_fstype if d not in by_label]) - - # We are looking for a block device or partition with necessary label or - # an unpartitioned block device (ex sda, not sda1) - devices = [d for d in candidates - if d in by_label or not util.is_partition(d)] - return devices - - -# Legacy: Must be present in case we load an old pkl object -DataSourceConfigDriveNet = DataSourceConfigDrive - -# Used to match classes to dependencies -datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM,)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py deleted file mode 100644 index 44a17a00..00000000 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ /dev/null @@ -1,110 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Neal Shrader -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import ec2_utils -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util - -import functools - - -LOG = logging.getLogger(__name__) - -BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/metadata/v1/', - 'mirrors_url': 'http://mirrors.digitalocean.com/' -} -MD_RETRIES = 0 -MD_TIMEOUT = 1 - - -class DataSourceDigitalOcean(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = dict() - self.ds_cfg = util.mergemanydict([ - util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), - BUILTIN_DS_CONFIG]) - self.metadata_address = self.ds_cfg['metadata_url'] - - if self.ds_cfg.get('retries'): - self.retries = self.ds_cfg['retries'] - else: - self.retries = MD_RETRIES - - if self.ds_cfg.get('timeout'): - self.timeout = self.ds_cfg['timeout'] - else: - self.timeout = MD_TIMEOUT - - def get_data(self): - caller = functools.partial(util.read_file_or_url, - timeout=self.timeout, retries=self.retries) - - def mcaller(url): - return caller(url).contents - - md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address), - base_url=self.metadata_address, - caller=mcaller) - - self.metadata = md.materialize() - - if self.metadata.get('id'): - return True - else: - return False - - def get_userdata_raw(self): - return "\n".join(self.metadata['user-data']) - - def get_vendordata_raw(self): - return "\n".join(self.metadata['vendor-data']) - - def get_public_ssh_keys(self): - public_keys = self.metadata['public-keys'] - if isinstance(public_keys, list): - return public_keys - else: - return [public_keys] - - @property - def availability_zone(self): - return self.metadata['region'] - - def get_instance_id(self): - return self.metadata['id'] - - def get_hostname(self, fqdn=False, resolve_ip=False): - return self.metadata['hostname'] - - def get_package_mirror_info(self): - return self.ds_cfg['mirrors_url'] - - @property - def launch_index(self): - return None - -# Used to match classes to dependencies -datasources = [ - (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py deleted file mode 100644 index 6fe2a0bb..00000000 --- a/cloudinit/sources/DataSourceEc2.py +++ /dev/null @@ -1,211 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import time - -from cloudinit import ec2_utils as ec2 -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper as uhelp -from cloudinit import util - -LOG = logging.getLogger(__name__) - -DEF_MD_URL = "http://169.254.169.254" - -# Which version we are requesting of the ec2 metadata apis -DEF_MD_VERSION = '2009-04-04' - -# Default metadata urls that will be used if none are provided -# They will be checked for 'resolveability' and some of the -# following may be discarded if they do not resolve -DEF_MD_URLS = [DEF_MD_URL, "http://instance-data.:8773"] - - -class DataSourceEc2(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata_address = DEF_MD_URL - self.seed_dir = os.path.join(paths.seed_dir, "ec2") - self.api_ver = DEF_MD_VERSION - - def get_data(self): - seed_ret = {} - if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): - self.userdata_raw = seed_ret['user-data'] - self.metadata = seed_ret['meta-data'] - LOG.debug("Using seeded ec2 data from %s", self.seed_dir) - return True - - try: - if not self.wait_for_metadata_service(): - return False - start_time = time.time() - self.userdata_raw = \ - ec2.get_instance_userdata(self.api_ver, self.metadata_address) - self.metadata = ec2.get_instance_metadata(self.api_ver, - self.metadata_address) - LOG.debug("Crawl of metadata service took %s seconds", - int(time.time() - start_time)) - return True - except Exception: - util.logexc(LOG, "Failed reading from metadata address %s", - self.metadata_address) - return False - - @property - def launch_index(self): - if not self.metadata: - return None - return self.metadata.get('ami-launch-index') - - def get_instance_id(self): - return self.metadata['instance-id'] - - def _get_url_settings(self): - mcfg = self.ds_cfg - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - - timeout = 50 - try: - timeout = max(0, int(mcfg.get("timeout", timeout))) - except Exception: - util.logexc(LOG, "Failed to get timeout, using %s", timeout) - - return (max_wait, timeout) - - def wait_for_metadata_service(self): - mcfg = self.ds_cfg - - (max_wait, timeout) = self._get_url_settings() - if max_wait <= 0: - return False - - # Remove addresses from the list that wont resolve. - mdurls = mcfg.get("metadata_urls", DEF_MD_URLS) - filtered = [x for x in mdurls if util.is_resolvable_url(x)] - - if set(filtered) != set(mdurls): - LOG.debug("Removed the following from metadata urls: %s", - list((set(mdurls) - set(filtered)))) - - if len(filtered): - mdurls = filtered - else: - LOG.warn("Empty metadata url list! using default list") - mdurls = DEF_MD_URLS - - urls = [] - url2base = {} - for url in mdurls: - cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) - urls.append(cur) - url2base[cur] = url - - start_time = time.time() - url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) - - if url: - LOG.debug("Using metadata source: '%s'", url2base[url]) - else: - LOG.critical("Giving up on md from %s after %s seconds", - urls, int(time.time() - start_time)) - - self.metadata_address = url2base.get(url) - return bool(url) - - def device_name_to_device(self, name): - # Consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if 'block-device-mapping' not in self.metadata: - return None - - # Example: - # 'block-device-mapping': - # {'ami': '/dev/sda1', - # 'ephemeral0': '/dev/sdb', - # 'root': '/dev/sda1'} - found = None - bdm = self.metadata['block-device-mapping'] - for (entname, device) in bdm.items(): - if entname == name: - found = device - break - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - found = device - - if found is None: - LOG.debug("Unable to convert %s to a device", name) - return None - - ofound = found - if not found.startswith("/"): - found = "/dev/%s" % found - - if os.path.exists(found): - return found - - remapped = self._remap_device(os.path.basename(found)) - if remapped: - LOG.debug("Remapped device name %s => %s", found, remapped) - return remapped - - # On t1.micro, ephemeral0 will appear in block-device-mapping from - # metadata, but it will not exist on disk (and never will) - # at this point, we've verified that the path did not exist - # in the special case of 'ephemeral0' return None to avoid bogus - # fstab entry (LP: #744019) - if name == "ephemeral0": - return None - return ofound - - @property - def availability_zone(self): - try: - return self.metadata['placement']['availability-zone'] - except KeyError: - return None - - @property - def region(self): - az = self.availability_zone - if az is not None: - return az[:-1] - return None - -# Used to match classes to dependencies -datasources = [ - (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py deleted file mode 100644 index c660a350..00000000 --- a/cloudinit/sources/DataSourceGCE.py +++ /dev/null @@ -1,167 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Vaidas Jablonskis -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -from base64 import b64decode - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper -from cloudinit import util - -LOG = logging.getLogger(__name__) - -BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' -} -REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') - - -class GoogleMetadataFetcher(object): - headers = {'X-Google-Metadata-Request': True} - - def __init__(self, metadata_address): - self.metadata_address = metadata_address - - def get_value(self, path, is_text): - value = None - try: - resp = url_helper.readurl(url=self.metadata_address + path, - headers=self.headers) - except url_helper.UrlError as exc: - msg = "url %s raised exception %s" - LOG.debug(msg, path, exc) - else: - if resp.code == 200: - if is_text: - value = util.decode_binary(resp.contents) - else: - value = resp.contents - else: - LOG.debug("url %s returned code %s", path, resp.code) - return value - - -class DataSourceGCE(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = dict() - self.ds_cfg = util.mergemanydict([ - util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), - BUILTIN_DS_CONFIG]) - self.metadata_address = self.ds_cfg['metadata_url'] - - # GCE takes sshKeys attribute in the format of ':' - # so we have to trim each key to remove the username part - def _trim_key(self, public_key): - try: - index = public_key.index(':') - if index > 0: - return public_key[(index + 1):] - except Exception: - return public_key - - def get_data(self): - # url_map: (our-key, path, required, is_text) - url_map = [ - ('instance-id', ('instance/id',), True, True), - ('availability-zone', ('instance/zone',), True, True), - ('local-hostname', ('instance/hostname',), True, True), - ('public-keys', ('project/attributes/sshKeys', - 'instance/attributes/sshKeys'), False, True), - ('user-data', ('instance/attributes/user-data',), False, False), - ('user-data-encoding', ('instance/attributes/user-data-encoding',), - False, True), - ] - - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable_url(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) - return False - - metadata_fetcher = GoogleMetadataFetcher(self.metadata_address) - # iterate over url_map keys to get metadata items - running_on_gce = False - for (mkey, paths, required, is_text) in url_map: - value = None - for path in paths: - new_value = metadata_fetcher.get_value(path, is_text) - if new_value is not None: - value = new_value - if value: - running_on_gce = True - if required and value is None: - msg = "required key %s returned nothing. not GCE" - if not running_on_gce: - LOG.debug(msg, mkey) - else: - LOG.warn(msg, mkey) - return False - self.metadata[mkey] = value - - if self.metadata['public-keys']: - lines = self.metadata['public-keys'].splitlines() - self.metadata['public-keys'] = [self._trim_key(k) for k in lines] - - if self.metadata['availability-zone']: - self.metadata['availability-zone'] = self.metadata[ - 'availability-zone'].split('/')[-1] - - encoding = self.metadata.get('user-data-encoding') - if encoding: - if encoding == 'base64': - self.metadata['user-data'] = b64decode( - self.metadata['user-data']) - else: - LOG.warn('unknown user-data-encoding: %s, ignoring', encoding) - - return running_on_gce - - @property - def launch_index(self): - # GCE does not provide lauch_index property - return None - - def get_instance_id(self): - return self.metadata['instance-id'] - - def get_public_ssh_keys(self): - return self.metadata['public-keys'] - - def get_hostname(self, fqdn=False, resolve_ip=False): - # GCE has long FDQN's and has asked for short hostnames - return self.metadata['local-hostname'].split('.')[0] - - def get_userdata_raw(self): - return self.metadata['user-data'] - - @property - def availability_zone(self): - return self.metadata['availability-zone'] - - @property - def region(self): - return self.availability_zone.rsplit('-', 1)[0] - -# Used to match classes to dependencies -datasources = [ - (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py deleted file mode 100644 index d828f078..00000000 --- a/cloudinit/sources/DataSourceMAAS.py +++ /dev/null @@ -1,353 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from __future__ import print_function - -import errno -import os -import time - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper -from cloudinit import util - -LOG = logging.getLogger(__name__) -MD_VERSION = "2012-03-01" - -BINARY_FIELDS = ('user-data',) - - -class DataSourceMAAS(sources.DataSource): - """ - DataSourceMAAS reads instance information from MAAS. - Given a config metadata_url, and oauth tokens, it expects to find - files under the root named: - instance-id - user-data - hostname - """ - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.base_url = None - self.seed_dir = os.path.join(paths.seed_dir, 'maas') - self.oauth_helper = self._get_helper() - - def _get_helper(self): - mcfg = self.ds_cfg - # If we are missing token_key, token_secret or consumer_key - # then just do non-authed requests - for required in ('token_key', 'token_secret', 'consumer_key'): - if required not in mcfg: - return url_helper.OauthUrlHelper() - - return url_helper.OauthUrlHelper( - consumer_key=mcfg['consumer_key'], token_key=mcfg['token_key'], - token_secret=mcfg['token_secret'], - consumer_secret=mcfg.get('consumer_secret')) - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [%s]" % (root, self.base_url) - - def get_data(self): - mcfg = self.ds_cfg - - try: - (userdata, metadata) = read_maas_seed_dir(self.seed_dir) - self.userdata_raw = userdata - self.metadata = metadata - self.base_url = self.seed_dir - return True - except MAASSeedDirNone: - pass - except MAASSeedDirMalformed as exc: - LOG.warn("%s was malformed: %s" % (self.seed_dir, exc)) - raise - - # If there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if not url: - return False - - try: - # doing this here actually has a side affect of - # getting oauth time-fix in place. As no where else would - # retry by default, so even if we could fix the timestamp - # we would not. - if not self.wait_for_metadata_service(url): - return False - - self.base_url = url - - (userdata, metadata) = read_maas_seed_url( - self.base_url, read_file_or_url=self.oauth_helper.readurl, - paths=self.paths, retries=1) - self.userdata_raw = userdata - self.metadata = metadata - return True - except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", url) - return False - - def wait_for_metadata_service(self, url): - mcfg = self.ds_cfg - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - if timeout in mcfg: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - LOG.warn("Failed to get timeout, using %s" % timeout) - - starttime = time.time() - check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - urls = [check_url] - url = self.oauth_helper.wait_for_url( - urls=urls, max_wait=max_wait, timeout=timeout) - - if url: - LOG.debug("Using metadata source: '%s'", url) - else: - LOG.critical("Giving up on md from %s after %i seconds", - urls, int(time.time() - starttime)) - - return bool(url) - - -def read_maas_seed_dir(seed_d): - """ - Return user-data and metadata for a maas seed dir in seed_d. - Expected format of seed_d are the following files: - * instance-id - * local-hostname - * user-data - """ - if not os.path.isdir(seed_d): - raise MAASSeedDirNone("%s: not a directory") - - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - for fname in files: - try: - md[fname] = util.load_file(os.path.join(seed_d, fname), - decode=fname not in BINARY_FIELDS) - except IOError as e: - if e.errno != errno.ENOENT: - raise - - return check_seed_contents(md, seed_d) - - -def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, - version=MD_VERSION, paths=None, retries=None): - """ - Read the maas datasource at seed_url. - read_file_or_url is a method that should provide an interface - like util.read_file_or_url - - Expected format of seed_url is are the following files: - * //meta-data/instance-id - * //meta-data/local-hostname - * //user-data - """ - base_url = "%s/%s" % (seed_url, version) - file_order = [ - 'local-hostname', - 'instance-id', - 'public-keys', - 'user-data', - ] - files = { - 'local-hostname': "%s/%s" % (base_url, 'meta-data/local-hostname'), - 'instance-id': "%s/%s" % (base_url, 'meta-data/instance-id'), - 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), - 'user-data': "%s/%s" % (base_url, 'user-data'), - } - - if read_file_or_url is None: - read_file_or_url = util.read_file_or_url - - md = {} - for name in file_order: - url = files.get(name) - if name == 'user-data': - item_retries = 0 - else: - item_retries = retries - - try: - ssl_details = util.fetch_ssl_details(paths) - resp = read_file_or_url(url, retries=item_retries, - timeout=timeout, ssl_details=ssl_details) - if resp.ok(): - if name in BINARY_FIELDS: - md[name] = resp.contents - else: - md[name] = util.decode_binary(resp.contents) - else: - LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.code) - except url_helper.UrlError as e: - if e.code != 404: - raise - return check_seed_contents(md, seed_url) - - -def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a - return for a datasource. - Either return a (userdata, metadata) tuple or - Raise MAASSeedDirMalformed or MAASSeedDirNone - """ - md_required = ('instance-id', 'local-hostname') - if len(content) == 0: - raise MAASSeedDirNone("%s: no data files found" % seed) - - found = list(content.keys()) - missing = [k for k in md_required if k not in found] - if len(missing): - raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - - userdata = content.get('user-data', b"") - md = {} - for (key, val) in content.items(): - if key == 'user-data': - continue - md[key] = val - - return (userdata, md) - - -class MAASSeedDirNone(Exception): - pass - - -class MAASSeedDirMalformed(Exception): - pass - - -# Used to match classes to dependencies -datasources = [ - (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", - default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - cfg = util.read_conf(args.config) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] is None: - creds[key] = cfg[key] - - oauth_helper = url_helper.OauthUrlHelper(**creds) - - def geturl(url): - # the retry is to ensure that oauth timestamp gets fixed - return oauth_helper.readurl(url, retries=1).contents - - def printurl(url): - print("== %s ==\n%s\n" % (url, geturl(url).decode())) - - def crawl(url): - if url.endswith("/"): - for line in geturl(url).decode().splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line)) - elif line == "meta-data": - # meta-data is a dir, it *should* end in a / - crawl("%s%s" % (url, "meta-data/")) - else: - printurl("%s%s" % (url, line)) - else: - printurl(url) - - if args.subcmd == "check-seed": - readurl = oauth_helper.readurl - if args.url[0] == "/" or args.url.startswith("file://"): - readurl = None - (userdata, metadata) = read_maas_seed_url( - args.url, version=args.apiver, read_file_or_url=readurl, - retries=2) - print("=== userdata ===") - print(userdata.decode()) - print("=== metadata ===") - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url) - - main() diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py deleted file mode 100644 index cdc9eef5..00000000 --- a/cloudinit/sources/DataSourceNoCloud.py +++ /dev/null @@ -1,323 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import errno -import os - -from cloudinit import log as logging -from cloudinit.net import eni -from cloudinit import sources -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -class DataSourceNoCloud(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed = None - self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), - os.path.join(paths.seed_dir, 'nocloud-net')] - self.seed_dir = None - self.supported_seed_starts = ("/", "file://") - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) - - def get_data(self): - defaults = { - "instance-id": "nocloud", - "dsmode": self.dsmode, - } - - found = [] - mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", - 'network-config': {}} - - try: - # Parse the kernel command line, getting data passed in - md = {} - if load_cmdline_data(md): - found.append("cmdline") - mydata = _merge_new_seed(mydata, {'meta-data': md}) - except Exception: - util.logexc(LOG, "Unable to parse command line data") - return False - - # Check to see if the seed dir has data. - pp2d_kwargs = {'required': ['user-data', 'meta-data'], - 'optional': ['vendor-data', 'network-config']} - - for path in self.seed_dirs: - try: - seeded = util.pathprefix2dict(path, **pp2d_kwargs) - found.append(path) - LOG.debug("Using seeded data from %s", path) - mydata = _merge_new_seed(mydata, seeded) - break - except ValueError as e: - pass - - # If the datasource config had a 'seedfrom' entry, then that takes - # precedence over a 'seedfrom' that was found in a filesystem - # but not over external media - if self.ds_cfg.get('seedfrom'): - found.append("ds_config_seedfrom") - mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] - - # fields appropriately named can also just come from the datasource - # config (ie, 'user-data', 'meta-data', 'vendor-data' there) - if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: - mydata = _merge_new_seed(mydata, self.ds_cfg) - found.append("ds_config") - - def _pp2d_callback(mp, data): - return util.pathprefix2dict(mp, **data) - - label = self.ds_cfg.get('fs_label', "cidata") - if label is not None: - # Query optical drive to get it in blkid cache for 2.6 kernels - util.find_devs_with(path="/dev/sr0") - util.find_devs_with(path="/dev/sr1") - - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=%s" % label) - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - LOG.debug("Attempting to use data from %s", dev) - - try: - seeded = util.mount_cb(dev, _pp2d_callback, - pp2d_kwargs) - except ValueError as e: - if dev in label_list: - LOG.warn("device %s with label=%s not a" - "valid seed.", dev, label) - continue - - mydata = _merge_new_seed(mydata, seeded) - - LOG.debug("Using data from %s", dev) - found.append(dev) - break - except OSError as e: - if e.errno != errno.ENOENT: - raise - except util.MountFailedError: - util.logexc(LOG, "Failed to mount %s when looking for " - "data", dev) - - # There was no indication on kernel cmdline or data - # in the seeddir suggesting this handler should be used. - if len(found) == 0: - return False - - # The special argument "seedfrom" indicates we should - # attempt to seed the userdata / metadata from its value - # its primarily value is in allowing the user to type less - # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in mydata['meta-data']: - seedfrom = mydata['meta-data']["seedfrom"] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - LOG.debug("Seed from %s not supported by %s", seedfrom, self) - return False - - # This could throw errors, but the user told us to do it - # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - LOG.debug("Using seeded cache data from %s", seedfrom) - - # Values in the command line override those from the seed - mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], - md_seed]) - mydata['user-data'] = ud - found.append(seedfrom) - - # Now that we have exhausted any other places merge in the defaults - mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], - defaults]) - - self.dsmode = self._determine_dsmode( - [mydata['meta-data'].get('dsmode')]) - - if self.dsmode == sources.DSMODE_DISABLED: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - self.dsmode) - return False - - self.seed = ",".join(found) - self.metadata = mydata['meta-data'] - self.userdata_raw = mydata['user-data'] - self.vendordata_raw = mydata['vendor-data'] - self._network_config = mydata['network-config'] - self._network_eni = mydata['meta-data'].get('network-interfaces') - return True - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still valid - # we check kernel command line or files. - current = self.get_instance_id() - if not current: - return None - - # LP: #1568150 need getattr in the case that an old class object - # has been loaded from a pickled file and now executing new source. - dirs = getattr(self, 'seed_dirs', [self.seed_dir]) - quick_id = _quick_read_instance_id(dirs=dirs) - if not quick_id: - return None - return quick_id == current - - @property - def network_config(self): - if self._network_config is None: - if self._network_eni is not None: - self._network_config = eni.convert_eni_data(self._network_eni) - return self._network_config - - -def _quick_read_instance_id(dirs=None): - if dirs is None: - dirs = [] - - iid_key = 'instance-id' - fill = {} - if load_cmdline_data(fill) and iid_key in fill: - return fill[iid_key] - - for d in dirs: - if d is None: - continue - try: - data = util.pathprefix2dict(d, required=['meta-data']) - md = util.load_yaml(data['meta-data']) - if iid_key in md: - return md[iid_key] - except ValueError: - pass - - return None - - -def load_cmdline_data(fill, cmdline=None): - pairs = [("ds=nocloud", sources.DSMODE_LOCAL), - ("ds=nocloud-net", sources.DSMODE_NETWORK)] - for idstr, dsmode in pairs: - if parse_cmdline_data(idstr, fill, cmdline): - # if dsmode was explicitly in the commanad line, then - # prefer it to the dsmode based on the command line id - if 'dsmode' not in fill: - fill['dsmode'] = dsmode - return True - return False - - -# Returns true or false indicating if cmdline indicated -# that this module should be used. Updates dictionary 'fill' -# with data that was found. -# Example cmdline: -# root=LABEL=uec-rootfs ro ds=nocloud -def parse_cmdline_data(ds_id, fill, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - cmdline = " %s " % cmdline - - if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): - return False - - argline = "" - # cmdline can contain: - # ds=nocloud[;key=val;key=val] - for tok in cmdline.split(): - if tok.startswith(ds_id): - argline = tok.split("=", 1) - - # argline array is now 'nocloud' followed optionally by - # a ';' and then key=value pairs also terminated with ';' - tmp = argline[1].split(";") - if len(tmp) > 1: - kvpairs = tmp[1:] - else: - kvpairs = () - - # short2long mapping to save cmdline typing - s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} - for item in kvpairs: - if item == "": - continue - try: - (k, v) = item.split("=", 1) - except Exception: - k = item - v = None - if k in s2l: - k = s2l[k] - fill[k] = v - - return True - - -def _merge_new_seed(cur, seeded): - ret = cur.copy() - - newmd = seeded.get('meta-data', {}) - if not isinstance(seeded['meta-data'], dict): - newmd = util.load_yaml(seeded['meta-data']) - ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd]) - - if seeded.get('network-config'): - ret['network-config'] = util.load_yaml(seeded['network-config']) - - if 'user-data' in seeded: - ret['user-data'] = seeded['user-data'] - if 'vendor-data' in seeded: - ret['vendor-data'] = seeded['vendor-data'] - return ret - - -class DataSourceNoCloudNet(DataSourceNoCloud): - def __init__(self, sys_cfg, distro, paths): - DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) - self.supported_seed_starts = ("http://", "https://", "ftp://") - - -# Used to match classes to dependencies -datasources = [ - (DataSourceNoCloud, (sources.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py deleted file mode 100644 index d1a62b2a..00000000 --- a/cloudinit/sources/DataSourceNone.py +++ /dev/null @@ -1,57 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import log as logging -from cloudinit import sources - -LOG = logging.getLogger(__name__) - - -class DataSourceNone(sources.DataSource): - def __init__(self, sys_cfg, distro, paths, ud_proc=None): - sources.DataSource.__init__(self, sys_cfg, distro, paths, ud_proc) - self.metadata = {} - self.userdata_raw = '' - - def get_data(self): - # If the datasource config has any provided 'fallback' - # userdata or metadata, use it... - if 'userdata_raw' in self.ds_cfg: - self.userdata_raw = self.ds_cfg['userdata_raw'] - if 'metadata' in self.ds_cfg: - self.metadata = self.ds_cfg['metadata'] - return True - - def get_instance_id(self): - return 'iid-datasource-none' - - @property - def is_disconnected(self): - return True - - -# Used to match classes to dependencies -datasources = [ - (DataSourceNone, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), - (DataSourceNone, []), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py deleted file mode 100644 index 43347cfb..00000000 --- a/cloudinit/sources/DataSourceOVF.py +++ /dev/null @@ -1,429 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from xml.dom import minidom - -import base64 -import os -import re -import time - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util - -from cloudinit.sources.helpers.vmware.imc.config \ - import Config -from cloudinit.sources.helpers.vmware.imc.config_file \ - import ConfigFile -from cloudinit.sources.helpers.vmware.imc.config_nic \ - import NicConfigurator -from cloudinit.sources.helpers.vmware.imc.guestcust_error \ - import GuestCustErrorEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_event \ - import GuestCustEventEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_state \ - import GuestCustStateEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( - enable_nics, - get_nics_to_enable, - set_customization_status -) - -LOG = logging.getLogger(__name__) - - -class DataSourceOVF(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed = None - self.seed_dir = os.path.join(paths.seed_dir, 'ovf') - self.environment = None - self.cfg = {} - self.supported_seed_starts = ("/", "file://") - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def get_data(self): - found = [] - md = {} - ud = "" - vmwarePlatformFound = False - vmwareImcConfigFilePath = '' - - defaults = { - "instance-id": "iid-dsovf", - } - - (seedfile, contents) = get_ovf_env(self.paths.seed_dir) - - system_type = util.read_dmi_data("system-product-name") - if system_type is None: - LOG.debug("No system-product-name found") - - if seedfile: - # Found a seed dir - seed = os.path.join(self.paths.seed_dir, seedfile) - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - found.append(seed) - elif system_type and 'vmware' in system_type.lower(): - LOG.debug("VMware Virtualization Platform found") - if not util.get_cfg_option_bool( - self.sys_cfg, "disable_vmware_customization", True): - deployPkgPluginPath = search_file("/usr/lib/vmware-tools", - "libdeployPkgPlugin.so") - if not deployPkgPluginPath: - deployPkgPluginPath = search_file("/usr/lib/open-vm-tools", - "libdeployPkgPlugin.so") - if deployPkgPluginPath: - # When the VM is powered on, the "VMware Tools" daemon - # copies the customization specification file to - # /var/run/vmware-imc directory. cloud-init code needs - # to search for the file in that directory. - vmwareImcConfigFilePath = util.log_time( - logfunc=LOG.debug, - msg="waiting for configuration file", - func=wait_for_imc_cfg_file, - args=("/var/run/vmware-imc", "cust.cfg")) - - if vmwareImcConfigFilePath: - LOG.debug("Found VMware DeployPkg Config File at %s" % - vmwareImcConfigFilePath) - else: - LOG.debug("Did not find VMware DeployPkg Config File Path") - else: - LOG.debug("Customization for VMware platform is disabled.") - - if vmwareImcConfigFilePath: - nics = "" - try: - cf = ConfigFile(vmwareImcConfigFilePath) - conf = Config(cf) - (md, ud, cfg) = read_vmware_imc(conf) - dirpath = os.path.dirname(vmwareImcConfigFilePath) - nics = get_nics_to_enable(dirpath) - except Exception as e: - LOG.debug("Error parsing the customization Config File") - LOG.exception(e) - set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) - return False - finally: - util.del_dir(os.path.dirname(vmwareImcConfigFilePath)) - - try: - LOG.debug("Applying the Network customization") - nicConfigurator = NicConfigurator(conf.nics) - nicConfigurator.configure() - except Exception as e: - LOG.debug("Error applying the Network Configuration") - LOG.exception(e) - set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) - enable_nics(nics) - return False - - vmwarePlatformFound = True - set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_DONE, - GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) - enable_nics(nics) - else: - np = {'iso': transport_iso9660, - 'vmware-guestd': transport_vmware_guestd, } - name = None - for (name, transfunc) in np.items(): - (contents, _dev, _fname) = transfunc() - if contents: - break - if contents: - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - found.append(name) - - # There was no OVF transports found - if len(found) == 0 and not vmwarePlatformFound: - return False - - if 'seedfrom' in md and md['seedfrom']: - seedfrom = md['seedfrom'] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - LOG.debug("Seed from %s not supported by %s", - seedfrom, self) - return False - - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - LOG.debug("Using seeded cache data from %s", seedfrom) - - md = util.mergemanydict([md, md_seed]) - found.append(seedfrom) - - # Now that we have exhausted any other places merge in the defaults - md = util.mergemanydict([md, defaults]) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - self.cfg = cfg - return True - - def get_public_ssh_keys(self): - if 'public-keys' not in self.metadata: - return [] - pks = self.metadata['public-keys'] - if isinstance(pks, (list)): - return pks - else: - return [pks] - - # The data sources' config_obj is a cloud-config formatted - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return self.cfg - - -class DataSourceOVFNet(DataSourceOVF): - def __init__(self, sys_cfg, distro, paths): - DataSourceOVF.__init__(self, sys_cfg, distro, paths) - self.seed_dir = os.path.join(paths.seed_dir, 'ovf-net') - self.supported_seed_starts = ("http://", "https://", "ftp://") - - -def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): - waited = 0 - - while waited < maxwait: - fileFullPath = search_file(dirpath, filename) - if fileFullPath: - return fileFullPath - time.sleep(naplen) - waited += naplen - return None - - -# This will return a dict with some content -# meta-data, user-data, some config -def read_vmware_imc(config): - md = {} - cfg = {} - ud = "" - if config.host_name: - if config.domain_name: - md['local-hostname'] = config.host_name + "." + config.domain_name - else: - md['local-hostname'] = config.host_name - - if config.timezone: - cfg['timezone'] = config.timezone - - return (md, ud, cfg) - - -# This will return a dict with some content -# meta-data, user-data, some config -def read_ovf_environment(contents): - props = get_properties(contents) - md = {} - cfg = {} - ud = "" - cfg_props = ['password'] - md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for (prop, val) in props.items(): - if prop == 'hostname': - prop = "local-hostname" - if prop in md_props: - md[prop] = val - elif prop in cfg_props: - cfg[prop] = val - elif prop == "user-data": - try: - ud = base64.decodestring(val) - except Exception: - ud = val - return (md, ud, cfg) - - -# Returns tuple of filename (in 'dirname', and the contents of the file) -# on "not found", returns 'None' for filename and False for contents -def get_ovf_env(dirname): - env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") - for fname in env_names: - full_fn = os.path.join(dirname, fname) - if os.path.isfile(full_fn): - try: - contents = util.load_file(full_fn) - return (fname, contents) - except Exception: - util.logexc(LOG, "Failed loading ovf file %s", full_fn) - return (None, False) - - -# Transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): - - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" - - devname_regex = os.environ.get(envname, default_regex) - cdmatch = re.compile(devname_regex) - - # Go through mounts to see if it was already mounted - mounts = util.mounts() - for (dev, info) in mounts.items(): - fstype = info['fstype'] - if fstype != "iso9660" and require_iso: - continue - if cdmatch.match(dev[5:]) is None: # take off '/dev/' - continue - mp = info['mountpoint'] - (fname, contents) = get_ovf_env(mp) - if contents is not False: - return (contents, dev, fname) - - if require_iso: - mtype = "iso9660" - else: - mtype = None - - devs = os.listdir("/dev/") - devs.sort() - for dev in devs: - fullp = os.path.join("/dev/", dev) - - if (fullp in mounts or - not cdmatch.match(dev) or os.path.isdir(fullp)): - continue - - try: - # See if we can read anything at all...?? - util.peek_file(fullp, 512) - except IOError: - continue - - try: - (fname, contents) = util.mount_cb(fullp, get_ovf_env, mtype=mtype) - except util.MountFailedError: - LOG.debug("%s not mountable as iso9660" % fullp) - continue - - if contents is not False: - return (contents, fullp, fname) - - return (False, None, None) - - -def transport_vmware_guestd(): - # http://blogs.vmware.com/vapp/2009/07/ \ - # selfconfiguration-and-the-ovf-environment.html - # try: - # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] - # (out, err) = subp(cmd) - # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') - # except: - # # would need to error check here and see why this failed - # # to know if log/error should be raised - # return(False, None, None) - return (False, None, None) - - -def find_child(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return ret - - -def get_properties(contents): - - dom = minidom.parseString(contents) - if dom.documentElement.localName != "Environment": - raise XmlError("No Environment Node") - - if not dom.documentElement.hasChildNodes(): - raise XmlError("No Child Nodes") - - envNsURI = "http://schemas.dmtf.org/ovf/environment/1" - - # could also check here that elem.namespaceURI == - # "http://schemas.dmtf.org/ovf/environment/1" - propSections = find_child(dom.documentElement, - lambda n: n.localName == "PropertySection") - - if len(propSections) == 0: - raise XmlError("No 'PropertySection's") - - props = {} - propElems = find_child(propSections[0], - (lambda n: n.localName == "Property")) - - for elem in propElems: - key = elem.attributes.getNamedItemNS(envNsURI, "key").value - val = elem.attributes.getNamedItemNS(envNsURI, "value").value - props[key] = val - - return props - - -def search_file(dirpath, filename): - if not dirpath or not filename: - return None - - for root, dirs, files in os.walk(dirpath): - if filename in files: - return os.path.join(root, filename) - - return None - - -class XmlError(Exception): - pass - - -# Used to match classes to dependencies -datasources = ( - (DataSourceOVF, (sources.DEP_FILESYSTEM, )), - (DataSourceOVFNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -) - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py deleted file mode 100644 index 7b3a76b9..00000000 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ /dev/null @@ -1,429 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2012-2013 CERIT Scientific Cloud -# Copyright (C) 2012-2013 OpenNebula.org -# Copyright (C) 2014 Consejo Superior de Investigaciones Cientificas -# -# Author: Scott Moser -# Author: Joshua Harlow -# Author: Vlastimil Holer -# Author: Javier Fontan -# Author: Enol Fernandez -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import pwd -import re -import string - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util - - -LOG = logging.getLogger(__name__) - -DEFAULT_IID = "iid-dsopennebula" -DEFAULT_PARSEUSER = 'nobody' -CONTEXT_DISK_FILES = ["context.sh"] - - -class DataSourceOpenNebula(sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed = None - self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) - - def get_data(self): - defaults = {"instance-id": DEFAULT_IID} - results = None - seed = None - - # decide parseuser for context.sh shell reader - parseuser = DEFAULT_PARSEUSER - if 'parseuser' in self.ds_cfg: - parseuser = self.ds_cfg.get('parseuser') - - candidates = [self.seed_dir] - candidates.extend(find_candidate_devs()) - for cdev in candidates: - try: - if os.path.isdir(self.seed_dir): - results = read_context_disk_dir(cdev, asuser=parseuser) - elif cdev.startswith("/dev"): - results = util.mount_cb(cdev, read_context_disk_dir, - data=parseuser) - except NonContextDiskDir: - continue - except BrokenContextDiskDir as exc: - raise exc - except util.MountFailedError: - LOG.warn("%s was not mountable" % cdev) - - if results: - seed = cdev - LOG.debug("found datasource in %s", cdev) - break - - if not seed: - return False - - # merge fetched metadata with datasource defaults - md = results['metadata'] - md = util.mergemanydict([md, defaults]) - - # check for valid user specified dsmode - self.dsmode = self._determine_dsmode( - [results.get('DSMODE'), self.ds_cfg.get('dsmode')]) - - if self.dsmode == sources.DSMODE_DISABLED: - return False - - self.seed = seed - self.network_eni = results.get("network_config") - self.metadata = md - self.userdata_raw = results.get('userdata') - return True - - def get_hostname(self, fqdn=False, resolve_ip=None): - if resolve_ip is None: - if self.dsmode == sources.DSMODE_NETWORK: - resolve_ip = True - else: - resolve_ip = False - return sources.DataSource.get_hostname(self, fqdn, resolve_ip) - - -class NonContextDiskDir(Exception): - pass - - -class BrokenContextDiskDir(Exception): - pass - - -class OpenNebulaNetwork(object): - REG_DEV_MAC = re.compile( - r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', - re.MULTILINE | re.DOTALL) - - def __init__(self, ip, context): - self.ip = ip - self.context = context - self.ifaces = self.get_ifaces() - - def get_ifaces(self): - return self.REG_DEV_MAC.findall(self.ip) - - def mac2ip(self, mac): - components = mac.split(':')[2:] - return [str(int(c, 16)) for c in components] - - def get_ip(self, dev, components): - var_name = dev.upper() + '_IP' - if var_name in self.context: - return self.context[var_name] - else: - return '.'.join(components) - - def get_mask(self, dev): - var_name = dev.upper() + '_MASK' - if var_name in self.context: - return self.context[var_name] - else: - return '255.255.255.0' - - def get_network(self, dev, components): - var_name = dev.upper() + '_NETWORK' - if var_name in self.context: - return self.context[var_name] - else: - return '.'.join(components[:-1]) + '.0' - - def get_gateway(self, dev): - var_name = dev.upper() + '_GATEWAY' - if var_name in self.context: - return self.context[var_name] - else: - return None - - def get_dns(self, dev): - var_name = dev.upper() + '_DNS' - if var_name in self.context: - return self.context[var_name] - else: - return None - - def get_domain(self, dev): - var_name = dev.upper() + '_DOMAIN' - if var_name in self.context: - return self.context[var_name] - else: - return None - - def gen_conf(self): - global_dns = [] - if 'DNS' in self.context: - global_dns.append(self.context['DNS']) - - conf = [] - conf.append('auto lo') - conf.append('iface lo inet loopback') - conf.append('') - - for i in self.ifaces: - dev = i[0] - mac = i[1] - ip_components = self.mac2ip(mac) - - conf.append('auto ' + dev) - conf.append('iface ' + dev + ' inet static') - conf.append(' address ' + self.get_ip(dev, ip_components)) - conf.append(' network ' + self.get_network(dev, ip_components)) - conf.append(' netmask ' + self.get_mask(dev)) - - gateway = self.get_gateway(dev) - if gateway: - conf.append(' gateway ' + gateway) - - domain = self.get_domain(dev) - if domain: - conf.append(' dns-search ' + domain) - - # add global DNS servers to all interfaces - dns = self.get_dns(dev) - if global_dns or dns: - all_dns = global_dns - if dns: - all_dns.append(dns) - conf.append(' dns-nameservers ' + ' '.join(all_dns)) - - conf.append('') - - return "\n".join(conf) - - -def find_candidate_devs(): - """ - Return a list of devices that may contain the context disk. - """ - combined = [] - for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'): - devs = util.find_devs_with(f) - devs.sort() - for d in devs: - if d not in combined: - combined.append(d) - - return combined - - -def switch_user_cmd(user): - return ['sudo', '-u', user] - - -def parse_shell_config(content, keylist=None, bash=None, asuser=None, - switch_user_cb=None): - - if isinstance(bash, str): - bash = [bash] - elif bash is None: - bash = ['bash', '-e'] - - if switch_user_cb is None: - switch_user_cb = switch_user_cmd - - # allvars expands to all existing variables by using '${!x*}' notation - # where x is lower or upper case letters or '_' - allvars = ["${!%s*}" % x for x in string.ascii_letters + "_"] - - keylist_in = keylist - if keylist is None: - keylist = allvars - keylist_in = [] - - setup = '\n'.join(('__v="";', '',)) - - def varprinter(vlist): - # output '\0'.join(['_start_', key=value NULL for vars in vlist] - return '\n'.join(( - 'printf "%s\\0" _start_', - 'for __v in %s; do' % ' '.join(vlist), - ' printf "%s=%s\\0" "$__v" "${!__v}";', - 'done', - '' - )) - - # the rendered 'bcmd' is bash syntax that does - # setup: declare variables we use (so they show up in 'all') - # varprinter(allvars): print all variables known at beginning - # content: execute the provided content - # varprinter(keylist): print all variables known after content - # - # output is then a null terminated array of: - # literal '_start_' - # key=value (for each preset variable) - # literal '_start_' - # key=value (for each post set variable) - bcmd = ('unset IFS\n' + - setup + - varprinter(allvars) + - '{\n%s\n\n:\n} > /dev/null\n' % content + - 'unset IFS\n' + - varprinter(keylist) + "\n") - - cmd = [] - if asuser is not None: - cmd = switch_user_cb(asuser) - - cmd.extend(bash) - - (output, _error) = util.subp(cmd, data=bcmd) - - # exclude vars in bash that change on their own or that we used - excluded = ("RANDOM", "LINENO", "SECONDS", "_", "__v") - preset = {} - ret = {} - target = None - output = output[0:-1] # remove trailing null - - # go through output. First _start_ is for 'preset', second for 'target'. - # Add to target only things were changed and not in volitile - for line in output.split("\x00"): - try: - (key, val) = line.split("=", 1) - if target is preset: - target[key] = val - elif (key not in excluded and - (key in keylist_in or preset.get(key) != val)): - ret[key] = val - except ValueError: - if line != "_start_": - raise - if target is None: - target = preset - elif target is preset: - target = ret - - return ret - - -def read_context_disk_dir(source_dir, asuser=None): - """ - read_context_disk_dir(source_dir): - read source_dir and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a NonContextDiskDir - """ - found = {} - for af in CONTEXT_DISK_FILES: - fn = os.path.join(source_dir, af) - if os.path.isfile(fn): - found[af] = fn - - if not found: - raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - - context = {} - results = {'userdata': None, 'metadata': {}} - - if "context.sh" in found: - if asuser is not None: - try: - pwd.getpwnam(asuser) - except KeyError as e: - raise BrokenContextDiskDir("configured user '%s' " - "does not exist", asuser) - try: - path = os.path.join(source_dir, 'context.sh') - content = util.load_file(path) - context = parse_shell_config(content, asuser=asuser) - except util.ProcessExecutionError as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) - except IOError as e: - raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - else: - raise NonContextDiskDir("Missing context.sh") - - if not context: - return results - - results['metadata'] = context - - # process single or multiple SSH keys - ssh_key_var = None - if "SSH_KEY" in context: - ssh_key_var = "SSH_KEY" - elif "SSH_PUBLIC_KEY" in context: - ssh_key_var = "SSH_PUBLIC_KEY" - - if ssh_key_var: - lines = context.get(ssh_key_var).splitlines() - results['metadata']['public-keys'] = [l for l in lines - if len(l) and not - l.startswith("#")] - - # custom hostname -- try hostname or leave cloud-init - # itself create hostname from IP address later - for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): - if k in context: - results['metadata']['local-hostname'] = context[k] - break - - # raw user data - if "USER_DATA" in context: - results['userdata'] = context["USER_DATA"] - elif "USERDATA" in context: - results['userdata'] = context["USERDATA"] - - # b64decode user data if necessary (default) - if 'userdata' in results: - encoding = context.get('USERDATA_ENCODING', - context.get('USER_DATA_ENCODING')) - if encoding == "base64": - try: - results['userdata'] = util.b64d(results['userdata']) - except TypeError: - LOG.warn("Failed base64 decoding of userdata") - - # generate static /etc/network/interfaces - # only if there are any required context variables - # http://opennebula.org/documentation:rel3.8:cong#network_configuration - for k in context: - if re.match(r'^ETH\d+_IP$', k): - (out, _) = util.subp(['/sbin/ip', 'link']) - net = OpenNebulaNetwork(out, context) - results['network-interfaces'] = net.gen_conf() - break - - return results - - -# Legacy: Must be present in case we load an old pkl object -DataSourceOpenNebulaNet = DataSourceOpenNebula - -# Used to match classes to dependencies -datasources = [ - (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py deleted file mode 100644 index c06d17f3..00000000 --- a/cloudinit/sources/DataSourceOpenStack.py +++ /dev/null @@ -1,168 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2014 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import time - -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper -from cloudinit import util - -from cloudinit.sources.helpers import openstack - -LOG = logging.getLogger(__name__) - -# Various defaults/constants... -DEF_MD_URL = "http://169.254.169.254" -DEFAULT_IID = "iid-dsopenstack" -DEFAULT_METADATA = { - "instance-id": DEFAULT_IID, -} - - -class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): - def __init__(self, sys_cfg, distro, paths): - super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) - self.metadata_address = None - self.ssl_details = util.fetch_ssl_details(self.paths) - self.version = None - self.files = {} - self.ec2_metadata = None - - def __str__(self): - root = sources.DataSource.__str__(self) - mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) - return mstr - - def _get_url_settings(self): - # TODO(harlowja): this is shared with ec2 datasource, we should just - # move it to a shared location instead... - # Note: the defaults here are different though. - - # max_wait < 0 indicates do not wait - max_wait = -1 - timeout = 10 - - try: - max_wait = int(self.ds_cfg.get("max_wait", max_wait)) - except Exception: - util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - - try: - timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) - except Exception: - util.logexc(LOG, "Failed to get timeout, using %s", timeout) - return (max_wait, timeout) - - def wait_for_metadata_service(self): - urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL]) - filtered = [x for x in urls if util.is_resolvable_url(x)] - if set(filtered) != set(urls): - LOG.debug("Removed the following from metadata urls: %s", - list((set(urls) - set(filtered)))) - if len(filtered): - urls = filtered - else: - LOG.warn("Empty metadata url list! using default list") - urls = [DEF_MD_URL] - - md_urls = [] - url2base = {} - for url in urls: - md_url = url_helper.combine_url(url, 'openstack') - md_urls.append(md_url) - url2base[md_url] = url - - (max_wait, timeout) = self._get_url_settings() - start_time = time.time() - avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, - timeout=timeout) - if avail_url: - LOG.debug("Using metadata source: '%s'", url2base[avail_url]) - else: - LOG.debug("Giving up on OpenStack md from %s after %s seconds", - md_urls, int(time.time() - start_time)) - - self.metadata_address = url2base.get(avail_url) - return bool(avail_url) - - def get_data(self, retries=5, timeout=5): - try: - if not self.wait_for_metadata_service(): - return False - except IOError: - return False - - try: - results = util.log_time(LOG.debug, - 'Crawl of openstack metadata service', - read_metadata_service, - args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details, - 'retries': retries, - 'timeout': timeout}) - except openstack.NonReadable: - return False - except (openstack.BrokenMetadata, IOError): - util.logexc(LOG, "Broken metadata address %s", - self.metadata_address) - return False - - self.dsmode = self._determine_dsmode([results.get('dsmode')]) - if self.dsmode == sources.DSMODE_DISABLED: - return False - - md = results.get('metadata', {}) - md = util.mergemanydict([md, DEFAULT_METADATA]) - self.metadata = md - self.ec2_metadata = results.get('ec2-metadata') - self.userdata_raw = results.get('userdata') - self.version = results['version'] - self.files.update(results.get('files', {})) - - vd = results.get('vendordata') - self.vendordata_pure = vd - try: - self.vendordata_raw = openstack.convert_vendordata_json(vd) - except ValueError as e: - LOG.warn("Invalid content in vendor-data: %s", e) - self.vendordata_raw = None - - return True - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still valid - return sources.instance_id_matches_system_uuid(self.get_instance_id()) - - -def read_metadata_service(base_url, ssl_details=None, - timeout=5, retries=5): - reader = openstack.MetadataReader(base_url, ssl_details=ssl_details, - timeout=timeout, retries=retries) - return reader.read_v2() - - -# Used to match classes to dependencies -datasources = [ - (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py deleted file mode 100644 index ccc86883..00000000 --- a/cloudinit/sources/DataSourceSmartOS.py +++ /dev/null @@ -1,781 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# Datasource for provisioning on SmartOS. This works on Joyent -# and public/private Clouds using SmartOS. -# -# SmartOS hosts use a serial console (/dev/ttyS1) on KVM Linux Guests -# The meta-data is transmitted via key/value pairs made by -# requests on the console. For example, to get the hostname, you -# would send "GET hostname" on /dev/ttyS1. -# For Linux Guests running in LX-Brand Zones on SmartOS hosts -# a socket (/native/.zonecontrol/metadata.sock) is used instead -# of a serial console. -# -# Certain behavior is defined by the DataDictionary -# http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html -# Comments with "@datadictionary" are snippets of the definition - -import base64 -import binascii -import json -import os -import random -import re -import socket - -from cloudinit import log as logging -from cloudinit import serial -from cloudinit import sources -from cloudinit import util - -LOG = logging.getLogger(__name__) - -SMARTOS_ATTRIB_MAP = { - # Cloud-init Key : (SmartOS Key, Strip line endings) - 'instance-id': ('sdc:uuid', True), - 'local-hostname': ('hostname', True), - 'public-keys': ('root_authorized_keys', True), - 'user-script': ('user-script', False), - 'legacy-user-data': ('user-data', False), - 'user-data': ('cloud-init:user-data', False), - 'iptables_disable': ('iptables_disable', True), - 'motd_sys_info': ('motd_sys_info', True), - 'availability_zone': ('sdc:datacenter_name', True), - 'vendor-data': ('sdc:vendor-data', False), - 'operator-script': ('sdc:operator-script', False), -} - -SMARTOS_ATTRIB_JSON = { - # Cloud-init Key : (SmartOS Key known JSON) - 'network-data': 'sdc:nics', -} - -SMARTOS_ENV_LX_BRAND = "lx-brand" -SMARTOS_ENV_KVM = "kvm" - -DS_NAME = 'SmartOS' -DS_CFG_PATH = ['datasource', DS_NAME] -NO_BASE64_DECODE = [ - 'iptables_disable', - 'motd_sys_info', - 'root_authorized_keys', - 'sdc:datacenter_name', - 'sdc:uuid' - 'user-data', - 'user-script', -] - -METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' -SERIAL_DEVICE = '/dev/ttyS1' -SERIAL_TIMEOUT = 60 - -# BUILT-IN DATASOURCE CONFIGURATION -# The following is the built-in configuration. If the values -# are not set via the system configuration, then these default -# will be used: -# serial_device: which serial device to use for the meta-data -# serial_timeout: how long to wait on the device -# no_base64_decode: values which are not base64 encoded and -# are fetched directly from SmartOS, not meta-data values -# base64_keys: meta-data keys that are delivered in base64 -# base64_all: with the exclusion of no_base64_decode values, -# treat all meta-data as base64 encoded -# disk_setup: describes how to partition the ephemeral drive -# fs_setup: describes how to format the ephemeral drive -# -BUILTIN_DS_CONFIG = { - 'serial_device': SERIAL_DEVICE, - 'serial_timeout': SERIAL_TIMEOUT, - 'metadata_sockfile': METADATA_SOCKFILE, - 'no_base64_decode': NO_BASE64_DECODE, - 'base64_keys': [], - 'base64_all': False, - 'disk_aliases': {'ephemeral0': '/dev/vdb'}, -} - -BUILTIN_CLOUD_CONFIG = { - 'disk_setup': { - 'ephemeral0': {'table_type': 'mbr', - 'layout': False, - 'overwrite': False} - }, - 'fs_setup': [{'label': 'ephemeral0', - 'filesystem': 'ext3', - 'device': 'ephemeral0'}], -} - -# builtin vendor-data is a boothook that writes a script into -# /var/lib/cloud/scripts/per-boot. *That* script then handles -# executing the 'operator-script' and 'user-script' files -# that cloud-init writes into /var/lib/cloud/instance/data/ -# if they exist. -# -# This is all very indirect, but its done like this so that at -# some point in the future, perhaps cloud-init wouldn't do it at -# all, but rather the vendor actually provide vendor-data that accomplished -# their desires. (That is the point of vendor-data). -# -# cloud-init does cheat a bit, and write the operator-script and user-script -# itself. It could have the vendor-script do that, but it seems better -# to not require the image to contain a tool (mdata-get) to read those -# keys when we have a perfectly good one inside cloud-init. -BUILTIN_VENDOR_DATA = """\ -#cloud-boothook -#!/bin/sh -fname="%(per_boot_d)s/01_smartos_vendor_data.sh" -mkdir -p "${fname%%/*}" -cat > "$fname" <<"END_SCRIPT" -#!/bin/sh -## -# This file is written as part of the default vendor data for SmartOS. -# The SmartOS datasource writes the listed file from the listed metadata key -# sdc:operator-script -> %(operator_script)s -# user-script -> %(user_script)s -# -# You can view content with 'mdata-get ' -# -for script in "%(operator_script)s" "%(user_script)s"; do - [ -x "$script" ] || continue - echo "executing '$script'" 1>&2 - "$script" -done -END_SCRIPT -chmod +x "$fname" -""" - - -# @datadictionary: this is legacy path for placing files from metadata -# per the SmartOS location. It is not preferable, but is done for -# legacy reasons -LEGACY_USER_D = "/var/db" - - -class DataSourceSmartOS(sources.DataSource): - _unset = "_unset" - smartos_type = _unset - md_client = _unset - - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.ds_cfg = util.mergemanydict([ - self.ds_cfg, - util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), - BUILTIN_DS_CONFIG]) - - self.metadata = {} - self.network_data = None - self._network_config = None - - self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - - self._init() - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [client=%s]" % (root, self.md_client) - - def _init(self): - if self.smartos_type == self._unset: - self.smartos_type = get_smartos_environ() - if self.smartos_type is None: - self.md_client = None - - if self.md_client == self._unset: - self.md_client = jmc_client_factory( - smartos_type=self.smartos_type, - metadata_sockfile=self.ds_cfg['metadata_sockfile'], - serial_device=self.ds_cfg['serial_device'], - serial_timeout=self.ds_cfg['serial_timeout']) - - def _set_provisioned(self): - '''Mark the instance provisioning state as successful. - - When run in a zone, the host OS will look for /var/svc/provisioning - to be renamed as /var/svc/provision_success. This should be done - after meta-data is successfully retrieved and from this point - the host considers the provision of the zone to be a success and - keeps the zone running. - ''' - - LOG.debug('Instance provisioning state set as successful') - svc_path = '/var/svc' - if os.path.exists('/'.join([svc_path, 'provisioning'])): - os.rename('/'.join([svc_path, 'provisioning']), - '/'.join([svc_path, 'provision_success'])) - - def get_data(self): - self._init() - - md = {} - ud = "" - - if not self.smartos_type: - LOG.debug("Not running on smartos") - return False - - if not self.md_client.exists(): - LOG.debug("No metadata device '%r' found for SmartOS datasource", - self.md_client) - return False - - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): - smartos_noun, strip = attribute - md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) - - for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): - md[ci_noun] = self.md_client.get_json(smartos_noun) - - # @datadictionary: This key may contain a program that is written - # to a file in the filesystem of the guest on each boot and then - # executed. It may be of any format that would be considered - # executable in the guest instance. - # - # We write 'user-script' and 'operator-script' into the - # instance/data directory. The default vendor-data then handles - # executing them later. - data_d = os.path.join(self.paths.get_cpath(), 'instances', - md['instance-id'], 'data') - user_script = os.path.join(data_d, 'user-script') - u_script_l = "%s/user-script" % LEGACY_USER_D - write_boot_content(md.get('user-script'), content_f=user_script, - link=u_script_l, shebang=True, mode=0o700) - - operator_script = os.path.join(data_d, 'operator-script') - write_boot_content(md.get('operator-script'), - content_f=operator_script, shebang=False, - mode=0o700) - - # @datadictionary: This key has no defined format, but its value - # is written to the file /var/db/mdata-user-data on each boot prior - # to the phase that runs user-script. This file is not to be executed. - # This allows a configuration file of some kind to be injected into - # the machine to be consumed by the user-script when it runs. - u_data = md.get('legacy-user-data') - u_data_f = "%s/mdata-user-data" % LEGACY_USER_D - write_boot_content(u_data, u_data_f) - - # Handle the cloud-init regular meta - if not md['local-hostname']: - md['local-hostname'] = md['instance-id'] - - ud = None - if md['user-data']: - ud = md['user-data'] - - if not md['vendor-data']: - md['vendor-data'] = BUILTIN_VENDOR_DATA % { - 'user_script': user_script, - 'operator_script': operator_script, - 'per_boot_d': os.path.join(self.paths.get_cpath("scripts"), - 'per-boot'), - } - - self.metadata = util.mergemanydict([md, self.metadata]) - self.userdata_raw = ud - self.vendordata_raw = md['vendor-data'] - self.network_data = md['network-data'] - - self._set_provisioned() - return True - - def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) - - def get_config_obj(self): - if self.smartos_type == SMARTOS_ENV_KVM: - return BUILTIN_CLOUD_CONFIG - return {} - - def get_instance_id(self): - return self.metadata['instance-id'] - - @property - def network_config(self): - if self._network_config is None: - if self.network_data is not None: - self._network_config = ( - convert_smartos_network_data(self.network_data)) - return self._network_config - - -class JoyentMetadataFetchException(Exception): - pass - - -class JoyentMetadataClient(object): - """ - A client implementing v2 of the Joyent Metadata Protocol Specification. - - The full specification can be found at - http://eng.joyent.com/mdata/protocol.html - """ - line_regex = re.compile( - r'V2 (?P\d+) (?P[0-9a-f]+)' - r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' - r'( (?P.+))?)') - - def __init__(self, smartos_type=None, fp=None): - if smartos_type is None: - smartos_type = get_smartos_environ() - self.smartos_type = smartos_type - self.fp = fp - - def _checksum(self, body): - return '{0:08x}'.format( - binascii.crc32(body.encode('utf-8')) & 0xffffffff) - - def _get_value_from_frame(self, expected_request_id, frame): - frame_data = self.line_regex.match(frame).groupdict() - if int(frame_data['length']) != len(frame_data['body']): - raise JoyentMetadataFetchException( - 'Incorrect frame length given ({0} != {1}).'.format( - frame_data['length'], len(frame_data['body']))) - expected_checksum = self._checksum(frame_data['body']) - if frame_data['checksum'] != expected_checksum: - raise JoyentMetadataFetchException( - 'Invalid checksum (expected: {0}; got {1}).'.format( - expected_checksum, frame_data['checksum'])) - if frame_data['request_id'] != expected_request_id: - raise JoyentMetadataFetchException( - 'Request ID mismatch (expected: {0}; got {1}).'.format( - expected_request_id, frame_data['request_id'])) - if not frame_data.get('payload', None): - LOG.debug('No value found.') - return None - value = util.b64d(frame_data['payload']) - LOG.debug('Value "%s" found.', value) - return value - - def request(self, rtype, param=None): - request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) - message_body = ' '.join((request_id, rtype,)) - if param: - message_body += ' ' + base64.b64encode(param.encode()).decode() - msg = 'V2 {0} {1} {2}\n'.format( - len(message_body), self._checksum(message_body), message_body) - LOG.debug('Writing "%s" to metadata transport.', msg) - - need_close = False - if not self.fp: - self.open_transport() - need_close = True - - self.fp.write(msg.encode('ascii')) - self.fp.flush() - - response = bytearray() - response.extend(self.fp.read(1)) - while response[-1:] != b'\n': - response.extend(self.fp.read(1)) - - if need_close: - self.close_transport() - - response = response.rstrip().decode('ascii') - LOG.debug('Read "%s" from metadata transport.', response) - - if 'SUCCESS' not in response: - return None - - value = self._get_value_from_frame(request_id, response) - return value - - def get(self, key, default=None, strip=False): - result = self.request(rtype='GET', param=key) - if result is None: - return default - if result and strip: - result = result.strip() - return result - - def get_json(self, key, default=None): - result = self.get(key, default=default) - if result is None: - return default - return json.loads(result) - - def list(self): - result = self.request(rtype='KEYS') - if result: - result = result.split('\n') - return result - - def put(self, key, val): - param = b' '.join([base64.b64encode(i.encode()) - for i in (key, val)]).decode() - return self.request(rtype='PUT', param=param) - - def delete(self, key): - return self.request(rtype='DELETE', param=key) - - def close_transport(self): - if self.fp: - self.fp.close() - self.fp = None - - def __enter__(self): - if self.fp: - return self - self.open_transport() - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close_transport() - return - - def open_transport(self): - raise NotImplementedError - - -class JoyentMetadataSocketClient(JoyentMetadataClient): - def __init__(self, socketpath): - self.socketpath = socketpath - - def open_transport(self): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.socketpath) - self.fp = sock.makefile('rwb') - - def exists(self): - return os.path.exists(self.socketpath) - - def __repr__(self): - return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) - - -class JoyentMetadataSerialClient(JoyentMetadataClient): - def __init__(self, device, timeout=10, smartos_type=None): - super(JoyentMetadataSerialClient, self).__init__(smartos_type) - self.device = device - self.timeout = timeout - - def exists(self): - return os.path.exists(self.device) - - def open_transport(self): - ser = serial.Serial(self.device, timeout=self.timeout) - if not ser.isOpen(): - raise SystemError("Unable to open %s" % self.device) - self.fp = ser - - def __repr__(self): - return "%s(device=%s, timeout=%s)" % ( - self.__class__.__name__, self.device, self.timeout) - - -class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): - """V1 of the protocol was not safe for all values. - Thus, we allowed the user to pass values in as base64 encoded. - Users may still reasonably expect to be able to send base64 data - and have it transparently decoded. So even though the V2 format is - now used, and is safe (using base64 itself), we keep legacy support. - - The way for a user to do this was: - a.) specify 'base64_keys' key whose value is a comma delimited - list of keys that were base64 encoded. - b.) base64_all: string interpreted as a boolean that indicates - if all keys are base64 encoded. - c.) set a key named b64- with a boolean indicating that - is base64 encoded.""" - - def __init__(self, device, timeout=10, smartos_type=None): - s = super(JoyentMetadataLegacySerialClient, self) - s.__init__(device, timeout, smartos_type) - self.base64_keys = None - self.base64_all = None - - def _init_base64_keys(self, reset=False): - if reset: - self.base64_keys = None - self.base64_all = None - - keys = None - if self.base64_all is None: - keys = self.list() - if 'base64_all' in keys: - self.base64_all = util.is_true(self._get("base64_all")) - else: - self.base64_all = False - - if self.base64_all: - # short circuit if base64_all is true - return - - if self.base64_keys is None: - if keys is None: - keys = self.list() - b64_keys = set() - if 'base64_keys' in keys: - b64_keys = set(self._get("base64_keys").split(",")) - - # now add any b64- that has a true value - for key in [k[3:] for k in keys if k.startswith("b64-")]: - if util.is_true(self._get(key)): - b64_keys.add(key) - else: - if key in b64_keys: - b64_keys.remove(key) - - self.base64_keys = b64_keys - - def _get(self, key, default=None, strip=False): - return (super(JoyentMetadataLegacySerialClient, self). - get(key, default=default, strip=strip)) - - def is_b64_encoded(self, key, reset=False): - if key in NO_BASE64_DECODE: - return False - - self._init_base64_keys(reset=reset) - if self.base64_all: - return True - - return key in self.base64_keys - - def get(self, key, default=None, strip=False): - mdefault = object() - val = self._get(key, strip=False, default=mdefault) - if val is mdefault: - return default - - if self.is_b64_encoded(key): - try: - val = base64.b64decode(val.encode()).decode() - # Bogus input produces different errors in Python 2 and 3 - except (TypeError, binascii.Error): - LOG.warn("Failed base64 decoding key '%s': %s", key, val) - - if strip: - val = val.strip() - - return val - - -def jmc_client_factory( - smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, - serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, - uname_version=None): - - if smartos_type is None: - smartos_type = get_smartos_environ(uname_version) - - if smartos_type is None: - return None - elif smartos_type == SMARTOS_ENV_KVM: - return JoyentMetadataLegacySerialClient( - device=serial_device, timeout=serial_timeout, - smartos_type=smartos_type) - elif smartos_type == SMARTOS_ENV_LX_BRAND: - return JoyentMetadataSocketClient(socketpath=metadata_sockfile) - - raise ValueError("Unknown value for smartos_type: %s" % smartos_type) - - -def write_boot_content(content, content_f, link=None, shebang=False, - mode=0o400): - """ - Write the content to content_f. Under the following rules: - 1. If no content, remove the file - 2. Write the content - 3. If executable and no file magic, add it - 4. If there is a link, create it - - @param content: what to write - @param content_f: the file name - @param backup_d: the directory to save the backup at - @param link: if defined, location to create a symlink to - @param shebang: if no file magic, set shebang - @param mode: file mode - - Becuase of the way that Cloud-init executes scripts (no shell), - a script will fail to execute if does not have a magic bit (shebang) set - for the file. If shebang=True, then the script will be checked for a magic - bit and to the SmartOS default of assuming that bash. - """ - - if not content and os.path.exists(content_f): - os.unlink(content_f) - if link and os.path.islink(link): - os.unlink(link) - if not content: - return - - util.write_file(content_f, content, mode=mode) - - if shebang and not content.startswith("#!"): - try: - cmd = ["file", "--brief", "--mime-type", content_f] - (f_type, _err) = util.subp(cmd) - LOG.debug("script %s mime type is %s", content_f, f_type) - if f_type.strip() == "text/plain": - new_content = "\n".join(["#!/bin/bash", content]) - util.write_file(content_f, new_content, mode=mode) - LOG.debug("added shebang to file %s", content_f) - - except Exception as e: - util.logexc(LOG, ("Failed to identify script type for %s" % - content_f, e)) - - if link: - try: - if os.path.islink(link): - os.unlink(link) - if content and os.path.exists(content_f): - util.ensure_dir(os.path.dirname(link)) - os.symlink(content_f, link) - except IOError as e: - util.logexc(LOG, "failed establishing content link: %s", e) - - -def get_smartos_environ(uname_version=None, product_name=None, - uname_arch=None): - uname = os.uname() - if uname_arch is None: - uname_arch = uname[4] - - if uname_arch.startswith("arm") or uname_arch == "aarch64": - return None - - # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but - # report 'BrandZ virtual linux' as the kernel version - if uname_version is None: - uname_version = uname[3] - if uname_version.lower() == 'brandz virtual linux': - return SMARTOS_ENV_LX_BRAND - - if product_name is None: - system_type = util.read_dmi_data("system-product-name") - else: - system_type = product_name - - if system_type and 'smartdc' in system_type.lower(): - return SMARTOS_ENV_KVM - - return None - - -# Covert SMARTOS 'sdc:nics' data to network_config yaml -def convert_smartos_network_data(network_data=None): - """Return a dictionary of network_config by parsing provided - SMARTOS sdc:nics configuration data - - sdc:nics data is a dictionary of properties of a nic and the ip - configuration desired. Additional nic dictionaries are appended - to the list. - - Converting the format is straightforward though it does include - duplicate information as well as data which appears to be relevant - to the hostOS rather than the guest. - - For each entry in the nics list returned from query sdc:nics, we - create a type: physical entry, and extract the interface properties: - 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining - keys are related to ip configuration. For each ip in the 'ips' list - we create a subnet entry under 'subnets' pairing the ip to a one in - the 'gateways' list. - """ - - valid_keys = { - 'physical': [ - 'mac_address', - 'mtu', - 'name', - 'params', - 'subnets', - 'type', - ], - 'subnet': [ - 'address', - 'broadcast', - 'dns_nameservers', - 'dns_search', - 'gateway', - 'metric', - 'netmask', - 'pointopoint', - 'routes', - 'scope', - 'type', - ], - } - - config = [] - for nic in network_data: - cfg = dict((k, v) for k, v in nic.items() - if k in valid_keys['physical']) - cfg.update({ - 'type': 'physical', - 'name': nic['interface']}) - if 'mac' in nic: - cfg.update({'mac_address': nic['mac']}) - - subnets = [] - for ip, gw in zip(nic['ips'], nic['gateways']): - subnet = dict((k, v) for k, v in nic.items() - if k in valid_keys['subnet']) - subnet.update({ - 'type': 'static', - 'address': ip, - 'gateway': gw, - }) - subnets.append(subnet) - cfg.update({'subnets': subnets}) - config.append(cfg) - - return {'version': 1, 'config': config} - - -# Used to match classes to dependencies -datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - -if __name__ == "__main__": - import sys - jmc = jmc_client_factory() - if jmc is None: - print("Do not appear to be on smartos.") - sys.exit(1) - if len(sys.argv) == 1: - keys = (list(SMARTOS_ATTRIB_JSON.keys()) + - list(SMARTOS_ATTRIB_MAP.keys())) - else: - keys = sys.argv[1:] - - data = {} - for key in keys: - if key in SMARTOS_ATTRIB_JSON: - keyname = SMARTOS_ATTRIB_JSON[key] - data[key] = jmc.get_json(keyname) - else: - if key in SMARTOS_ATTRIB_MAP: - keyname, strip = SMARTOS_ATTRIB_MAP[key] - else: - keyname, strip = (key, False) - val = jmc.get(keyname, strip=strip) - data[key] = jmc.get(keyname, strip=strip) - - print(json.dumps(data, indent=1)) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py deleted file mode 100644 index 87b8e524..00000000 --- a/cloudinit/sources/__init__.py +++ /dev/null @@ -1,371 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import abc -import os - -import six - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import type_utils -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.filters import launch_index -from cloudinit.reporting import events - -DSMODE_DISABLED = "disabled" -DSMODE_LOCAL = "local" -DSMODE_NETWORK = "net" -DSMODE_PASS = "pass" - -VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" -DS_PREFIX = 'DataSource' - -LOG = logging.getLogger(__name__) - - -class DataSourceNotFoundException(Exception): - pass - - -@six.add_metaclass(abc.ABCMeta) -class DataSource(object): - - dsmode = DSMODE_NETWORK - - def __init__(self, sys_cfg, distro, paths, ud_proc=None): - self.sys_cfg = sys_cfg - self.distro = distro - self.paths = paths - self.userdata = None - self.metadata = None - self.userdata_raw = None - self.vendordata = None - self.vendordata_raw = None - - # find the datasource config name. - # remove 'DataSource' from classname on front, and remove 'Net' on end. - # Both Foo and FooNet sources expect config in cfg['sources']['Foo'] - name = type_utils.obj_name(self) - if name.startswith(DS_PREFIX): - name = name[len(DS_PREFIX):] - if name.endswith('Net'): - name = name[0:-3] - - self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", name), {}) - if not self.ds_cfg: - self.ds_cfg = {} - - if not ud_proc: - self.ud_proc = ud.UserDataProcessor(self.paths) - else: - self.ud_proc = ud_proc - - def __str__(self): - return type_utils.obj_name(self) - - def get_userdata(self, apply_filter=False): - if self.userdata is None: - self.userdata = self.ud_proc.process(self.get_userdata_raw()) - if apply_filter: - return self._filter_xdata(self.userdata) - return self.userdata - - def get_vendordata(self): - if self.vendordata is None: - self.vendordata = self.ud_proc.process(self.get_vendordata_raw()) - return self.vendordata - - @property - def launch_index(self): - if not self.metadata: - return None - if 'launch-index' in self.metadata: - return self.metadata['launch-index'] - return None - - def _filter_xdata(self, processed_ud): - filters = [ - launch_index.Filter(util.safe_int(self.launch_index)), - ] - new_ud = processed_ud - for f in filters: - new_ud = f.apply(new_ud) - return new_ud - - @property - def is_disconnected(self): - return False - - def get_userdata_raw(self): - return self.userdata_raw - - def get_vendordata_raw(self): - return self.vendordata_raw - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return {} - - def get_public_ssh_keys(self): - return normalize_pubkey_data(self.metadata.get('public-keys')) - - def _remap_device(self, short_name): - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd", "vtb")} - for (nfrom, tlist) in mappings.items(): - if not short_name.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) - if os.path.exists(cand): - return cand - return None - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return None - - def get_locale(self): - return 'en_US.UTF-8' - - @property - def availability_zone(self): - return self.metadata.get('availability-zone', - self.metadata.get('availability_zone')) - - @property - def region(self): - return self.metadata.get('region') - - def get_instance_id(self): - if not self.metadata or 'instance-id' not in self.metadata: - # Return a magic not really instance id string - return "iid-datasource" - return str(self.metadata['instance-id']) - - def get_hostname(self, fqdn=False, resolve_ip=False): - defdomain = "localdomain" - defhost = "localhost" - domain = defdomain - - if not self.metadata or 'local-hostname' not in self.metadata: - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - hostname = util.get_hostname() - fqdn = util.get_fqdn_from_hosts(hostname) - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] - if util.is_ipv4(lhost): - toks = [] - if resolve_ip: - toks = util.gethostbyaddr(lhost) - - if toks: - toks = str(toks).split('.') - else: - toks = ["ip-%s" % lhost.replace(".", "-")] - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = '.'.join(toks[1:]) - else: - hostname = toks[0] - - if fqdn: - return "%s.%s" % (hostname, domain) - else: - return hostname - - def get_package_mirror_info(self): - return self.distro.get_package_mirror_info(data_source=self) - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still - return False - - @staticmethod - def _determine_dsmode(candidates, default=None, valid=None): - # return the first candidate that is non None, warn if not valid - if default is None: - default = DSMODE_NETWORK - - if valid is None: - valid = VALID_DSMODES - - for candidate in candidates: - if candidate is None: - continue - if candidate in valid: - return candidate - else: - LOG.warn("invalid dsmode '%s', using default=%s", - candidate, default) - return default - - return default - - @property - def network_config(self): - return None - - @property - def first_instance_boot(self): - return - - -def normalize_pubkey_data(pubkey_data): - keys = [] - - if not pubkey_data: - return keys - - if isinstance(pubkey_data, six.string_types): - return str(pubkey_data).splitlines() - - if isinstance(pubkey_data, (list, set)): - return list(pubkey_data) - - if isinstance(pubkey_data, (dict)): - for (_keyname, klist) in pubkey_data.items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, six.string_types): - klist = [klist] - if isinstance(klist, (list, set)): - for pkey in klist: - # There is an empty string at - # the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return keys - - -def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): - ds_list = list_sources(cfg_list, ds_deps, pkg_list) - ds_names = [type_utils.obj_name(f) for f in ds_list] - mode = "network" if DEP_NETWORK in ds_deps else "local" - LOG.debug("Searching for %s data source in: %s", mode, ds_names) - - for name, cls in zip(ds_names, ds_list): - myrep = events.ReportEventStack( - name="search-%s" % name.replace("DataSource", ""), - description="searching for %s data from %s" % (mode, name), - message="no %s data found from %s" % (mode, name), - parent=reporter) - try: - with myrep: - LOG.debug("Seeing if we can get any data from %s", cls) - s = cls(sys_cfg, distro, paths) - if s.get_data(): - myrep.message = "found %s data from %s" % (mode, name) - return (s, type_utils.obj_name(cls)) - except Exception: - util.logexc(LOG, "Getting data from %s failed", cls) - - msg = ("Did not find any data source," - " searched classes: (%s)") % (", ".join(ds_names)) - raise DataSourceNotFoundException(msg) - - -# Return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSource*" modules -# and calling their "get_datasource_list". -# Return an ordered list of classes that match (if any) -def list_sources(cfg_list, depends, pkg_list): - src_list = [] - LOG.debug(("Looking for for data source in: %s," - " via packages %s that matches dependencies %s"), - cfg_list, pkg_list, depends) - for ds_name in cfg_list: - if not ds_name.startswith(DS_PREFIX): - ds_name = '%s%s' % (DS_PREFIX, ds_name) - m_locs, _looked_locs = importer.find_module(ds_name, - pkg_list, - ['get_datasource_list']) - for m_loc in m_locs: - mod = importer.import_module(m_loc) - lister = getattr(mod, "get_datasource_list") - matches = lister(depends) - if matches: - src_list.extend(matches) - break - return src_list - - -def instance_id_matches_system_uuid(instance_id, field='system-uuid'): - # quickly (local check only) if self.instance_id is still valid - # we check kernel command line or files. - if not instance_id: - return False - - dmi_value = util.read_dmi_data(field) - if not dmi_value: - return False - return instance_id.lower() == dmi_value.lower() - - -# 'depends' is a list of dependencies (DEP_FILESYSTEM) -# ds_list is a list of 2 item lists -# ds_list = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# It returns a list of 'class' that matched these deps exactly -# It mainly is a helper function for DataSourceCollections -def list_from_depends(depends, ds_list): - ret_list = [] - depset = set(depends) - for (cls, deps) in ds_list: - if depset == set(deps): - ret_list.append(cls) - return ret_list diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py deleted file mode 100644 index 386225d5..00000000 --- a/cloudinit/sources/helpers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py deleted file mode 100644 index 63ccf10e..00000000 --- a/cloudinit/sources/helpers/azure.py +++ /dev/null @@ -1,279 +0,0 @@ -import logging -import os -import re -import socket -import struct -import tempfile -import time - -from contextlib import contextmanager -from xml.etree import ElementTree - -from cloudinit import util - - -LOG = logging.getLogger(__name__) - - -@contextmanager -def cd(newdir): - prevdir = os.getcwd() - os.chdir(os.path.expanduser(newdir)) - try: - yield - finally: - os.chdir(prevdir) - - -class AzureEndpointHttpClient(object): - - headers = { - 'x-ms-agent-name': 'WALinuxAgent', - 'x-ms-version': '2012-11-30', - } - - def __init__(self, certificate): - self.extra_secure_headers = { - "x-ms-cipher-name": "DES_EDE3_CBC", - "x-ms-guest-agent-public-x509-cert": certificate, - } - - def get(self, url, secure=False): - headers = self.headers - if secure: - headers = self.headers.copy() - headers.update(self.extra_secure_headers) - return util.read_file_or_url(url, headers=headers) - - def post(self, url, data=None, extra_headers=None): - headers = self.headers - if extra_headers is not None: - headers = self.headers.copy() - headers.update(extra_headers) - return util.read_file_or_url(url, data=data, headers=headers) - - -class GoalState(object): - - def __init__(self, xml, http_client): - self.http_client = http_client - self.root = ElementTree.fromstring(xml) - self._certificates_xml = None - - def _text_from_xpath(self, xpath): - element = self.root.find(xpath) - if element is not None: - return element.text - return None - - @property - def container_id(self): - return self._text_from_xpath('./Container/ContainerId') - - @property - def incarnation(self): - return self._text_from_xpath('./Incarnation') - - @property - def instance_id(self): - return self._text_from_xpath( - './Container/RoleInstanceList/RoleInstance/InstanceId') - - @property - def certificates_xml(self): - if self._certificates_xml is None: - url = self._text_from_xpath( - './Container/RoleInstanceList/RoleInstance' - '/Configuration/Certificates') - if url is not None: - self._certificates_xml = self.http_client.get( - url, secure=True).contents - return self._certificates_xml - - -class OpenSSLManager(object): - - certificate_names = { - 'private_key': 'TransportPrivate.pem', - 'certificate': 'TransportCert.pem', - } - - def __init__(self): - self.tmpdir = tempfile.mkdtemp() - self.certificate = None - self.generate_certificate() - - def clean_up(self): - util.del_dir(self.tmpdir) - - def generate_certificate(self): - LOG.debug('Generating certificate for communication with fabric...') - if self.certificate is not None: - LOG.debug('Certificate already generated.') - return - with cd(self.tmpdir): - util.subp([ - 'openssl', 'req', '-x509', '-nodes', '-subj', - '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', - '-keyout', self.certificate_names['private_key'], - '-out', self.certificate_names['certificate'], - ]) - certificate = '' - for line in open(self.certificate_names['certificate']): - if "CERTIFICATE" not in line: - certificate += line.rstrip() - self.certificate = certificate - LOG.debug('New certificate generated.') - - def parse_certificates(self, certificates_xml): - tag = ElementTree.fromstring(certificates_xml).find( - './/Data') - certificates_content = tag.text - lines = [ - b'MIME-Version: 1.0', - b'Content-Disposition: attachment; filename="Certificates.p7m"', - b'Content-Type: application/x-pkcs7-mime; name="Certificates.p7m"', - b'Content-Transfer-Encoding: base64', - b'', - certificates_content.encode('utf-8'), - ] - with cd(self.tmpdir): - with open('Certificates.p7m', 'wb') as f: - f.write(b'\n'.join(lines)) - out, _ = util.subp( - 'openssl cms -decrypt -in Certificates.p7m -inkey' - ' {private_key} -recip {certificate} | openssl pkcs12 -nodes' - ' -password pass:'.format(**self.certificate_names), - shell=True) - private_keys, certificates = [], [] - current = [] - for line in out.splitlines(): - current.append(line) - if re.match(r'[-]+END .*?KEY[-]+$', line): - private_keys.append('\n'.join(current)) - current = [] - elif re.match(r'[-]+END .*?CERTIFICATE[-]+$', line): - certificates.append('\n'.join(current)) - current = [] - keys = [] - for certificate in certificates: - with cd(self.tmpdir): - public_key, _ = util.subp( - 'openssl x509 -noout -pubkey |' - 'ssh-keygen -i -m PKCS8 -f /dev/stdin', - data=certificate, - shell=True) - keys.append(public_key) - return keys - - -class WALinuxAgentShim(object): - - REPORT_READY_XML_TEMPLATE = '\n'.join([ - '', - '', - ' {incarnation}', - ' ', - ' {container_id}', - ' ', - ' ', - ' {instance_id}', - ' ', - ' Ready', - ' ', - ' ', - ' ', - ' ', - '']) - - def __init__(self): - LOG.debug('WALinuxAgentShim instantiated...') - self.endpoint = self.find_endpoint() - self.openssl_manager = None - self.values = {} - - def clean_up(self): - if self.openssl_manager is not None: - self.openssl_manager.clean_up() - - @staticmethod - def get_ip_from_lease_value(lease_value): - unescaped_value = lease_value.replace('\\', '') - if len(unescaped_value) > 4: - hex_string = '' - for hex_pair in unescaped_value.split(':'): - if len(hex_pair) == 1: - hex_pair = '0' + hex_pair - hex_string += hex_pair - packed_bytes = struct.pack( - '>L', int(hex_string.replace(':', ''), 16)) - else: - packed_bytes = unescaped_value.encode('utf-8') - return socket.inet_ntoa(packed_bytes) - - @staticmethod - def find_endpoint(): - LOG.debug('Finding Azure endpoint...') - content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') - value = None - for line in content.splitlines(): - if 'unknown-245' in line: - value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') - if value is None: - raise ValueError('No endpoint found in DHCP config.') - endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) - LOG.debug('Azure endpoint found at %s', endpoint_ip_address) - return endpoint_ip_address - - def register_with_azure_and_fetch_data(self): - self.openssl_manager = OpenSSLManager() - http_client = AzureEndpointHttpClient(self.openssl_manager.certificate) - LOG.info('Registering with Azure...') - attempts = 0 - while True: - try: - response = http_client.get( - 'http://{0}/machine/?comp=goalstate'.format(self.endpoint)) - except Exception: - if attempts < 10: - time.sleep(attempts + 1) - else: - raise - else: - break - attempts += 1 - LOG.debug('Successfully fetched GoalState XML.') - goal_state = GoalState(response.contents, http_client) - public_keys = [] - if goal_state.certificates_xml is not None: - LOG.debug('Certificate XML found; parsing out public keys.') - public_keys = self.openssl_manager.parse_certificates( - goal_state.certificates_xml) - data = { - 'public-keys': public_keys, - } - self._report_ready(goal_state, http_client) - return data - - def _report_ready(self, goal_state, http_client): - LOG.debug('Reporting ready to Azure fabric.') - document = self.REPORT_READY_XML_TEMPLATE.format( - incarnation=goal_state.incarnation, - container_id=goal_state.container_id, - instance_id=goal_state.instance_id, - ) - http_client.post( - "http://{0}/machine?comp=health".format(self.endpoint), - data=document, - extra_headers={'Content-Type': 'text/xml; charset=utf-8'}, - ) - LOG.info('Reported ready to Azure fabric.') - - -def get_metadata_from_fabric(): - shim = WALinuxAgentShim() - try: - return shim.register_with_azure_and_fetch_data() - finally: - shim.clean_up() diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py deleted file mode 100644 index 2e7a1d47..00000000 --- a/cloudinit/sources/helpers/openstack.py +++ /dev/null @@ -1,648 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import abc -import base64 -import copy -import functools -import os - -import six - -from cloudinit import ec2_utils -from cloudinit import log as logging -from cloudinit import net -from cloudinit import sources -from cloudinit import url_helper -from cloudinit import util - -# For reference: http://tinyurl.com/laora4c - -LOG = logging.getLogger(__name__) - -FILES_V1 = { - # Path <-> (metadata key name, translator function, default value) - 'etc/network/interfaces': ('network_config', lambda x: x, ''), - 'meta.js': ('meta_js', util.load_json, {}), - "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''), -} -KEY_COPIES = ( - # Cloud-init metadata names <-> (metadata key, is required) - ('local-hostname', 'hostname', False), - ('instance-id', 'uuid', True), -) -OS_LATEST = 'latest' -OS_FOLSOM = '2012-08-10' -OS_GRIZZLY = '2013-04-04' -OS_HAVANA = '2013-10-17' -OS_LIBERTY = '2015-10-15' -# keep this in chronological order. new supported versions go at the end. -OS_VERSIONS = ( - OS_FOLSOM, - OS_GRIZZLY, - OS_HAVANA, - OS_LIBERTY, -) - - -class NonReadable(IOError): - pass - - -class BrokenMetadata(IOError): - pass - - -class SourceMixin(object): - def _ec2_name_to_device(self, name): - if not self.ec2_metadata: - return None - bdm = self.ec2_metadata.get('block-device-mapping', {}) - for (ent_name, device) in bdm.items(): - if name == ent_name: - return device - return None - - def get_public_ssh_keys(self): - name = "public_keys" - if self.version == 1: - name = "public-keys" - return sources.normalize_pubkey_data(self.metadata.get(name)) - - def _os_name_to_device(self, name): - device = None - try: - criteria = 'LABEL=%s' % (name) - if name == 'swap': - criteria = 'TYPE=%s' % (name) - dev_entries = util.find_devs_with(criteria) - if dev_entries: - device = dev_entries[0] - except util.ProcessExecutionError: - pass - return device - - def _validate_device_name(self, device): - if not device: - return None - if not device.startswith("/"): - device = "/dev/%s" % device - if os.path.exists(device): - return device - # Durn, try adjusting the mapping - remapped = self._remap_device(os.path.basename(device)) - if remapped: - LOG.debug("Remapped device name %s => %s", device, remapped) - return remapped - return None - - def device_name_to_device(self, name): - # Translate a 'name' to a 'physical' device - if not name: - return None - # Try the ec2 mapping first - names = [name] - if name == 'root': - names.insert(0, 'ami') - if name == 'ami': - names.append('root') - device = None - LOG.debug("Using ec2 style lookup to find device %s", names) - for n in names: - device = self._ec2_name_to_device(n) - device = self._validate_device_name(device) - if device: - break - # Try the openstack way second - if not device: - LOG.debug("Using openstack style lookup to find device %s", names) - for n in names: - device = self._os_name_to_device(n) - device = self._validate_device_name(device) - if device: - break - # Ok give up... - if not device: - return None - else: - LOG.debug("Mapped %s to device %s", name, device) - return device - - -@six.add_metaclass(abc.ABCMeta) -class BaseReader(object): - - def __init__(self, base_path): - self.base_path = base_path - - @abc.abstractmethod - def _path_join(self, base, *add_ons): - pass - - @abc.abstractmethod - def _path_read(self, path, decode=False): - pass - - @abc.abstractmethod - def _fetch_available_versions(self): - pass - - @abc.abstractmethod - def _read_ec2_metadata(self): - pass - - def _find_working_version(self): - try: - versions_available = self._fetch_available_versions() - except Exception as e: - LOG.debug("Unable to read openstack versions from %s due to: %s", - self.base_path, e) - versions_available = [] - - # openstack.OS_VERSIONS is stored in chronological order, so - # reverse it to check newest first. - supported = [v for v in reversed(list(OS_VERSIONS))] - selected_version = OS_LATEST - - for potential_version in supported: - if potential_version not in versions_available: - continue - selected_version = potential_version - break - - LOG.debug("Selected version '%s' from %s", selected_version, - versions_available) - return selected_version - - def _read_content_path(self, item, decode=False): - path = item.get('content_path', '').lstrip("/") - path_pieces = path.split("/") - valid_pieces = [p for p in path_pieces if len(p)] - if not valid_pieces: - raise BrokenMetadata("Item %s has no valid content path" % (item)) - path = self._path_join(self.base_path, "openstack", *path_pieces) - return self._path_read(path, decode=decode) - - def read_v2(self): - """Reads a version 2 formatted location. - - Return a dict with metadata, userdata, ec2-metadata, dsmode, - network_config, files and version (2). - - If not a valid location, raise a NonReadable exception. - """ - - load_json_anytype = functools.partial( - util.load_json, root_types=(dict, list) + six.string_types) - - def datafiles(version): - files = {} - files['metadata'] = ( - # File path to read - self._path_join("openstack", version, 'meta_data.json'), - # Is it required? - True, - # Translator function (applied after loading) - util.load_json, - ) - files['userdata'] = ( - self._path_join("openstack", version, 'user_data'), - False, - lambda x: x, - ) - files['vendordata'] = ( - self._path_join("openstack", version, 'vendor_data.json'), - False, - load_json_anytype, - ) - files['networkdata'] = ( - self._path_join("openstack", version, 'network_data.json'), - False, - load_json_anytype, - ) - return files - - results = { - 'userdata': '', - 'version': 2, - } - data = datafiles(self._find_working_version()) - for (name, (path, required, translator)) in data.items(): - path = self._path_join(self.base_path, path) - data = None - found = False - try: - data = self._path_read(path) - except IOError as e: - if not required: - LOG.debug("Failed reading optional path %s due" - " to: %s", path, e) - else: - LOG.debug("Failed reading mandatory path %s due" - " to: %s", path, e) - else: - found = True - if required and not found: - raise NonReadable("Missing mandatory path: %s" % path) - if found and translator: - try: - data = translator(data) - except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) - if found: - results[name] = data - - metadata = results['metadata'] - if 'random_seed' in metadata: - random_seed = metadata['random_seed'] - try: - metadata['random_seed'] = base64.b64decode(random_seed) - except (ValueError, TypeError) as e: - raise BrokenMetadata("Badly formatted metadata" - " random_seed entry: %s" % e) - - # load any files that were provided - files = {} - metadata_files = metadata.get('files', []) - for item in metadata_files: - if 'path' not in item: - continue - path = item['path'] - try: - files[path] = self._read_content_path(item) - except Exception as e: - raise BrokenMetadata("Failed to read provided " - "file %s: %s" % (path, e)) - results['files'] = files - - # The 'network_config' item in metadata is a content pointer - # to the network config that should be applied. It is just a - # ubuntu/debian '/etc/network/interfaces' file. - net_item = metadata.get("network_config", None) - if net_item: - try: - content = self._read_content_path(net_item, decode=True) - results['network_config'] = content - except IOError as e: - raise BrokenMetadata("Failed to read network" - " configuration: %s" % (e)) - - # To openstack, user can specify meta ('nova boot --meta=key=value') - # and those will appear under metadata['meta']. - # if they specify 'dsmode' they're indicating the mode that they intend - # for this datasource to operate in. - try: - results['dsmode'] = metadata['meta']['dsmode'] - except KeyError: - pass - - # Read any ec2-metadata (if applicable) - results['ec2-metadata'] = self._read_ec2_metadata() - - # Perform some misc. metadata key renames... - for (target_key, source_key, is_required) in KEY_COPIES: - if is_required and source_key not in metadata: - raise BrokenMetadata("No '%s' entry in metadata" % source_key) - if source_key in metadata: - metadata[target_key] = metadata.get(source_key) - return results - - -class ConfigDriveReader(BaseReader): - def __init__(self, base_path): - super(ConfigDriveReader, self).__init__(base_path) - self._versions = None - - def _path_join(self, base, *add_ons): - components = [base] + list(add_ons) - return os.path.join(*components) - - def _path_read(self, path, decode=False): - return util.load_file(path, decode=decode) - - def _fetch_available_versions(self): - if self._versions is None: - path = self._path_join(self.base_path, 'openstack') - found = [d for d in os.listdir(path) - if os.path.isdir(os.path.join(path))] - self._versions = sorted(found) - return self._versions - - def _read_ec2_metadata(self): - path = self._path_join(self.base_path, - 'ec2', 'latest', 'meta-data.json') - if not os.path.exists(path): - return {} - else: - try: - return util.load_json(self._path_read(path)) - except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) - - def read_v1(self): - """Reads a version 1 formatted location. - - Return a dict with metadata, userdata, dsmode, files and version (1). - - If not a valid path, raise a NonReadable exception. - """ - - found = {} - for name in FILES_V1.keys(): - path = self._path_join(self.base_path, name) - if os.path.exists(path): - found[name] = path - if len(found) == 0: - raise NonReadable("%s: no files found" % (self.base_path)) - - md = {} - for (name, (key, translator, default)) in FILES_V1.items(): - if name in found: - path = found[name] - try: - contents = self._path_read(path) - except IOError: - raise BrokenMetadata("Failed to read: %s" % path) - try: - md[key] = translator(contents) - except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) - else: - md[key] = copy.deepcopy(default) - - keydata = md['authorized_keys'] - meta_js = md['meta_js'] - - # keydata in meta_js is preferred over "injected" - keydata = meta_js.get('public-keys', keydata) - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - # config-drive-v1 has no way for openstack to provide the instance-id - # so we copy that into metadata from the user input - if 'instance-id' in meta_js: - md['instance-id'] = meta_js['instance-id'] - - results = { - 'version': 1, - 'metadata': md, - } - - # allow the user to specify 'dsmode' in a meta tag - if 'dsmode' in meta_js: - results['dsmode'] = meta_js['dsmode'] - - # config-drive-v1 has no way of specifying user-data, so the user has - # to cheat and stuff it in a meta tag also. - results['userdata'] = meta_js.get('user-data', '') - - # this implementation does not support files other than - # network/interfaces and authorized_keys... - results['files'] = {} - - return results - - -class MetadataReader(BaseReader): - def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): - super(MetadataReader, self).__init__(base_url) - self.ssl_details = ssl_details - self.timeout = float(timeout) - self.retries = int(retries) - self._versions = None - - def _fetch_available_versions(self): - # /openstack/ returns a newline separated list of versions - if self._versions is not None: - return self._versions - found = [] - version_path = self._path_join(self.base_path, "openstack") - content = self._path_read(version_path) - for line in content.splitlines(): - line = line.strip() - if not line: - continue - found.append(line) - self._versions = found - return self._versions - - def _path_read(self, path, decode=False): - - def should_retry_cb(_request_args, cause): - try: - code = int(cause.code) - if code >= 400: - return False - except (TypeError, ValueError): - # Older versions of requests didn't have a code. - pass - return True - - response = url_helper.readurl(path, - retries=self.retries, - ssl_details=self.ssl_details, - timeout=self.timeout, - exception_cb=should_retry_cb) - if decode: - return response.contents.decode() - else: - return response.contents - - def _path_join(self, base, *add_ons): - return url_helper.combine_url(base, *add_ons) - - def _read_ec2_metadata(self): - return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, - timeout=self.timeout, - retries=self.retries) - - -# Convert OpenStack ConfigDrive NetworkData json to network_config yaml -def convert_net_json(network_json=None, known_macs=None): - """Return a dictionary of network_config by parsing provided - OpenStack ConfigDrive NetworkData json format - - OpenStack network_data.json provides a 3 element dictionary - - "links" (links are network devices, physical or virtual) - - "networks" (networks are ip network configurations for one or more - links) - - services (non-ip services, like dns) - - networks and links are combined via network items referencing specific - links via a 'link_id' which maps to a links 'id' field. - - To convert this format to network_config yaml, we first iterate over the - links and then walk the network list to determine if any of the networks - utilize the current link; if so we generate a subnet entry for the device - - We also need to map network_data.json fields to network_config fields. For - example, the network_data links 'id' field is equivalent to network_config - 'name' field for devices. We apply more of this mapping to the various - link types that we encounter. - - There are additional fields that are populated in the network_data.json - from OpenStack that are not relevant to network_config yaml, so we - enumerate a dictionary of valid keys for network_yaml and apply filtering - to drop these superflous keys from the network_config yaml. - """ - if network_json is None: - return None - - # dict of network_config key for filtering network_json - valid_keys = { - 'physical': [ - 'name', - 'type', - 'mac_address', - 'subnets', - 'params', - 'mtu', - ], - 'subnet': [ - 'type', - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'scope', - 'dns_nameservers', - 'dns_search', - 'routes', - ], - } - - links = network_json.get('links', []) - networks = network_json.get('networks', []) - services = network_json.get('services', []) - - config = [] - for link in links: - subnets = [] - cfg = dict((k, v) for k, v in link.items() - if k in valid_keys['physical']) - # 'name' is not in openstack spec yet, but we will support it if it is - # present. The 'id' in the spec is currently implemented as the host - # nic's name, meaning something like 'tap-adfasdffd'. We do not want - # to name guest devices with such ugly names. - if 'name' in link: - cfg['name'] = link['name'] - - for network in [n for n in networks - if n['link'] == link['id']]: - subnet = dict((k, v) for k, v in network.items() - if k in valid_keys['subnet']) - if 'dhcp' in network['type']: - t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' - subnet.update({ - 'type': t, - }) - else: - subnet.update({ - 'type': 'static', - 'address': network.get('ip_address'), - }) - if network['type'] == 'ipv4': - subnet['ipv4'] = True - if network['type'] == 'ipv6': - subnet['ipv6'] = True - subnets.append(subnet) - cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']: - cfg.update({ - 'type': 'physical', - 'mac_address': link['ethernet_mac_address']}) - elif link['type'] in ['bond']: - params = {} - for k, v in link.items(): - if k == 'bond_links': - continue - elif k.startswith('bond'): - params.update({k: v}) - cfg.update({ - 'bond_interfaces': copy.deepcopy(link['bond_links']), - 'params': params, - }) - elif link['type'] in ['vlan']: - cfg.update({ - 'name': "%s.%s" % (link['vlan_link'], - link['vlan_id']), - 'vlan_link': link['vlan_link'], - 'vlan_id': link['vlan_id'], - 'mac_address': link['vlan_mac_address'], - }) - else: - raise ValueError( - 'Unknown network_data link type: %s' % link['type']) - - config.append(cfg) - - need_names = [d for d in config - if d.get('type') == 'physical' and 'name' not in d] - - if need_names: - if known_macs is None: - known_macs = net.get_interfaces_by_mac() - - for d in need_names: - mac = d.get('mac_address') - if not mac: - raise ValueError("No mac_address or name entry for %s" % d) - if mac not in known_macs: - raise ValueError("Unable to find a system nic for %s" % d) - d['name'] = known_macs[mac] - - for service in services: - cfg = service - cfg.update({'type': 'nameserver'}) - config.append(cfg) - - return {'version': 1, 'config': config} - - -def convert_vendordata_json(data, recurse=True): - """data: a loaded json *object* (strings, arrays, dicts). - return something suitable for cloudinit vendordata_raw. - - if data is: - None: return None - string: return string - list: return data - the list is then processed in UserDataProcessor - dict: return convert_vendordata_json(data.get('cloud-init')) - """ - if not data: - return None - if isinstance(data, six.string_types): - return data - if isinstance(data, list): - return copy.deepcopy(data) - if isinstance(data, dict): - if recurse is True: - return convert_vendordata_json(data.get('cloud-init'), - recurse=False) - raise ValueError("vendordata['cloud-init'] cannot be dict") - raise ValueError("Unknown data type for vendordata: %s" % type(data)) diff --git a/cloudinit/sources/helpers/vmware/__init__.py b/cloudinit/sources/helpers/vmware/__init__.py deleted file mode 100644 index 386225d5..00000000 --- a/cloudinit/sources/helpers/vmware/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/vmware/imc/__init__.py b/cloudinit/sources/helpers/vmware/imc/__init__.py deleted file mode 100644 index 386225d5..00000000 --- a/cloudinit/sources/helpers/vmware/imc/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py deleted file mode 100644 index fb53ec1d..00000000 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ /dev/null @@ -1,25 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class BootProtoEnum(object): - """Specifies the NIC Boot Settings.""" - - DHCP = 'dhcp' - STATIC = 'static' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py deleted file mode 100644 index d645c497..00000000 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ /dev/null @@ -1,95 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from .nic import Nic - - -class Config(object): - """ - Stores the Contents specified in the Customization - Specification file. - """ - - DNS = 'DNS|NAMESERVER|' - SUFFIX = 'DNS|SUFFIX|' - PASS = 'PASSWORD|-PASS' - TIMEZONE = 'DATETIME|TIMEZONE' - UTC = 'DATETIME|UTC' - HOSTNAME = 'NETWORK|HOSTNAME' - DOMAINNAME = 'NETWORK|DOMAINNAME' - - def __init__(self, configFile): - self._configFile = configFile - - @property - def host_name(self): - """Return the hostname.""" - return self._configFile.get(Config.HOSTNAME, None) - - @property - def domain_name(self): - """Return the domain name.""" - return self._configFile.get(Config.DOMAINNAME, None) - - @property - def timezone(self): - """Return the timezone.""" - return self._configFile.get(Config.TIMEZONE, None) - - @property - def utc(self): - """Retrieves whether to set time to UTC or Local.""" - return self._configFile.get(Config.UTC, None) - - @property - def admin_password(self): - """Return the root password to be set.""" - return self._configFile.get(Config.PASS, None) - - @property - def name_servers(self): - """Return the list of DNS servers.""" - res = [] - cnt = self._configFile.get_count_with_prefix(Config.DNS) - for i in range(1, cnt + 1): - key = Config.DNS + str(i) - res.append(self._configFile[key]) - - return res - - @property - def dns_suffixes(self): - """Return the list of DNS Suffixes.""" - res = [] - cnt = self._configFile.get_count_with_prefix(Config.SUFFIX) - for i in range(1, cnt + 1): - key = Config.SUFFIX + str(i) - res.append(self._configFile[key]) - - return res - - @property - def nics(self): - """Return the list of associated NICs.""" - res = [] - nics = self._configFile['NIC-CONFIG|NICS'] - for nic in nics.split(','): - res.append(Nic(nic, self._configFile)) - - return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py deleted file mode 100644 index bb9fb7dc..00000000 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ /dev/null @@ -1,129 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging - -try: - import configparser -except ImportError: - import ConfigParser as configparser - -from .config_source import ConfigSource - -logger = logging.getLogger(__name__) - - -class ConfigFile(ConfigSource, dict): - """ConfigFile module to load the content from a specified source.""" - - def __init__(self, filename): - self._loadConfigFile(filename) - pass - - def _insertKey(self, key, val): - """ - Inserts a Key Value pair. - - Keyword arguments: - key -- The key to insert - val -- The value to insert for the key - - """ - key = key.strip() - val = val.strip() - - if key.startswith('-') or '|-' in key: - canLog = False - else: - canLog = True - - # "sensitive" settings shall not be logged - if canLog: - logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) - else: - logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) - - self[key] = val - - def _loadConfigFile(self, filename): - """ - Parses properties from the specified config file. - - Any previously available properties will be removed. - Sensitive data will not be logged in case the key starts - from '-'. - - Keyword arguments: - filename - The full path to the config file. - """ - logger.info('Parsing the config file %s.' % filename) - - config = configparser.ConfigParser() - config.optionxform = str - config.read(filename) - - self.clear() - - for category in config.sections(): - logger.debug("FOUND CATEGORY = '%s'" % category) - - for (key, value) in config.items(category): - self._insertKey(category + '|' + key, value) - - def should_keep_current_value(self, key): - """ - Determines whether a value for a property must be kept. - - If the propery is missing, it is treated as it should be not - changed by the engine. - - Keyword arguments: - key -- The key to search for. - """ - # helps to distinguish from "empty" value which is used to indicate - # "removal" - return key not in self - - def should_remove_current_value(self, key): - """ - Determines whether a value for the property must be removed. - - If the specified key is empty, it is treated as it should be - removed by the engine. - - Return true if the value can be removed, false otherwise. - - Keyword arguments: - key -- The key to search for. - """ - # helps to distinguish from "missing" value which is used to indicate - # "keeping unchanged" - if key in self: - return not bool(self[key]) - else: - return False - - def get_count_with_prefix(self, prefix): - """ - Return the total count of keys that start with the specified prefix. - - Keyword arguments: - prefix -- prefix of the key - """ - return len([key for key in self if key.startswith(prefix)]) diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py deleted file mode 100644 index b28830f5..00000000 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ /dev/null @@ -1,25 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from .config_source import ConfigSource - - -class ConfigNamespace(ConfigSource): - """Specifies the Config Namespace.""" - pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py deleted file mode 100644 index 511cc918..00000000 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ /dev/null @@ -1,247 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2016 VMware INC. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -import os -import re - -from cloudinit import util - -logger = logging.getLogger(__name__) - - -class NicConfigurator(object): - def __init__(self, nics): - """ - Initialize the Nic Configurator - @param nics (list) an array of nics to configure - """ - self.nics = nics - self.mac2Name = {} - self.ipv4PrimaryGateway = None - self.ipv6PrimaryGateway = None - self.find_devices() - self._primaryNic = self.get_primary_nic() - - def get_primary_nic(self): - """ - Retrieve the primary nic if it exists - @return (NicBase): the primary nic if exists, None otherwise - """ - primary_nics = [nic for nic in self.nics if nic.primary] - if not primary_nics: - return None - elif len(primary_nics) > 1: - raise Exception('There can only be one primary nic', - [nic.mac for nic in primary_nics]) - else: - return primary_nics[0] - - def find_devices(self): - """ - Create the mac2Name dictionary - The mac address(es) are in the lower case - """ - cmd = ['ip', 'addr', 'show'] - (output, err) = util.subp(cmd) - sections = re.split(r'\n\d+: ', '\n' + output)[1:] - - macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' - for section in sections: - match = re.search(macPat, section) - if not match: # Only keep info about nics - continue - mac = match.group(1).lower() - name = section.split(':', 1)[0] - self.mac2Name[mac] = name - - def gen_one_nic(self, nic): - """ - Return the lines needed to configure a nic - @return (str list): the string list to configure the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - name = self.mac2Name.get(nic.mac.lower()) - if not name: - raise ValueError('No known device has MACADDR: %s' % nic.mac) - - if nic.onboot: - lines.append('auto %s' % name) - - # Customize IPv4 - lines.extend(self.gen_ipv4(name, nic)) - - # Customize IPv6 - lines.extend(self.gen_ipv6(name, nic)) - - lines.append('') - - return lines - - def gen_ipv4(self, name, nic): - """ - Return the lines needed to configure the IPv4 setting of a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - - bootproto = nic.bootProto.lower() - if nic.ipv4_mode.lower() == 'disabled': - bootproto = 'manual' - lines.append('iface %s inet %s' % (name, bootproto)) - - if bootproto != 'static': - return lines - - # Static Ipv4 - v4 = nic.staticIpv4 - if v4.ip: - lines.append(' address %s' % v4.ip) - if v4.netmask: - lines.append(' netmask %s' % v4.netmask) - - # Add the primary gateway - if nic.primary and v4.gateways: - self.ipv4PrimaryGateway = v4.gateways[0] - lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) - return lines - - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self.gen_ipv4_route(nic, v4.gateways)) - - return lines - - def gen_ipv4_route(self, nic, gateways): - """ - Return the lines needed to configure additional Ipv4 route - @return (str list): the string list to configure the gateways - @param nic (NicBase): the nic to configure - @param gateways (str list): the list of gateways - """ - lines = [] - - for gateway in gateways: - lines.append(' up route add default gw %s metric 10000' % - gateway) - - return lines - - def gen_ipv6(self, name, nic): - """ - Return the lines needed to configure the gateways for a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - - if not nic.staticIpv6: - return lines - - # Static Ipv6 - addrs = nic.staticIpv6 - lines.append('iface %s inet6 static' % name) - lines.append(' address %s' % addrs[0].ip) - lines.append(' netmask %s' % addrs[0].netmask) - - for addr in addrs[1:]: - lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, - addr.netmask)) - # Add the primary gateway - if nic.primary: - for addr in addrs: - if addr.gateway: - self.ipv6PrimaryGateway = addr.gateway - lines.append(' gateway %s' % self.ipv6PrimaryGateway) - return lines - - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self._genIpv6Route(name, nic, addrs)) - - return lines - - def _genIpv6Route(self, name, nic, addrs): - lines = [] - - for addr in addrs: - lines.append(' up route -A inet6 add default gw ' - '%s metric 10000' % addr.gateway) - - return lines - - def generate(self): - """Return the lines that is needed to configure the nics""" - lines = [] - lines.append('iface lo inet loopback') - lines.append('auto lo') - lines.append('') - - for nic in self.nics: - lines.extend(self.gen_one_nic(nic)) - - return lines - - def clear_dhcp(self): - logger.info('Clearing DHCP leases') - - # Ignore the return code 1. - util.subp(["pkill", "dhclient"], rcs=[0, 1]) - util.subp(["rm", "-f", "/var/lib/dhcp/*"]) - - def if_down_up(self): - names = [] - for nic in self.nics: - name = self.mac2Name.get(nic.mac.lower()) - names.append(name) - - for name in names: - logger.info('Bring down interface %s' % name) - util.subp(["ifdown", "%s" % name]) - - self.clear_dhcp() - - for name in names: - logger.info('Bring up interface %s' % name) - util.subp(["ifup", "%s" % name]) - - def configure(self): - """ - Configure the /etc/network/intefaces - Make a back up of the original - """ - containingDir = '/etc/network' - - interfaceFile = os.path.join(containingDir, 'interfaces') - originalFile = os.path.join(containingDir, - 'interfaces.before_vmware_customization') - - if not os.path.exists(originalFile) and os.path.exists(interfaceFile): - os.rename(interfaceFile, originalFile) - - lines = self.generate() - with open(interfaceFile, 'w') as fp: - for line in lines: - fp.write('%s\n' % line) - - self.if_down_up() diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py deleted file mode 100644 index 28ef306a..00000000 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ /dev/null @@ -1,23 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class ConfigSource(object): - """Specifies a source for the Config Content.""" - pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py deleted file mode 100644 index d1546852..00000000 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ /dev/null @@ -1,24 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class GuestCustErrorEnum(object): - """Specifies different errors of Guest Customization engine""" - - GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py deleted file mode 100644 index ce90c898..00000000 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ /dev/null @@ -1,27 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class GuestCustEventEnum(object): - """Specifies different types of Guest Customization Events""" - - GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 - GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101 - GUESTCUST_EVENT_ENABLE_NICS = 103 - GUESTCUST_EVENT_QUERY_NICS = 104 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py deleted file mode 100644 index 422a096d..00000000 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ /dev/null @@ -1,25 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class GuestCustStateEnum(object): - """Specifies different states of Guest Customization engine""" - - GUESTCUST_STATE_RUNNING = 4 - GUESTCUST_STATE_DONE = 5 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py deleted file mode 100644 index c07c5949..00000000 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ /dev/null @@ -1,128 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -import os -import time - -from cloudinit import util - -from .guestcust_event import GuestCustEventEnum -from .guestcust_state import GuestCustStateEnum - -logger = logging.getLogger(__name__) - - -CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" -QUERY_NICS_SUPPORTED = "queryNicsSupported" -NICS_STATUS_CONNECTED = "connected" - - -# This will send a RPC command to the underlying -# VMware Virtualization Platform. -def send_rpc(rpc): - if not rpc: - return None - - out = "" - err = "Error sending the RPC command" - - try: - logger.debug("Sending RPC command: %s", rpc) - (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) - # Remove the trailing newline in the output. - if out: - out = out.rstrip() - except Exception as e: - logger.debug("Failed to send RPC command") - logger.exception(e) - - return (out, err) - - -# This will send the customization status to the -# underlying VMware Virtualization Platform. -def set_customization_status(custstate, custerror, errormessage=None): - message = "" - - if errormessage: - message = CLOUDINIT_LOG_FILE + "@" + errormessage - else: - message = CLOUDINIT_LOG_FILE - - rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) - (out, err) = send_rpc(rpc) - return (out, err) - - -# This will read the file nics.txt in the specified directory -# and return the content -def get_nics_to_enable(dirpath): - if not dirpath: - return None - - NICS_SIZE = 1024 - nicsfilepath = os.path.join(dirpath, "nics.txt") - if not os.path.exists(nicsfilepath): - return None - - with open(nicsfilepath, 'r') as fp: - nics = fp.read(NICS_SIZE) - - return nics - - -# This will send a RPC command to the underlying VMware Virtualization platform -# and enable nics. -def enable_nics(nics): - if not nics: - logger.warning("No Nics found") - return - - enableNicsWaitRetries = 5 - enableNicsWaitCount = 5 - enableNicsWaitSeconds = 1 - - for attempt in range(0, enableNicsWaitRetries): - logger.debug("Trying to connect interfaces, attempt %d", attempt) - (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, - nics) - if not out: - time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) - continue - - if out != QUERY_NICS_SUPPORTED: - logger.warning("NICS connection status query is not supported") - return - - for count in range(0, enableNicsWaitCount): - (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, - nics) - if out and out == NICS_STATUS_CONNECTED: - logger.info("NICS are connected on %d second", count) - return - - time.sleep(enableNicsWaitSeconds) - - logger.warning("Can't connect network interfaces after %d attempts", - enableNicsWaitRetries) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py deleted file mode 100644 index 873ddc3b..00000000 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ /dev/null @@ -1,45 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class Ipv4ModeEnum(object): - """ - The IPv4 configuration mode which directly represents the user's goal. - - This mode effectively acts as a contract of the in-guest customization - engine. It must be set based on what the user has requested and should - not be changed by those layers. It's up to the in-guest engine to - interpret and materialize the user's request. - """ - - # The legacy mode which only allows dhcp/static based on whether IPv4 - # addresses list is empty or not - IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' - - # IPv4 must use static address. Reserved for future use - IPV4_MODE_STATIC = 'STATIC' - - # IPv4 must use DHCPv4. Reserved for future use - IPV4_MODE_DHCP = 'DHCP' - - # IPv4 must be disabled - IPV4_MODE_DISABLED = 'DISABLED' - - # IPv4 settings should be left untouched. Reserved for future use - IPV4_MODE_AS_IS = 'AS_IS' diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py deleted file mode 100644 index b5d704ea..00000000 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ /dev/null @@ -1,147 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from .boot_proto import BootProtoEnum -from .nic_base import NicBase, StaticIpv4Base, StaticIpv6Base - - -class Nic(NicBase): - """ - Holds the information about each NIC specified - in the customization specification file - """ - - def __init__(self, name, configFile): - self._name = name - self._configFile = configFile - - def _get(self, what): - return self._configFile.get(self.name + '|' + what, None) - - def _get_count_with_prefix(self, prefix): - return self._configFile.get_count_with_prefix(self.name + prefix) - - @property - def name(self): - return self._name - - @property - def mac(self): - return self._get('MACADDR').lower() - - @property - def primary(self): - value = self._get('PRIMARY') - if value: - value = value.lower() - return value == 'yes' or value == 'true' - else: - return False - - @property - def onboot(self): - value = self._get('ONBOOT') - if value: - value = value.lower() - return value == 'yes' or value == 'true' - else: - return False - - @property - def bootProto(self): - value = self._get('BOOTPROTO') - if value: - return value.lower() - else: - return "" - - @property - def ipv4_mode(self): - value = self._get('IPv4_MODE') - if value: - return value.lower() - else: - return "" - - @property - def staticIpv4(self): - """ - Checks the BOOTPROTO property and returns StaticIPv4Addr - configuration object if STATIC configuration is set. - """ - if self.bootProto == BootProtoEnum.STATIC: - return [StaticIpv4Addr(self)] - else: - return None - - @property - def staticIpv6(self): - cnt = self._get_count_with_prefix('|IPv6ADDR|') - - if not cnt: - return None - - result = [] - for index in range(1, cnt + 1): - result.append(StaticIpv6Addr(self, index)) - - return result - - -class StaticIpv4Addr(StaticIpv4Base): - """Static IPV4 Setting.""" - - def __init__(self, nic): - self._nic = nic - - @property - def ip(self): - return self._nic._get('IPADDR') - - @property - def netmask(self): - return self._nic._get('NETMASK') - - @property - def gateways(self): - value = self._nic._get('GATEWAY') - if value: - return [x.strip() for x in value.split(',')] - else: - return None - - -class StaticIpv6Addr(StaticIpv6Base): - """Static IPV6 Address.""" - - def __init__(self, nic, index): - self._nic = nic - self._index = index - - @property - def ip(self): - return self._nic._get('IPv6ADDR|' + str(self._index)) - - @property - def netmask(self): - return self._nic._get('IPv6NETMASK|' + str(self._index)) - - @property - def gateway(self): - return self._nic._get('IPv6GATEWAY|' + str(self._index)) diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py deleted file mode 100644 index 3c892db0..00000000 --- a/cloudinit/sources/helpers/vmware/imc/nic_base.py +++ /dev/null @@ -1,154 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class NicBase(object): - """ - Define what are expected of each nic. - The following properties should be provided in an implementation class. - """ - - @property - def mac(self): - """ - Retrieves the mac address of the nic - @return (str) : the MACADDR setting - """ - raise NotImplementedError('MACADDR') - - @property - def primary(self): - """ - Retrieves whether the nic is the primary nic - Indicates whether NIC will be used to define the default gateway. - If none of the NICs is configured to be primary, default gateway won't - be set. - @return (bool): the PRIMARY setting - """ - raise NotImplementedError('PRIMARY') - - @property - def onboot(self): - """ - Retrieves whether the nic should be up at the boot time - @return (bool) : the ONBOOT setting - """ - raise NotImplementedError('ONBOOT') - - @property - def bootProto(self): - """ - Retrieves the boot protocol of the nic - @return (str): the BOOTPROTO setting, valid values: dhcp and static. - """ - raise NotImplementedError('BOOTPROTO') - - @property - def ipv4_mode(self): - """ - Retrieves the IPv4_MODE - @return (str): the IPv4_MODE setting, valid values: - backwards_compatible, static, dhcp, disabled, as_is - """ - raise NotImplementedError('IPv4_MODE') - - @property - def staticIpv4(self): - """ - Retrieves the static IPv4 configuration of the nic - @return (StaticIpv4Base list): the static ipv4 setting - """ - raise NotImplementedError('Static IPv4') - - @property - def staticIpv6(self): - """ - Retrieves the IPv6 configuration of the nic - @return (StaticIpv6Base list): the static ipv6 setting - """ - raise NotImplementedError('Static Ipv6') - - def validate(self): - """ - Validate the object - For example, the staticIpv4 property is required and should not be - empty when ipv4Mode is STATIC - """ - raise NotImplementedError('Check constraints on properties') - - -class StaticIpv4Base(object): - """ - Define what are expected of a static IPv4 setting - The following properties should be provided in an implementation class. - """ - - @property - def ip(self): - """ - Retrieves the Ipv4 address - @return (str): the IPADDR setting - """ - raise NotImplementedError('Ipv4 Address') - - @property - def netmask(self): - """ - Retrieves the Ipv4 NETMASK setting - @return (str): the NETMASK setting - """ - raise NotImplementedError('Ipv4 NETMASK') - - @property - def gateways(self): - """ - Retrieves the gateways on this Ipv4 subnet - @return (str list): the GATEWAY setting - """ - raise NotImplementedError('Ipv4 GATEWAY') - - -class StaticIpv6Base(object): - """Define what are expected of a static IPv6 setting - The following properties should be provided in an implementation class. - """ - - @property - def ip(self): - """ - Retrieves the Ipv6 address - @return (str): the IPv6ADDR setting - """ - raise NotImplementedError('Ipv6 Address') - - @property - def netmask(self): - """ - Retrieves the Ipv6 NETMASK setting - @return (str): the IPv6NETMASK setting - """ - raise NotImplementedError('Ipv6 NETMASK') - - @property - def gateway(self): - """ - Retrieves the Ipv6 GATEWAY setting - @return (str): the IPv6GATEWAY setting - """ - raise NotImplementedError('Ipv6 GATEWAY') diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py deleted file mode 100644 index c74a7ae2..00000000 --- a/cloudinit/ssh_util.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import pwd - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - -# See: man sshd_config -DEF_SSHD_CFG = "/etc/ssh/sshd_config" - -# taken from openssh source key.c/key_type_from_name -VALID_KEY_TYPES = ( - "rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", - "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", - "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", - "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com", - "ecdsa-sha2-nistp256-cert-v01@openssh.com", - "ecdsa-sha2-nistp384-cert-v01@openssh.com", - "ecdsa-sha2-nistp521-cert-v01@openssh.com") - - -class AuthKeyLine(object): - def __init__(self, source, keytype=None, base64=None, - comment=None, options=None): - self.base64 = base64 - self.comment = comment - self.options = options - self.keytype = keytype - self.source = source - - def valid(self): - return (self.base64 and self.keytype) - - def __str__(self): - toks = [] - if self.options: - toks.append(self.options) - if self.keytype: - toks.append(self.keytype) - if self.base64: - toks.append(self.base64) - if self.comment: - toks.append(self.comment) - if not toks: - return self.source - else: - return ' '.join(toks) - - -class AuthKeyLineParser(object): - """ - AUTHORIZED_KEYS FILE FORMAT - AuthorizedKeysFile specifies the file containing public keys for public - key authentication; if none is specified, the default is - ~/.ssh/authorized_keys. Each line of the file contains one key (empty - (because of the size of the public key encoding) up to a limit of 8 kilo- - bytes, which permits DSA keys up to 8 kilobits and RSA keys up to 16 - kilobits. You don't want to type them in; instead, copy the - identity.pub, id_dsa.pub, or the id_rsa.pub file and edit it. - - sshd enforces a minimum RSA key modulus size for protocol 1 and protocol - 2 keys of 768 bits. - - The options (if present) consist of comma-separated option specifica- - tions. No spaces are permitted, except within double quotes. The fol- - lowing option specifications are supported (note that option keywords are - case-insensitive): - """ - - def _extract_options(self, ent): - """ - The options (if present) consist of comma-separated option specifica- - tions. No spaces are permitted, except within double quotes. - Note that option keywords are case-insensitive. - """ - quoted = False - i = 0 - while (i < len(ent) and - ((quoted) or (ent[i] not in (" ", "\t")))): - curc = ent[i] - if i + 1 >= len(ent): - i = i + 1 - break - nextc = ent[i + 1] - if curc == "\\" and nextc == '"': - i = i + 1 - elif curc == '"': - quoted = not quoted - i = i + 1 - - options = ent[0:i] - - # Return the rest of the string in 'remain' - remain = ent[i:].lstrip() - return (options, remain) - - def parse(self, src_line, options=None): - # modeled after opensshes auth2-pubkey.c:user_key_allowed2 - line = src_line.rstrip("\r\n") - if line.startswith("#") or line.strip() == '': - return AuthKeyLine(src_line) - - def parse_ssh_key(ent): - # return ketype, key, [comment] - toks = ent.split(None, 2) - if len(toks) < 2: - raise TypeError("To few fields: %s" % len(toks)) - if toks[0] not in VALID_KEY_TYPES: - raise TypeError("Invalid keytype %s" % toks[0]) - - # valid key type and 2 or 3 fields: - if len(toks) == 2: - # no comment in line - toks.append("") - - return toks - - ent = line.strip() - try: - (keytype, base64, comment) = parse_ssh_key(ent) - except TypeError: - (keyopts, remain) = self._extract_options(ent) - if options is None: - options = keyopts - - try: - (keytype, base64, comment) = parse_ssh_key(remain) - except TypeError: - return AuthKeyLine(src_line) - - return AuthKeyLine(src_line, keytype=keytype, base64=base64, - comment=comment, options=options) - - -def parse_authorized_keys(fname): - lines = [] - try: - if os.path.isfile(fname): - lines = util.load_file(fname).splitlines() - except (IOError, OSError): - util.logexc(LOG, "Error reading lines from %s", fname) - lines = [] - - parser = AuthKeyLineParser() - contents = [] - for line in lines: - contents.append(parser.parse(line)) - return contents - - -def update_authorized_keys(old_entries, keys): - to_add = list(keys) - - for i in range(0, len(old_entries)): - ent = old_entries[i] - if not ent.valid(): - continue - # Replace those with the same base64 - for k in keys: - if not ent.valid(): - continue - if k.base64 == ent.base64: - # Replace it with our better one - ent = k - # Don't add it later - if k in to_add: - to_add.remove(k) - old_entries[i] = ent - - # Now append any entries we did not match above - for key in to_add: - old_entries.append(key) - - # Now format them back to strings... - lines = [str(b) for b in old_entries] - - # Ensure it ends with a newline - lines.append('') - return '\n'.join(lines) - - -def users_ssh_info(username): - pw_ent = pwd.getpwnam(username) - if not pw_ent or not pw_ent.pw_dir: - raise RuntimeError("Unable to get ssh info for user %r" % (username)) - return (os.path.join(pw_ent.pw_dir, '.ssh'), pw_ent) - - -def extract_authorized_keys(username): - (ssh_dir, pw_ent) = users_ssh_info(username) - auth_key_fn = None - with util.SeLinuxGuard(ssh_dir, recursive=True): - try: - # The 'AuthorizedKeysFile' may contain tokens - # of the form %T which are substituted during connection set-up. - # The following tokens are defined: %% is replaced by a literal - # '%', %h is replaced by the home directory of the user being - # authenticated and %u is replaced by the username of that user. - ssh_cfg = parse_ssh_config_map(DEF_SSHD_CFG) - auth_key_fn = ssh_cfg.get("authorizedkeysfile", '').strip() - if not auth_key_fn: - auth_key_fn = "%h/.ssh/authorized_keys" - auth_key_fn = auth_key_fn.replace("%h", pw_ent.pw_dir) - auth_key_fn = auth_key_fn.replace("%u", username) - auth_key_fn = auth_key_fn.replace("%%", '%') - if not auth_key_fn.startswith('/'): - auth_key_fn = os.path.join(pw_ent.pw_dir, auth_key_fn) - except (IOError, OSError): - # Give up and use a default key filename - auth_key_fn = os.path.join(ssh_dir, 'authorized_keys') - util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in ssh " - "config from %r, using 'AuthorizedKeysFile' file " - "%r instead", DEF_SSHD_CFG, auth_key_fn) - return (auth_key_fn, parse_authorized_keys(auth_key_fn)) - - -def setup_user_keys(keys, username, options=None): - # Make sure the users .ssh dir is setup accordingly - (ssh_dir, pwent) = users_ssh_info(username) - if not os.path.isdir(ssh_dir): - util.ensure_dir(ssh_dir, mode=0o700) - util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid) - - # Turn the 'update' keys given into actual entries - parser = AuthKeyLineParser() - key_entries = [] - for k in keys: - key_entries.append(parser.parse(str(k), options=options)) - - # Extract the old and make the new - (auth_key_fn, auth_key_entries) = extract_authorized_keys(username) - with util.SeLinuxGuard(ssh_dir, recursive=True): - content = update_authorized_keys(auth_key_entries, key_entries) - util.ensure_dir(os.path.dirname(auth_key_fn), mode=0o700) - util.write_file(auth_key_fn, content, mode=0o600) - util.chownbyid(auth_key_fn, pwent.pw_uid, pwent.pw_gid) - - -class SshdConfigLine(object): - def __init__(self, line, k=None, v=None): - self.line = line - self._key = k - self.value = v - - @property - def key(self): - if self._key is None: - return None - # Keywords are case-insensitive - return self._key.lower() - - def __str__(self): - if self._key is None: - return str(self.line) - else: - v = str(self._key) - if self.value: - v += " " + str(self.value) - return v - - -def parse_ssh_config(fname): - # See: man sshd_config - # The file contains keyword-argument pairs, one per line. - # Lines starting with '#' and empty lines are interpreted as comments. - # Note: key-words are case-insensitive and arguments are case-sensitive - lines = [] - if not os.path.isfile(fname): - return lines - for line in util.load_file(fname).splitlines(): - line = line.strip() - if not line or line.startswith("#"): - lines.append(SshdConfigLine(line)) - continue - try: - key, val = line.split(None, 1) - except ValueError: - key, val = line.split('=', 1) - lines.append(SshdConfigLine(line, key, val)) - return lines - - -def parse_ssh_config_map(fname): - lines = parse_ssh_config(fname) - if not lines: - return {} - ret = {} - for line in lines: - if not line.key: - continue - ret[line.key] = line.value - return ret diff --git a/cloudinit/stages.py b/cloudinit/stages.py deleted file mode 100644 index 47deac6e..00000000 --- a/cloudinit/stages.py +++ /dev/null @@ -1,890 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import copy -import os -import sys - -import six -from six.moves import cPickle as pickle - -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES, CLOUD_CONFIG) - -from cloudinit import handlers - -# Default handlers (used if not overridden) -from cloudinit.handlers import boot_hook as bh_part -from cloudinit.handlers import cloud_config as cc_part -from cloudinit.handlers import shell_script as ss_part -from cloudinit.handlers import upstart_job as up_part - -from cloudinit import cloud -from cloudinit import config -from cloudinit import distros -from cloudinit import helpers -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import net -from cloudinit.net import cmdline -from cloudinit.reporting import events -from cloudinit import sources -from cloudinit import type_utils -from cloudinit import util - -LOG = logging.getLogger(__name__) - -NULL_DATA_SOURCE = None -NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID" - - -class Init(object): - def __init__(self, ds_deps=None, reporter=None): - if ds_deps is not None: - self.ds_deps = ds_deps - else: - self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - # Created on first use - self._cfg = None - self._paths = None - self._distro = None - # Changed only when a fetch occurs - self.datasource = NULL_DATA_SOURCE - self.ds_restored = False - self._previous_iid = None - - if reporter is None: - reporter = events.ReportEventStack( - name="init-reporter", description="init-desc", - reporting_enabled=False) - self.reporter = reporter - - def _reset(self, reset_ds=False): - # Recreated on access - self._cfg = None - self._paths = None - self._distro = None - if reset_ds: - self.datasource = NULL_DATA_SOURCE - self.ds_restored = False - - @property - def distro(self): - if not self._distro: - # Try to find the right class to use - system_config = self._extract_cfg('system') - distro_name = system_config.pop('distro', 'ubuntu') - distro_cls = distros.fetch(distro_name) - LOG.debug("Using distro class %s", distro_cls) - self._distro = distro_cls(distro_name, system_config, self.paths) - # If we have an active datasource we need to adjust - # said datasource and move its distro/system config - # from whatever it was to a new set... - if self.datasource is not NULL_DATA_SOURCE: - self.datasource.distro = self._distro - self.datasource.sys_cfg = system_config - return self._distro - - @property - def cfg(self): - return self._extract_cfg('restricted') - - def _extract_cfg(self, restriction): - # Ensure actually read - self.read_cfg() - # Nobody gets the real config - ocfg = copy.deepcopy(self._cfg) - if restriction == 'restricted': - ocfg.pop('system_info', None) - elif restriction == 'system': - ocfg = util.get_cfg_by_path(ocfg, ('system_info',), {}) - elif restriction == 'paths': - ocfg = util.get_cfg_by_path(ocfg, ('system_info', 'paths'), {}) - if not isinstance(ocfg, (dict)): - ocfg = {} - return ocfg - - @property - def paths(self): - if not self._paths: - path_info = self._extract_cfg('paths') - self._paths = helpers.Paths(path_info, self.datasource) - return self._paths - - def _initial_subdirs(self): - c_dir = self.paths.cloud_dir - initial_dirs = [ - c_dir, - os.path.join(c_dir, 'scripts'), - os.path.join(c_dir, 'scripts', 'per-instance'), - os.path.join(c_dir, 'scripts', 'per-once'), - os.path.join(c_dir, 'scripts', 'per-boot'), - os.path.join(c_dir, 'scripts', 'vendor'), - os.path.join(c_dir, 'seed'), - os.path.join(c_dir, 'instances'), - os.path.join(c_dir, 'handlers'), - os.path.join(c_dir, 'sem'), - os.path.join(c_dir, 'data'), - ] - return initial_dirs - - def purge_cache(self, rm_instance_lnk=False): - rm_list = [] - rm_list.append(self.paths.boot_finished) - if rm_instance_lnk: - rm_list.append(self.paths.instance_link) - for f in rm_list: - util.del_file(f) - return len(rm_list) - - def initialize(self): - self._initialize_filesystem() - - def _initialize_filesystem(self): - util.ensure_dirs(self._initial_subdirs()) - log_file = util.get_cfg_option_str(self.cfg, 'def_log_file') - if log_file: - util.ensure_file(log_file) - perms = self.cfg.get('syslog_fix_perms') - if not perms: - perms = {} - if not isinstance(perms, list): - perms = [perms] - - error = None - for perm in perms: - u, g = util.extract_usergroup(perm) - try: - util.chownbyname(log_file, u, g) - return - except OSError as e: - error = e - - LOG.warn("Failed changing perms on '%s'. tried: %s. %s", - log_file, ','.join(perms), error) - - def read_cfg(self, extra_fns=None): - # None check so that we don't keep on re-loading if empty - if self._cfg is None: - self._cfg = self._read_cfg(extra_fns) - # LOG.debug("Loaded 'init' config %s", self._cfg) - - def _read_cfg(self, extra_fns): - no_cfg_paths = helpers.Paths({}, self.datasource) - merger = helpers.ConfigMerger(paths=no_cfg_paths, - datasource=self.datasource, - additional_fns=extra_fns, - base_cfg=fetch_base_config()) - return merger.cfg - - def _restore_from_cache(self): - # We try to restore from a current link and static path - # by using the instance link, if purge_cache was called - # the file wont exist. - return _pkl_load(self.paths.get_ipath_cur('obj_pkl')) - - def _write_to_cache(self): - if self.datasource is NULL_DATA_SOURCE: - return False - return _pkl_store(self.datasource, self.paths.get_ipath_cur("obj_pkl")) - - def _get_datasources(self): - # Any config provided??? - pkg_list = self.cfg.get('datasource_pkg_list') or [] - # Add the defaults at the end - for n in ['', type_utils.obj_name(sources)]: - if n not in pkg_list: - pkg_list.append(n) - cfg_list = self.cfg.get('datasource_list') or [] - return (cfg_list, pkg_list) - - def _restore_from_checked_cache(self, existing): - if existing not in ("check", "trust"): - raise ValueError("Unexpected value for existing: %s" % existing) - - ds = self._restore_from_cache() - if not ds: - return (None, "no cache found") - - run_iid_fn = self.paths.get_runpath('instance_id') - if os.path.exists(run_iid_fn): - run_iid = util.load_file(run_iid_fn).strip() - else: - run_iid = None - - if run_iid == ds.get_instance_id(): - return (ds, "restored from cache with run check: %s" % ds) - elif existing == "trust": - return (ds, "restored from cache: %s" % ds) - else: - if (hasattr(ds, 'check_instance_id') and - ds.check_instance_id(self.cfg)): - return (ds, "restored from checked cache: %s" % ds) - else: - return (None, "cache invalid in datasource: %s" % ds) - - def _get_data_source(self, existing): - if self.datasource is not NULL_DATA_SOURCE: - return self.datasource - - with events.ReportEventStack( - name="check-cache", - description="attempting to read from cache [%s]" % existing, - parent=self.reporter) as myrep: - - ds, desc = self._restore_from_checked_cache(existing) - myrep.description = desc - self.ds_restored = bool(ds) - LOG.debug(myrep.description) - - if not ds: - util.del_file(self.paths.instance_link) - (cfg_list, pkg_list) = self._get_datasources() - # Deep copy so that user-data handlers can not modify - # (which will affect user-data handlers down the line...) - (ds, dsname) = sources.find_source(self.cfg, - self.distro, - self.paths, - copy.deepcopy(self.ds_deps), - cfg_list, - pkg_list, self.reporter) - LOG.info("Loaded datasource %s - %s", dsname, ds) - self.datasource = ds - # Ensure we adjust our path members datasource - # now that we have one (thus allowing ipath to be used) - self._reset() - return ds - - def _get_instance_subdirs(self): - return ['handlers', 'scripts', 'sem'] - - def _get_ipath(self, subname=None): - # Force a check to see if anything - # actually comes back, if not - # then a datasource has not been assigned... - instance_dir = self.paths.get_ipath(subname) - if not instance_dir: - raise RuntimeError(("No instance directory is available." - " Has a datasource been fetched??")) - return instance_dir - - def _reflect_cur_instance(self): - # Remove the old symlink and attach a new one so - # that further reads/writes connect into the right location - idir = self._get_ipath() - util.del_file(self.paths.instance_link) - util.sym_link(idir, self.paths.instance_link) - - # Ensures these dirs exist - dir_list = [] - for d in self._get_instance_subdirs(): - dir_list.append(os.path.join(idir, d)) - util.ensure_dirs(dir_list) - - # Write out information on what is being used for the current instance - # and what may have been used for a previous instance... - dp = self.paths.get_cpath('data') - - # Write what the datasource was and is.. - ds = "%s: %s" % (type_utils.obj_name(self.datasource), self.datasource) - previous_ds = None - ds_fn = os.path.join(idir, 'datasource') - try: - previous_ds = util.load_file(ds_fn).strip() - except Exception: - pass - if not previous_ds: - previous_ds = ds - util.write_file(ds_fn, "%s\n" % ds) - util.write_file(os.path.join(dp, 'previous-datasource'), - "%s\n" % (previous_ds)) - - # What the instance id was and is... - iid = self.datasource.get_instance_id() - iid_fn = os.path.join(dp, 'instance-id') - - previous_iid = self.previous_iid() - util.write_file(iid_fn, "%s\n" % iid) - util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) - util.write_file(os.path.join(dp, 'previous-instance-id'), - "%s\n" % (previous_iid)) - - self._write_to_cache() - # Ensure needed components are regenerated - # after change of instance which may cause - # change of configuration - self._reset() - return iid - - def previous_iid(self): - if self._previous_iid is not None: - return self._previous_iid - - dp = self.paths.get_cpath('data') - iid_fn = os.path.join(dp, 'instance-id') - try: - self._previous_iid = util.load_file(iid_fn).strip() - except Exception: - self._previous_iid = NO_PREVIOUS_INSTANCE_ID - - LOG.debug("previous iid found to be %s", self._previous_iid) - return self._previous_iid - - def is_new_instance(self): - previous = self.previous_iid() - ret = (previous == NO_PREVIOUS_INSTANCE_ID or - previous != self.datasource.get_instance_id()) - return ret - - def fetch(self, existing="check"): - return self._get_data_source(existing=existing) - - def instancify(self): - return self._reflect_cur_instance() - - def cloudify(self): - # Form the needed options to cloudify our members - return cloud.Cloud(self.datasource, - self.paths, self.cfg, - self.distro, helpers.Runners(self.paths), - reporter=self.reporter) - - def update(self): - self._store_userdata() - self._store_vendordata() - - def _store_userdata(self): - raw_ud = self.datasource.get_userdata_raw() - if raw_ud is None: - raw_ud = b'' - util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600) - # processed userdata is a Mime message, so write it as string. - processed_ud = self.datasource.get_userdata() - if processed_ud is None: - raw_ud = '' - util.write_file(self._get_ipath('userdata'), str(processed_ud), 0o600) - - def _store_vendordata(self): - raw_vd = self.datasource.get_vendordata_raw() - if raw_vd is None: - raw_vd = b'' - util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600) - # processed vendor data is a Mime message, so write it as string. - processed_vd = str(self.datasource.get_vendordata()) - if processed_vd is None: - processed_vd = '' - util.write_file(self._get_ipath('vendordata'), str(processed_vd), - 0o600) - - def _default_handlers(self, opts=None): - if opts is None: - opts = {} - - opts.update({ - 'paths': self.paths, - 'datasource': self.datasource, - }) - # TODO(harlowja) Hmmm, should we dynamically import these?? - def_handlers = [ - cc_part.CloudConfigPartHandler(**opts), - ss_part.ShellScriptPartHandler(**opts), - bh_part.BootHookPartHandler(**opts), - up_part.UpstartJobPartHandler(**opts), - ] - return def_handlers - - def _default_userdata_handlers(self): - return self._default_handlers() - - def _default_vendordata_handlers(self): - return self._default_handlers( - opts={'script_path': 'vendor_scripts', - 'cloud_config_path': 'vendor_cloud_config'}) - - def _do_handlers(self, data_msg, c_handlers_list, frequency, - excluded=None): - """ - Generalized handlers suitable for use with either vendordata - or userdata - """ - if excluded is None: - excluded = [] - - cdir = self.paths.get_cpath("handlers") - idir = self._get_ipath("handlers") - - # Add the path to the plugins dir to the top of our list for importing - # new handlers. - # - # Note(harlowja): instance dir should be read before cloud-dir - for d in [cdir, idir]: - if d and d not in sys.path: - sys.path.insert(0, d) - - def register_handlers_in_dir(path): - # Attempts to register any handler modules under the given path. - if not path or not os.path.isdir(path): - return - potential_handlers = util.find_modules(path) - for (fname, mod_name) in potential_handlers.items(): - try: - mod_locs, looked_locs = importer.find_module( - mod_name, [''], ['list_types', 'handle_part']) - if not mod_locs: - LOG.warn("Could not find a valid user-data handler" - " named %s in file %s (searched %s)", - mod_name, fname, looked_locs) - continue - mod = importer.import_module(mod_locs[0]) - mod = handlers.fixup_handler(mod) - types = c_handlers.register(mod) - if types: - LOG.debug("Added custom handler for %s [%s] from %s", - types, mod, fname) - except Exception: - util.logexc(LOG, "Failed to register handler from %s", - fname) - - # This keeps track of all the active handlers - c_handlers = helpers.ContentHandlers() - - # Add any handlers in the cloud-dir - register_handlers_in_dir(cdir) - - # Register any other handlers that come from the default set. This - # is done after the cloud-dir handlers so that the cdir modules can - # take over the default user-data handler content-types. - for mod in c_handlers_list: - types = c_handlers.register(mod, overwrite=False) - if types: - LOG.debug("Added default handler for %s from %s", types, mod) - - # Form our cloud interface - data = self.cloudify() - - def init_handlers(): - # Init the handlers first - for (_ctype, mod) in c_handlers.items(): - if mod in c_handlers.initialized: - # Avoid initing the same module twice (if said module - # is registered to more than one content-type). - continue - handlers.call_begin(mod, data, frequency) - c_handlers.initialized.append(mod) - - def walk_handlers(excluded): - # Walk the user data - part_data = { - 'handlers': c_handlers, - # Any new handlers that are encountered get writen here - 'handlerdir': idir, - 'data': data, - # The default frequency if handlers don't have one - 'frequency': frequency, - # This will be used when new handlers are found - # to help write there contents to files with numbered - # names... - 'handlercount': 0, - 'excluded': excluded, - } - handlers.walk(data_msg, handlers.walker_callback, data=part_data) - - def finalize_handlers(): - # Give callbacks opportunity to finalize - for (_ctype, mod) in c_handlers.items(): - if mod not in c_handlers.initialized: - # Said module was never inited in the first place, so lets - # not attempt to finalize those that never got called. - continue - c_handlers.initialized.remove(mod) - try: - handlers.call_end(mod, data, frequency) - except Exception: - util.logexc(LOG, "Failed to finalize handler: %s", mod) - - try: - init_handlers() - walk_handlers(excluded) - finally: - finalize_handlers() - - def consume_data(self, frequency=PER_INSTANCE): - # Consume the userdata first, because we need want to let the part - # handlers run first (for merging stuff) - with events.ReportEventStack("consume-user-data", - "reading and applying user-data", - parent=self.reporter): - self._consume_userdata(frequency) - with events.ReportEventStack("consume-vendor-data", - "reading and applying vendor-data", - parent=self.reporter): - self._consume_vendordata(frequency) - - # Perform post-consumption adjustments so that - # modules that run during the init stage reflect - # this consumed set. - # - # They will be recreated on future access... - self._reset() - # Note(harlowja): the 'active' datasource will have - # references to the previous config, distro, paths - # objects before the load of the userdata happened, - # this is expected. - - def _consume_vendordata(self, frequency=PER_INSTANCE): - """ - Consume the vendordata and run the part handlers on it - """ - # User-data should have been consumed first. - # So we merge the other available cloud-configs (everything except - # vendor provided), and check whether or not we should consume - # vendor data at all. That gives user or system a chance to override. - if not self.datasource.get_vendordata_raw(): - LOG.debug("no vendordata from datasource") - return - - _cc_merger = helpers.ConfigMerger(paths=self._paths, - datasource=self.datasource, - additional_fns=[], - base_cfg=self.cfg, - include_vendor=False) - vdcfg = _cc_merger.cfg.get('vendor_data', {}) - - if not isinstance(vdcfg, dict): - vdcfg = {'enabled': False} - LOG.warn("invalid 'vendor_data' setting. resetting to: %s", vdcfg) - - enabled = vdcfg.get('enabled') - no_handlers = vdcfg.get('disabled_handlers', None) - - if not util.is_true(enabled): - LOG.debug("vendordata consumption is disabled.") - return - - LOG.debug("vendor data will be consumed. disabled_handlers=%s", - no_handlers) - - # Ensure vendordata source fetched before activation (just incase) - vendor_data_msg = self.datasource.get_vendordata() - - # This keeps track of all the active handlers, while excluding what the - # users doesn't want run, i.e. boot_hook, cloud_config, shell_script - c_handlers_list = self._default_vendordata_handlers() - - # Run the handlers - self._do_handlers(vendor_data_msg, c_handlers_list, frequency, - excluded=no_handlers) - - def _consume_userdata(self, frequency=PER_INSTANCE): - """ - Consume the userdata and run the part handlers - """ - - # Ensure datasource fetched before activation (just incase) - user_data_msg = self.datasource.get_userdata(True) - - # This keeps track of all the active handlers - c_handlers_list = self._default_handlers() - - # Run the handlers - self._do_handlers(user_data_msg, c_handlers_list, frequency) - - def _find_networking_config(self): - disable_file = os.path.join( - self.paths.get_cpath('data'), 'upgraded-network') - if os.path.exists(disable_file): - return (None, disable_file) - - cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config()) - dscfg = ('ds', None) - if self.datasource and hasattr(self.datasource, 'network_config'): - dscfg = ('ds', self.datasource.network_config) - sys_cfg = ('system_cfg', self.cfg.get('network')) - - for loc, ncfg in (cmdline_cfg, sys_cfg, dscfg): - if net.is_disabled_cfg(ncfg): - LOG.debug("network config disabled by %s", loc) - return (None, loc) - if ncfg: - return (ncfg, loc) - return (net.generate_fallback_config(), "fallback") - - def apply_network_config(self, bring_up): - netcfg, src = self._find_networking_config() - if netcfg is None: - LOG.info("network config is disabled by %s", src) - return - - try: - LOG.debug("applying net config names for %s" % netcfg) - self.distro.apply_network_config_names(netcfg) - except Exception as e: - LOG.warn("Failed to rename devices: %s", e) - - if (self.datasource is not NULL_DATA_SOURCE and - not self.is_new_instance()): - LOG.debug("not a new instance. network config is not applied.") - return - - LOG.info("Applying network configuration from %s bringup=%s: %s", - src, bring_up, netcfg) - try: - return self.distro.apply_network_config(netcfg, bring_up=bring_up) - except NotImplementedError: - LOG.warn("distro '%s' does not implement apply_network_config. " - "networking may not be configured properly." % - self.distro) - return - - -class Modules(object): - def __init__(self, init, cfg_files=None, reporter=None): - self.init = init - self.cfg_files = cfg_files - # Created on first use - self._cached_cfg = None - if reporter is None: - reporter = events.ReportEventStack( - name="module-reporter", description="module-desc", - reporting_enabled=False) - self.reporter = reporter - - @property - def cfg(self): - # None check to avoid empty case causing re-reading - if self._cached_cfg is None: - merger = helpers.ConfigMerger(paths=self.init.paths, - datasource=self.init.datasource, - additional_fns=self.cfg_files, - base_cfg=self.init.cfg) - self._cached_cfg = merger.cfg - # LOG.debug("Loading 'module' config %s", self._cached_cfg) - # Only give out a copy so that others can't modify this... - return copy.deepcopy(self._cached_cfg) - - def _read_modules(self, name): - module_list = [] - if name not in self.cfg: - return module_list - cfg_mods = self.cfg[name] - # Create 'module_list', an array of hashes - # Where hash['mod'] = module name - # hash['freq'] = frequency - # hash['args'] = arguments - for item in cfg_mods: - if not item: - continue - if isinstance(item, six.string_types): - module_list.append({ - 'mod': item.strip(), - }) - elif isinstance(item, (list)): - contents = {} - # Meant to fall through... - if len(item) >= 1: - contents['mod'] = item[0].strip() - if len(item) >= 2: - contents['freq'] = item[1].strip() - if len(item) >= 3: - contents['args'] = item[2:] - if contents: - module_list.append(contents) - elif isinstance(item, (dict)): - contents = {} - valid = False - if 'name' in item: - contents['mod'] = item['name'].strip() - valid = True - if 'frequency' in item: - contents['freq'] = item['frequency'].strip() - if 'args' in item: - contents['args'] = item['args'] or [] - if contents and valid: - module_list.append(contents) - else: - raise TypeError(("Failed to read '%s' item in config," - " unknown type %s") % - (item, type_utils.obj_name(item))) - return module_list - - def _fixup_modules(self, raw_mods): - mostly_mods = [] - for raw_mod in raw_mods: - raw_name = raw_mod['mod'] - freq = raw_mod.get('freq') - run_args = raw_mod.get('args') or [] - mod_name = config.form_module_name(raw_name) - if not mod_name: - continue - if freq and freq not in FREQUENCIES: - LOG.warn(("Config specified module %s" - " has an unknown frequency %s"), raw_name, freq) - # Reset it so when ran it will get set to a known value - freq = None - mod_locs, looked_locs = importer.find_module( - mod_name, ['', type_utils.obj_name(config)], ['handle']) - if not mod_locs: - LOG.warn("Could not find module named %s (searched %s)", - mod_name, looked_locs) - continue - mod = config.fixup_module(importer.import_module(mod_locs[0])) - mostly_mods.append([mod, raw_name, freq, run_args]) - return mostly_mods - - def _run_modules(self, mostly_mods): - cc = self.init.cloudify() - # Return which ones ran - # and which ones failed + the exception of why it failed - failures = [] - which_ran = [] - for (mod, name, freq, args) in mostly_mods: - try: - # Try the modules frequency, otherwise fallback to a known one - if not freq: - freq = mod.frequency - if freq not in FREQUENCIES: - freq = PER_INSTANCE - LOG.debug("Running module %s (%s) with frequency %s", - name, mod, freq) - - # Use the configs logger and not our own - # TODO(harlowja): possibly check the module - # for having a LOG attr and just give it back - # its own logger? - func_args = [name, self.cfg, - cc, config.LOG, args] - # Mark it as having started running - which_ran.append(name) - # This name will affect the semaphore name created - run_name = "config-%s" % (name) - - desc = "running %s with frequency %s" % (run_name, freq) - myrep = events.ReportEventStack( - name=run_name, description=desc, parent=self.reporter) - - with myrep: - ran, _r = cc.run(run_name, mod.handle, func_args, - freq=freq) - if ran: - myrep.message = "%s ran successfully" % run_name - else: - myrep.message = "%s previously ran" % run_name - - except Exception as e: - util.logexc(LOG, "Running module %s (%s) failed", name, mod) - failures.append((name, e)) - return (which_ran, failures) - - def run_single(self, mod_name, args=None, freq=None): - # Form the users module 'specs' - mod_to_be = { - 'mod': mod_name, - 'args': args, - 'freq': freq, - } - # Now resume doing the normal fixups and running - raw_mods = [mod_to_be] - mostly_mods = self._fixup_modules(raw_mods) - return self._run_modules(mostly_mods) - - def run_section(self, section_name): - raw_mods = self._read_modules(section_name) - mostly_mods = self._fixup_modules(raw_mods) - d_name = self.init.distro.name - - skipped = [] - forced = [] - overridden = self.cfg.get('unverified_modules', []) - for (mod, name, _freq, _args) in mostly_mods: - worked_distros = set(mod.distros) - worked_distros.update( - distros.Distro.expand_osfamily(mod.osfamilies)) - - # module does not declare 'distros' or lists this distro - if not worked_distros or d_name in worked_distros: - continue - - if name in overridden: - forced.append(name) - else: - skipped.append(name) - - if skipped: - LOG.info("Skipping modules %s because they are not verified " - "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) - if forced: - LOG.info("running unverified_modules: %s", forced) - - return self._run_modules(mostly_mods) - - -def fetch_base_config(): - base_cfgs = [] - default_cfg = util.get_builtin_cfg() - - # Anything in your conf.d location?? - # or the 'default' cloud.cfg location??? - base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) - - # Kernel/cmdline parameters override system config - kern_contents = util.read_cc_from_cmdline() - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) - - # And finally the default gets to play - if default_cfg: - base_cfgs.append(default_cfg) - - return util.mergemanydict(base_cfgs) - - -def _pkl_store(obj, fname): - try: - pk_contents = pickle.dumps(obj) - except Exception: - util.logexc(LOG, "Failed pickling datasource %s", obj) - return False - try: - util.write_file(fname, pk_contents, omode="wb", mode=0o400) - except Exception: - util.logexc(LOG, "Failed pickling datasource to %s", fname) - return False - return True - - -def _pkl_load(fname): - pickle_contents = None - try: - pickle_contents = util.load_file(fname, decode=False) - except Exception as e: - if os.path.isfile(fname): - LOG.warn("failed loading pickle in %s: %s" % (fname, e)) - pass - - # This is allowed so just return nothing successfully loaded... - if not pickle_contents: - return None - try: - return pickle.loads(pickle_contents) - except Exception: - util.logexc(LOG, "Failed loading pickled blob from %s", fname) - return None diff --git a/cloudinit/templater.py b/cloudinit/templater.py deleted file mode 100644 index 41ef27e3..00000000 --- a/cloudinit/templater.py +++ /dev/null @@ -1,155 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2016 Amazon.com, Inc. or its affiliates. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# Author: Andrew Jorgensen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import collections -import re - -try: - from Cheetah.Template import Template as CTemplate - CHEETAH_AVAILABLE = True -except (ImportError, AttributeError): - CHEETAH_AVAILABLE = False - -try: - import jinja2 - from jinja2 import Template as JTemplate - JINJA_AVAILABLE = True -except (ImportError, AttributeError): - JINJA_AVAILABLE = False - -from cloudinit import log as logging -from cloudinit import type_utils as tu -from cloudinit import util - -LOG = logging.getLogger(__name__) -TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) -BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') - - -def basic_render(content, params): - """This does simple replacement of bash variable like templates. - - It identifies patterns like ${a} or $a and can also identify patterns like - ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted - by key 'a'. - """ - - def replacer(match): - # Only 1 of the 2 groups will actually have a valid entry. - name = match.group(1) - if name is None: - name = match.group(2) - if name is None: - raise RuntimeError("Match encountered but no valid group present") - path = collections.deque(name.split(".")) - selected_params = params - while len(path) > 1: - key = path.popleft() - if not isinstance(selected_params, dict): - raise TypeError("Can not traverse into" - " non-dictionary '%s' of type %s while" - " looking for subkey '%s'" - % (selected_params, - tu.obj_name(selected_params), - key)) - selected_params = selected_params[key] - key = path.popleft() - if not isinstance(selected_params, dict): - raise TypeError("Can not extract key '%s' from non-dictionary" - " '%s' of type %s" - % (key, selected_params, - tu.obj_name(selected_params))) - return str(selected_params[key]) - - return BASIC_MATCHER.sub(replacer, content) - - -def detect_template(text): - - def cheetah_render(content, params): - return CTemplate(content, searchList=[params]).respond() - - def jinja_render(content, params): - # keep_trailing_newline is in jinja2 2.7+, not 2.6 - add = "\n" if content.endswith("\n") else "" - return JTemplate(content, - undefined=jinja2.StrictUndefined, - trim_blocks=True).render(**params) + add - - if text.find("\n") != -1: - ident, rest = text.split("\n", 1) - else: - ident = text - rest = '' - type_match = TYPE_MATCHER.match(ident) - if not type_match: - if CHEETAH_AVAILABLE: - LOG.debug("Using Cheetah as the renderer for unknown template.") - return ('cheetah', cheetah_render, text) - else: - return ('basic', basic_render, text) - else: - template_type = type_match.group(1).lower().strip() - if template_type not in ('jinja', 'cheetah', 'basic'): - raise ValueError("Unknown template rendering type '%s' requested" - % template_type) - if template_type == 'jinja' and not JINJA_AVAILABLE: - LOG.warn("Jinja not available as the selected renderer for" - " desired template, reverting to the basic renderer.") - return ('basic', basic_render, rest) - elif template_type == 'jinja' and JINJA_AVAILABLE: - return ('jinja', jinja_render, rest) - if template_type == 'cheetah' and not CHEETAH_AVAILABLE: - LOG.warn("Cheetah not available as the selected renderer for" - " desired template, reverting to the basic renderer.") - return ('basic', basic_render, rest) - elif template_type == 'cheetah' and CHEETAH_AVAILABLE: - return ('cheetah', cheetah_render, rest) - # Only thing left over is the basic renderer (it is always available). - return ('basic', basic_render, rest) - - -def render_from_file(fn, params): - if not params: - params = {} - template_type, renderer, content = detect_template(util.load_file(fn)) - LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type) - return renderer(content, params) - - -def render_to_file(fn, outfn, params, mode=0o644): - contents = render_from_file(fn, params) - util.write_file(outfn, contents, mode=mode) - - -def render_string_to_file(content, outfn, params, mode=0o644): - contents = render_string(content, params) - util.write_file(outfn, contents, mode=mode) - - -def render_string(content, params): - if not params: - params = {} - template_type, renderer, content = detect_template(content) - return renderer(content, params) diff --git a/cloudinit/type_utils.py b/cloudinit/type_utils.py deleted file mode 100644 index b93efd6a..00000000 --- a/cloudinit/type_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import types - -import six - - -if six.PY3: - _NAME_TYPES = ( - types.ModuleType, - types.FunctionType, - types.LambdaType, - type, - ) -else: - _NAME_TYPES = ( - types.TypeType, - types.ModuleType, - types.FunctionType, - types.LambdaType, - types.ClassType, - ) - - -def obj_name(obj): - if isinstance(obj, _NAME_TYPES): - return six.text_type(obj.__name__) - else: - if not hasattr(obj, '__class__'): - return repr(obj) - else: - return obj_name(obj.__class__) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py deleted file mode 100644 index c05e9d90..00000000 --- a/cloudinit/url_helper.py +++ /dev/null @@ -1,509 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import json -import os -import requests -import six -import time - -from email.utils import parsedate -from functools import partial - -import oauthlib.oauth1 as oauth1 -from requests import exceptions - -from six.moves.urllib.parse import ( - urlparse, urlunparse, - quote as urlquote) - -from cloudinit import log as logging -from cloudinit import version - -LOG = logging.getLogger(__name__) - -if six.PY2: - import httplib - NOT_FOUND = httplib.NOT_FOUND -else: - import http.client - NOT_FOUND = http.client.NOT_FOUND - - -# Check if requests has ssl support (added in requests >= 0.8.8) -SSL_ENABLED = False -CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) -_REQ_VER = None -try: - from distutils.version import LooseVersion - import pkg_resources - _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) - if _REQ_VER >= LooseVersion('0.8.8'): - SSL_ENABLED = True - if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): - CONFIG_ENABLED = True -except ImportError: - pass - - -def _cleanurl(url): - parsed_url = list(urlparse(url, scheme='http')) - if not parsed_url[1] and parsed_url[2]: - # Swap these since this seems to be a common - # occurrence when given urls like 'www.google.com' - parsed_url[1] = parsed_url[2] - parsed_url[2] = '' - return urlunparse(parsed_url) - - -def combine_url(base, *add_ons): - - def combine_single(url, add_on): - url_parsed = list(urlparse(url)) - path = url_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += urlquote(str(add_on), safe="/:") - url_parsed[2] = path - return urlunparse(url_parsed) - - url = base - for add_on in add_ons: - url = combine_single(url, add_on) - return url - - -# Made to have same accessors as UrlResponse so that the -# read_file_or_url can return this or that object and the -# 'user' of those objects will not need to know the difference. -class StringResponse(object): - def __init__(self, contents, code=200): - self.code = code - self.headers = {} - self.contents = contents - self.url = None - - def ok(self, *args, **kwargs): - if self.code != 200: - return False - return True - - def __str__(self): - return self.contents - - -class FileResponse(StringResponse): - def __init__(self, path, contents, code=200): - StringResponse.__init__(self, contents, code=code) - self.url = path - - -class UrlResponse(object): - def __init__(self, response): - self._response = response - - @property - def contents(self): - return self._response.content - - @property - def url(self): - return self._response.url - - def ok(self, redirects_ok=False): - upper = 300 - if redirects_ok: - upper = 400 - if self.code >= 200 and self.code < upper: - return True - else: - return False - - @property - def headers(self): - return self._response.headers - - @property - def code(self): - return self._response.status_code - - def __str__(self): - return self._response.text - - -class UrlError(IOError): - def __init__(self, cause, code=None, headers=None, url=None): - IOError.__init__(self, str(cause)) - self.cause = cause - self.code = code - self.headers = headers - if self.headers is None: - self.headers = {} - self.url = url - - -def _get_ssl_args(url, ssl_details): - ssl_args = {} - scheme = urlparse(url).scheme - if scheme == 'https' and ssl_details: - if not SSL_ENABLED: - LOG.warn("SSL is not supported in requests v%s, " - "cert. verification can not occur!", _REQ_VER) - else: - if 'ca_certs' in ssl_details and ssl_details['ca_certs']: - ssl_args['verify'] = ssl_details['ca_certs'] - else: - ssl_args['verify'] = True - if 'cert_file' in ssl_details and 'key_file' in ssl_details: - ssl_args['cert'] = [ssl_details['cert_file'], - ssl_details['key_file']] - elif 'cert_file' in ssl_details: - ssl_args['cert'] = str(ssl_details['cert_file']) - return ssl_args - - -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): - url = _cleanurl(url) - req_args = { - 'url': url, - } - req_args.update(_get_ssl_args(url, ssl_details)) - req_args['allow_redirects'] = allow_redirects - req_args['method'] = 'GET' - if timeout is not None: - req_args['timeout'] = max(float(timeout), 0) - if data: - req_args['method'] = 'POST' - # It doesn't seem like config - # was added in older library versions (or newer ones either), thus we - # need to manually do the retries if it wasn't... - if CONFIG_ENABLED: - req_config = { - 'store_cookies': False, - } - # Don't use the retry support built-in - # since it doesn't allow for 'sleep_times' - # in between tries.... - # if retries: - # req_config['max_retries'] = max(int(retries), 0) - req_args['config'] = req_config - manual_tries = 1 - if retries: - manual_tries = max(int(retries) + 1, 1) - - def_headers = { - 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), - } - if headers: - def_headers.update(headers) - headers = def_headers - - if not headers_cb: - def _cb(url): - return headers - headers_cb = _cb - if data: - req_args['data'] = data - if sec_between is None: - sec_between = -1 - - excps = [] - # Handle retrying ourselves since the built-in support - # doesn't handle sleeping between tries... - for i in range(0, manual_tries): - req_args['headers'] = headers_cb(url) - filtered_req_args = {} - for (k, v) in req_args.items(): - if k == 'data': - continue - filtered_req_args[k] = v - try: - LOG.debug("[%s/%s] open '%s' with %s configuration", i, - manual_tries, url, filtered_req_args) - - r = requests.request(**req_args) - if check_status: - r.raise_for_status() - LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - r.status_code, len(r.content), (i + 1)) - # Doesn't seem like we can make it use a different - # subclass for responses, so add our own backward-compat - # attrs - return UrlResponse(r) - except exceptions.RequestException as e: - if (isinstance(e, (exceptions.HTTPError)) and - hasattr(e, 'response') and # This appeared in v 0.10.8 - hasattr(e.response, 'status_code')): - excps.append(UrlError(e, code=e.response.status_code, - headers=e.response.headers, - url=url)) - else: - excps.append(UrlError(e, url=url)) - if SSL_ENABLED and isinstance(e, exceptions.SSLError): - # ssl exceptions are not going to get fixed by waiting a - # few seconds - break - if exception_cb and exception_cb(req_args.copy(), excps[-1]): - # if an exception callback was given it should return None - # a true-ish value means to break and re-raise the exception - break - if i + 1 < manual_tries and sec_between > 0: - LOG.debug("Please wait %s seconds while we wait to try again", - sec_between) - time.sleep(sec_between) - if excps: - raise excps[-1] - return None # Should throw before this... - - -def wait_for_url(urls, max_wait=None, timeout=None, - status_cb=None, headers_cb=None, sleep_time=1, - exception_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - a number <= 0 will always result in only one try - timeout: the timeout provided to urlopen - status_cb: call method with string message when a url is not available - headers_cb: call method with single argument of url to get headers - for request. - exception_cb: call method with 2 arguments 'msg' (per status_cb) and - 'exception', the exception that occurred. - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - start_time = time.time() - - def log_status_cb(msg, exc=None): - LOG.debug(msg) - - if status_cb is None: - status_cb = log_status_cb - - def timeup(max_wait, start_time): - return ((max_wait <= 0 or max_wait is None) or - (time.time() - start_time > max_wait)) - - loop_n = 0 - while True: - sleep_time = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, start_time): - break - if timeout and (now + timeout > (start_time + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((start_time + max_wait) - now) - - reason = "" - url_exc = None - try: - if headers_cb is not None: - headers = headers_cb(url) - else: - headers = {} - - response = readurl(url, headers=headers, timeout=timeout, - check_status=False) - if not response.contents: - reason = "empty response [%s]" % (response.code) - url_exc = UrlError(ValueError(reason), code=response.code, - headers=response.headers, url=url) - elif not response.ok(): - reason = "bad status code [%s]" % (response.code) - url_exc = UrlError(ValueError(reason), code=response.code, - headers=response.headers, url=url) - else: - return url - except UrlError as e: - reason = "request error [%s]" % e - url_exc = e - except Exception as e: - reason = "unexpected error [%s]" % e - url_exc = e - - time_taken = int(time.time() - start_time) - status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url, - time_taken, - max_wait, - reason) - status_cb(status_msg) - if exception_cb: - # This can be used to alter the headers that will be sent - # in the future, for example this is what the MAAS datasource - # does. - exception_cb(msg=status_msg, exception=url_exc) - - if timeup(max_wait, start_time): - break - - loop_n = loop_n + 1 - LOG.debug("Please wait %s seconds while we wait to try again", - sleep_time) - time.sleep(sleep_time) - - return False - - -class OauthUrlHelper(object): - def __init__(self, consumer_key=None, token_key=None, - token_secret=None, consumer_secret=None, - skew_data_file="/run/oauth_skew.json"): - self.consumer_key = consumer_key - self.consumer_secret = consumer_secret or "" - self.token_key = token_key - self.token_secret = token_secret - self.skew_data_file = skew_data_file - self._do_oauth = True - self.skew_change_limit = 5 - required = (self.token_key, self.token_secret, self.consumer_key) - if not any(required): - self._do_oauth = False - elif not all(required): - raise ValueError("all or none of token_key, token_secret, or " - "consumer_key can be set") - - old = self.read_skew_file() - self.skew_data = old or {} - - def read_skew_file(self): - if self.skew_data_file and os.path.isfile(self.skew_data_file): - with open(self.skew_data_file, mode="r") as fp: - return json.load(fp) - return None - - def update_skew_file(self, host, value): - # this is not atomic - if not self.skew_data_file: - return - cur = self.read_skew_file() - if cur is None: - cur = {} - cur[host] = value - with open(self.skew_data_file, mode="w") as fp: - fp.write(json.dumps(cur)) - - def exception_cb(self, msg, exception): - if not (isinstance(exception, UrlError) and - (exception.code == 403 or exception.code == 401)): - return - - if 'date' not in exception.headers: - LOG.warn("Missing header 'date' in %s response", exception.code) - return - - date = exception.headers['date'] - try: - remote_time = time.mktime(parsedate(date)) - except Exception as e: - LOG.warn("Failed to convert datetime '%s': %s", date, e) - return - - skew = int(remote_time - time.time()) - host = urlparse(exception.url).netloc - old_skew = self.skew_data.get(host, 0) - if abs(old_skew - skew) > self.skew_change_limit: - self.update_skew_file(host, skew) - LOG.warn("Setting oauth clockskew for %s to %d", host, skew) - self.skew_data[host] = skew - - return - - def headers_cb(self, url): - if not self._do_oauth: - return {} - - timestamp = None - host = urlparse(url).netloc - if self.skew_data and host in self.skew_data: - timestamp = int(time.time()) + self.skew_data[host] - - return oauth_headers( - url=url, consumer_key=self.consumer_key, - token_key=self.token_key, token_secret=self.token_secret, - consumer_secret=self.consumer_secret, timestamp=timestamp) - - def _wrapped(self, wrapped_func, args, kwargs): - kwargs['headers_cb'] = partial( - self._headers_cb, kwargs.get('headers_cb')) - kwargs['exception_cb'] = partial( - self._exception_cb, kwargs.get('exception_cb')) - return wrapped_func(*args, **kwargs) - - def wait_for_url(self, *args, **kwargs): - return self._wrapped(wait_for_url, args, kwargs) - - def readurl(self, *args, **kwargs): - return self._wrapped(readurl, args, kwargs) - - def _exception_cb(self, extra_exception_cb, msg, exception): - ret = None - try: - if extra_exception_cb: - ret = extra_exception_cb(msg, exception) - finally: - self.exception_cb(msg, exception) - return ret - - def _headers_cb(self, extra_headers_cb, url): - headers = {} - if extra_headers_cb: - headers = extra_headers_cb(url) - headers.update(self.headers_cb(url)) - return headers - - -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, - timestamp=None): - if timestamp: - timestamp = str(timestamp) - else: - timestamp = None - - client = oauth1.Client( - consumer_key, - client_secret=consumer_secret, - resource_owner_key=token_key, - resource_owner_secret=token_secret, - signature_method=oauth1.SIGNATURE_PLAINTEXT, - timestamp=timestamp) - uri, signed_headers, body = client.sign(url) - return signed_headers diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py deleted file mode 100644 index 393bf0bb..00000000 --- a/cloudinit/user_data.py +++ /dev/null @@ -1,356 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from email.mime.base import MIMEBase -from email.mime.multipart import MIMEMultipart -from email.mime.nonmultipart import MIMENonMultipart -from email.mime.text import MIMEText - -import six - -from cloudinit import handlers -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - -# Constants copied in from the handler module -NOT_MULTIPART_TYPE = handlers.NOT_MULTIPART_TYPE -PART_FN_TPL = handlers.PART_FN_TPL -OCTET_TYPE = handlers.OCTET_TYPE - -# Saves typing errors -CONTENT_TYPE = 'Content-Type' - -# Various special content types that cause special actions -TYPE_NEEDED = ["text/plain", "text/x-not-multipart"] -INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] -ARCHIVE_TYPES = ["text/cloud-config-archive"] -UNDEF_TYPE = "text/plain" -ARCHIVE_UNDEF_TYPE = "text/cloud-config" -ARCHIVE_UNDEF_BINARY_TYPE = "application/octet-stream" - -# This seems to hit most of the gzip possible content types. -DECOMP_TYPES = [ - 'application/gzip', - 'application/gzip-compressed', - 'application/gzipped', - 'application/x-compress', - 'application/x-compressed', - 'application/x-gunzip', - 'application/x-gzip', - 'application/x-gzip-compressed', -] - -# Msg header used to track attachments -ATTACHMENT_FIELD = 'Number-Attachments' - -# Only the following content types can have there launch index examined -# in there payload, evey other content type can still provide a header -EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"] - - -def _replace_header(msg, key, value): - del msg[key] - msg[key] = value - - -def _set_filename(msg, filename): - del msg['Content-Disposition'] - msg.add_header('Content-Disposition', - 'attachment', filename=str(filename)) - - -class UserDataProcessor(object): - def __init__(self, paths): - self.paths = paths - self.ssl_details = util.fetch_ssl_details(paths) - - def process(self, blob): - accumulating_msg = MIMEMultipart() - if isinstance(blob, list): - for b in blob: - self._process_msg(convert_string(b), accumulating_msg) - else: - self._process_msg(convert_string(blob), accumulating_msg) - return accumulating_msg - - def _process_msg(self, base_msg, append_msg): - - def find_ctype(payload): - return handlers.type_from_starts_with(payload) - - for part in base_msg.walk(): - if is_skippable(part): - continue - - ctype = None - ctype_orig = part.get_content_type() - payload = util.fully_decoded_payload(part) - was_compressed = False - - # When the message states it is of a gzipped content type ensure - # that we attempt to decode said payload so that the decompressed - # data can be examined (instead of the compressed data). - if ctype_orig in DECOMP_TYPES: - try: - payload = util.decomp_gzip(payload, quiet=False) - # At this point we don't know what the content-type is - # since we just decompressed it. - ctype_orig = None - was_compressed = True - except util.DecompressionError as e: - LOG.warn("Failed decompressing payload from %s of length" - " %s due to: %s", ctype_orig, len(payload), e) - continue - - # Attempt to figure out the payloads content-type - if not ctype_orig: - ctype_orig = UNDEF_TYPE - if ctype_orig in TYPE_NEEDED: - ctype = find_ctype(payload) - if ctype is None: - ctype = ctype_orig - - # In the case where the data was compressed, we want to make sure - # that we create a new message that contains the found content - # type with the uncompressed content since later traversals of the - # messages will expect a part not compressed. - if was_compressed: - maintype, subtype = ctype.split("/", 1) - n_part = MIMENonMultipart(maintype, subtype) - n_part.set_payload(payload) - # Copy various headers from the old part to the new one, - # but don't include all the headers since some are not useful - # after decoding and decompression. - if part.get_filename(): - _set_filename(n_part, part.get_filename()) - for h in ('Launch-Index',): - if h in part: - _replace_header(n_part, h, str(part[h])) - part = n_part - - if ctype != ctype_orig: - _replace_header(part, CONTENT_TYPE, ctype) - - if ctype in INCLUDE_TYPES: - self._do_include(payload, append_msg) - continue - - if ctype in ARCHIVE_TYPES: - self._explode_archive(payload, append_msg) - continue - - # TODO(harlowja): Should this be happening, shouldn't - # the part header be modified and not the base? - _replace_header(base_msg, CONTENT_TYPE, ctype) - - self._attach_part(append_msg, part) - - def _attach_launch_index(self, msg): - header_idx = msg.get('Launch-Index', None) - payload_idx = None - if msg.get_content_type() in EXAMINE_FOR_LAUNCH_INDEX: - try: - # See if it has a launch-index field - # that might affect the final header - payload = util.load_yaml(msg.get_payload(decode=True)) - if payload: - payload_idx = payload.get('launch-index') - except Exception: - pass - # Header overrides contents, for now (?) or the other way around? - if header_idx is not None: - payload_idx = header_idx - # Nothing found in payload, use header (if anything there) - if payload_idx is None: - payload_idx = header_idx - if payload_idx is not None: - try: - msg.add_header('Launch-Index', str(int(payload_idx))) - except (ValueError, TypeError): - pass - - def _get_include_once_filename(self, entry): - entry_fn = util.hash_blob(entry, 'md5', 64) - return os.path.join(self.paths.get_ipath_cur('data'), - 'urlcache', entry_fn) - - def _process_before_attach(self, msg, attached_id): - if not msg.get_filename(): - _set_filename(msg, PART_FN_TPL % (attached_id)) - self._attach_launch_index(msg) - - def _do_include(self, content, append_msg): - # Include a list of urls, one per line - # also support '#include ' - # or #include-once '' - include_once_on = False - for line in content.splitlines(): - lc_line = line.lower() - if lc_line.startswith("#include-once"): - line = line[len("#include-once"):].lstrip() - # Every following include will now - # not be refetched.... but will be - # re-read from a local urlcache (if it worked) - include_once_on = True - elif lc_line.startswith("#include"): - line = line[len("#include"):].lstrip() - # Disable the include once if it was on - # if it wasn't, then this has no effect. - include_once_on = False - if line.startswith("#"): - continue - include_url = line.strip() - if not include_url: - continue - - include_once_fn = None - content = None - if include_once_on: - include_once_fn = self._get_include_once_filename(include_url) - if include_once_on and os.path.isfile(include_once_fn): - content = util.load_file(include_once_fn) - else: - resp = util.read_file_or_url(include_url, - ssl_details=self.ssl_details) - if include_once_on and resp.ok(): - util.write_file(include_once_fn, resp.contents, mode=0o600) - if resp.ok(): - content = resp.contents - else: - LOG.warn(("Fetching from %s resulted in" - " a invalid http code of %s"), - include_url, resp.code) - - if content is not None: - new_msg = convert_string(content) - self._process_msg(new_msg, append_msg) - - def _explode_archive(self, archive, append_msg): - entries = util.load_yaml(archive, default=[], allowed=(list, set)) - for ent in entries: - # ent can be one of: - # dict { 'filename' : 'value', 'content' : - # 'value', 'type' : 'value' } - # filename and type not be present - # or - # scalar(payload) - if isinstance(ent, six.string_types): - ent = {'content': ent} - if not isinstance(ent, (dict)): - # TODO(harlowja) raise? - continue - - content = ent.get('content', '') - mtype = ent.get('type') - if not mtype: - default = ARCHIVE_UNDEF_TYPE - if isinstance(content, six.binary_type): - default = ARCHIVE_UNDEF_BINARY_TYPE - mtype = handlers.type_from_starts_with(content, default) - - maintype, subtype = mtype.split('/', 1) - if maintype == "text": - if isinstance(content, six.binary_type): - content = content.decode() - msg = MIMEText(content, _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(content) - - if 'filename' in ent: - _set_filename(msg, ent['filename']) - if 'launch-index' in ent: - msg.add_header('Launch-Index', str(ent['launch-index'])) - - for header in list(ent.keys()): - if header.lower() in ('content', 'filename', 'type', - 'launch-index', 'content-disposition', - ATTACHMENT_FIELD.lower(), - CONTENT_TYPE.lower()): - continue - msg.add_header(header, ent[header]) - - self._attach_part(append_msg, msg) - - def _multi_part_count(self, outer_msg, new_count=None): - """ - Return the number of attachments to this MIMEMultipart by looking - at its 'Number-Attachments' header. - """ - if ATTACHMENT_FIELD not in outer_msg: - outer_msg[ATTACHMENT_FIELD] = '0' - - if new_count is not None: - _replace_header(outer_msg, ATTACHMENT_FIELD, str(new_count)) - - fetched_count = 0 - try: - fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) - except (ValueError, TypeError): - _replace_header(outer_msg, ATTACHMENT_FIELD, str(fetched_count)) - return fetched_count - - def _attach_part(self, outer_msg, part): - """ - Attach a message to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in the outer message to keep track of number of - attachments. - """ - part_count = self._multi_part_count(outer_msg) - self._process_before_attach(part, part_count + 1) - outer_msg.attach(part) - self._multi_part_count(outer_msg, part_count + 1) - - -def is_skippable(part): - # multipart/* are just containers - part_maintype = part.get_content_maintype() or '' - if part_maintype.lower() == 'multipart': - return True - return False - - -# Coverts a raw string into a mime message -def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE): - if not raw_data: - raw_data = '' - - def create_binmsg(data, content_type): - maintype, subtype = content_type.split("/", 1) - msg = MIMEBase(maintype, subtype) - msg.set_payload(data) - return msg - - try: - data = util.decode_binary(util.decomp_gzip(raw_data)) - if "mime-version:" in data[0:4096].lower(): - msg = util.message_from_string(data) - else: - msg = create_binmsg(data, content_type) - except UnicodeDecodeError: - msg = create_binmsg(raw_data, content_type) - - return msg diff --git a/cloudinit/util.py b/cloudinit/util.py deleted file mode 100644 index e5dd61a0..00000000 --- a/cloudinit/util.py +++ /dev/null @@ -1,2246 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import contextlib -import copy as obj_copy -import ctypes -import email -import errno -import glob -import grp -import gzip -import hashlib -import json -import os -import os.path -import platform -import pwd -import random -import re -import shutil -import socket -import stat -import string -import subprocess -import sys -import tempfile -import time - -from base64 import b64decode, b64encode -from six.moves.urllib import parse as urlparse - -import six -import yaml - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import mergers -from cloudinit import safeyaml -from cloudinit import type_utils -from cloudinit import url_helper -from cloudinit import version - -from cloudinit.settings import (CFG_BUILTIN) - - -_DNS_REDIRECT_IP = None -LOG = logging.getLogger(__name__) - -# Helps cleanup filenames to ensure they aren't FS incompatible -FN_REPLACEMENTS = { - os.sep: '_', -} -FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) - -TRUE_STRINGS = ('true', '1', 'on', 'yes') -FALSE_STRINGS = ('off', '0', 'no', 'false') - - -# Helper utils to see if running in a container -CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], - ['running-in-container'], - ['lxc-is-container']) - -PROC_CMDLINE = None - - -def decode_binary(blob, encoding='utf-8'): - # Converts a binary type into a text type using given encoding. - if isinstance(blob, six.text_type): - return blob - return blob.decode(encoding) - - -def encode_text(text, encoding='utf-8'): - # Converts a text string into a binary type using given encoding. - if isinstance(text, six.binary_type): - return text - return text.encode(encoding) - - -def b64d(source): - # Base64 decode some data, accepting bytes or unicode/str, and returning - # str/unicode if the result is utf-8 compatible, otherwise returning bytes. - decoded = b64decode(source) - try: - return decoded.decode('utf-8') - except UnicodeDecodeError: - return decoded - - -def b64e(source): - # Base64 encode some data, accepting bytes or unicode/str, and returning - # str/unicode if the result is utf-8 compatible, otherwise returning bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return b64encode(source).decode('utf-8') - - -def fully_decoded_payload(part): - # In Python 3, decoding the payload will ironically hand us a bytes object. - # 'decode' means to decode according to Content-Transfer-Encoding, not - # according to any charset in the Content-Type. So, if we end up with - # bytes, first try to decode to str via CT charset, and failing that, try - # utf-8 using surrogate escapes. - cte_payload = part.get_payload(decode=True) - if (six.PY3 and - part.get_content_maintype() == 'text' and - isinstance(cte_payload, bytes)): - charset = part.get_charset() - if charset and charset.input_codec: - encoding = charset.input_codec - else: - encoding = 'utf-8' - return cte_payload.decode(encoding, errors='surrogateescape') - return cte_payload - - -# Path for DMI Data -DMI_SYS_PATH = "/sys/class/dmi/id" - -# dmidecode and /sys/class/dmi/id/* use different names for the same value, -# this allows us to refer to them by one canonical name -DMIDECODE_TO_DMI_SYS_MAPPING = { - 'baseboard-asset-tag': 'board_asset_tag', - 'baseboard-manufacturer': 'board_vendor', - 'baseboard-product-name': 'board_name', - 'baseboard-serial-number': 'board_serial', - 'baseboard-version': 'board_version', - 'bios-release-date': 'bios_date', - 'bios-vendor': 'bios_vendor', - 'bios-version': 'bios_version', - 'chassis-asset-tag': 'chassis_asset_tag', - 'chassis-manufacturer': 'chassis_vendor', - 'chassis-serial-number': 'chassis_serial', - 'chassis-version': 'chassis_version', - 'system-manufacturer': 'sys_vendor', - 'system-product-name': 'product_name', - 'system-serial-number': 'product_serial', - 'system-uuid': 'product_uuid', - 'system-version': 'product_version', -} - - -class ProcessExecutionError(IOError): - - MESSAGE_TMPL = ('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Reason: %(reason)s\n' - 'Stdout: %(stdout)r\n' - 'Stderr: %(stderr)r') - - def __init__(self, stdout=None, stderr=None, - exit_code=None, cmd=None, - description=None, reason=None, - errno=None): - if not cmd: - self.cmd = '-' - else: - self.cmd = cmd - - if not description: - self.description = 'Unexpected error while running command.' - else: - self.description = description - - if not isinstance(exit_code, six.integer_types): - self.exit_code = '-' - else: - self.exit_code = exit_code - - if not stderr: - self.stderr = '' - else: - self.stderr = stderr - - if not stdout: - self.stdout = '' - else: - self.stdout = stdout - - if reason: - self.reason = reason - else: - self.reason = '-' - - self.errno = errno - message = self.MESSAGE_TMPL % { - 'description': self.description, - 'cmd': self.cmd, - 'exit_code': self.exit_code, - 'stdout': self.stdout, - 'stderr': self.stderr, - 'reason': self.reason, - } - IOError.__init__(self, message) - # For backward compatibility with Python 2. - if not hasattr(self, 'message'): - self.message = message - - -class SeLinuxGuard(object): - def __init__(self, path, recursive=False): - # Late import since it might not always - # be possible to use this - try: - self.selinux = importer.import_module('selinux') - except ImportError: - self.selinux = None - self.path = path - self.recursive = recursive - - def __enter__(self): - if self.selinux and self.selinux.is_selinux_enabled(): - return True - else: - return False - - def __exit__(self, excp_type, excp_value, excp_traceback): - if not self.selinux or not self.selinux.is_selinux_enabled(): - return - if not os.path.lexists(self.path): - return - - path = os.path.realpath(self.path) - # path should be a string, not unicode - if six.PY2: - path = str(path) - try: - stats = os.lstat(path) - self.selinux.matchpathcon(path, stats[stat.ST_MODE]) - except OSError: - return - - LOG.debug("Restoring selinux mode for %s (recursive=%s)", - path, self.recursive) - self.selinux.restorecon(path, recursive=self.recursive) - - -class MountFailedError(Exception): - pass - - -class DecompressionError(Exception): - pass - - -def ExtendedTemporaryFile(**kwargs): - fh = tempfile.NamedTemporaryFile(**kwargs) - # Replace its unlink with a quiet version - # that does not raise errors when the - # file to unlink has been unlinked elsewhere.. - LOG.debug("Created temporary file %s", fh.name) - fh.unlink = del_file - - # Add a new method that will unlink - # right 'now' but still lets the exit - # method attempt to remove it (which will - # not throw due to our del file being quiet - # about files that are not there) - def unlink_now(): - fh.unlink(fh.name) - - setattr(fh, 'unlink_now', unlink_now) - return fh - - -def fork_cb(child_cb, *args, **kwargs): - fid = os.fork() - if fid == 0: - try: - child_cb(*args, **kwargs) - os._exit(0) - except Exception: - logexc(LOG, "Failed forking and calling callback %s", - type_utils.obj_name(child_cb)) - os._exit(1) - else: - LOG.debug("Forked child %s who will run callback %s", - fid, type_utils.obj_name(child_cb)) - - -def is_true(val, addons=None): - if isinstance(val, (bool)): - return val is True - check_set = TRUE_STRINGS - if addons: - check_set = list(check_set) + addons - if six.text_type(val).lower().strip() in check_set: - return True - return False - - -def is_false(val, addons=None): - if isinstance(val, (bool)): - return val is False - check_set = FALSE_STRINGS - if addons: - check_set = list(check_set) + addons - if six.text_type(val).lower().strip() in check_set: - return True - return False - - -def translate_bool(val, addons=None): - if not val: - # This handles empty lists and false and - # other things that python believes are false - return False - # If its already a boolean skip - if isinstance(val, (bool)): - return val - return is_true(val, addons) - - -def rand_str(strlen=32, select_from=None): - if not select_from: - select_from = string.ascii_letters + string.digits - return "".join([random.choice(select_from) for _x in range(0, strlen)]) - - -def rand_dict_key(dictionary, postfix=None): - if not postfix: - postfix = "" - while True: - newkey = rand_str(strlen=8) + "_" + postfix - if newkey not in dictionary: - break - return newkey - - -def read_conf(fname): - try: - return load_yaml(load_file(fname), default={}) - except IOError as e: - if e.errno == errno.ENOENT: - return {} - else: - raise - - -# Merges X lists, and then keeps the -# unique ones, but orders by sort order -# instead of by the original order -def uniq_merge_sorted(*lists): - return sorted(uniq_merge(*lists)) - - -# Merges X lists and then iterates over those -# and only keeps the unique items (order preserving) -# and returns that merged and uniqued list as the -# final result. -# -# Note: if any entry is a string it will be -# split on commas and empty entries will be -# evicted and merged in accordingly. -def uniq_merge(*lists): - combined_list = [] - for a_list in lists: - if isinstance(a_list, six.string_types): - a_list = a_list.strip().split(",") - # Kickout the empty ones - a_list = [a for a in a_list if len(a)] - combined_list.extend(a_list) - return uniq_list(combined_list) - - -def clean_filename(fn): - for (k, v) in FN_REPLACEMENTS.items(): - fn = fn.replace(k, v) - removals = [] - for k in fn: - if k not in FN_ALLOWED: - removals.append(k) - for k in removals: - fn = fn.replace(k, '') - fn = fn.strip() - return fn - - -def decomp_gzip(data, quiet=True, decode=True): - try: - buf = six.BytesIO(encode_text(data)) - with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: - if decode: - return decode_binary(gh.read()) - else: - return gh.read() - except Exception as e: - if quiet: - return data - else: - raise DecompressionError(six.text_type(e)) - - -def extract_usergroup(ug_pair): - if not ug_pair: - return (None, None) - ug_parted = ug_pair.split(':', 1) - u = ug_parted[0].strip() - if len(ug_parted) == 2: - g = ug_parted[1].strip() - else: - g = None - if not u or u == "-1" or u.lower() == "none": - u = None - if not g or g == "-1" or g.lower() == "none": - g = None - return (u, g) - - -def find_modules(root_dir): - entries = dict() - for fname in glob.glob(os.path.join(root_dir, "*.py")): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] - modname = modname.strip() - if modname and modname.find(".") == -1: - entries[fname] = modname - return entries - - -def multi_log(text, console=True, stderr=True, - log=None, log_level=logging.DEBUG): - if stderr: - sys.stderr.write(text) - if console: - conpath = "/dev/console" - if os.path.exists(conpath): - with open(conpath, 'w') as wfh: - wfh.write(text) - wfh.flush() - else: - # A container may lack /dev/console (arguably a container bug). If - # it does not exist, then write output to stdout. this will result - # in duplicate stderr and stdout messages if stderr was True. - # - # even though upstart or systemd might have set up output to go to - # /dev/console, the user may have configured elsewhere via - # cloud-config 'output'. If there is /dev/console, messages will - # still get there. - sys.stdout.write(text) - if log: - if text[-1] == "\n": - log.log(log_level, text[:-1]) - else: - log.log(log_level, text) - - -def load_json(text, root_types=(dict,)): - decoded = json.loads(decode_binary(text)) - if not isinstance(decoded, tuple(root_types)): - expected_types = ", ".join([str(t) for t in root_types]) - raise TypeError("(%s) root types expected, got %s instead" - % (expected_types, type(decoded))) - return decoded - - -def is_ipv4(instr): - """determine if input string is a ipv4 address. return boolean.""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if int(x) < 256 and int(x) >= 0] - except Exception: - return False - - return len(toks) == 4 - - -def get_cfg_option_bool(yobj, key, default=False): - if key not in yobj: - return default - return translate_bool(yobj[key]) - - -def get_cfg_option_str(yobj, key, default=None): - if key not in yobj: - return default - val = yobj[key] - if not isinstance(val, six.string_types): - val = str(val) - return val - - -def get_cfg_option_int(yobj, key, default=0): - return int(get_cfg_option_str(yobj, key, default=default)) - - -def system_info(): - return { - 'platform': platform.platform(), - 'release': platform.release(), - 'python': platform.python_version(), - 'uname': platform.uname(), - 'dist': platform.linux_distribution(), - } - - -def get_cfg_option_list(yobj, key, default=None): - """ - Gets the C{key} config option from C{yobj} as a list of strings. If the - key is present as a single string it will be returned as a list with one - string arg. - - @param yobj: The configuration object. - @param key: The configuration key to get. - @param default: The default to return if key is not found. - @return: The configuration option as a list of strings or default if key - is not found. - """ - if key not in yobj: - return default - if yobj[key] is None: - return [] - val = yobj[key] - if isinstance(val, (list)): - cval = [v for v in val] - return cval - if not isinstance(val, six.string_types): - val = str(val) - return [val] - - -# get a cfg entry by its path array -# for f['a']['b']: get_cfg_by_path(mycfg,('a','b')) -def get_cfg_by_path(yobj, keyp, default=None): - cur = yobj - for tok in keyp: - if tok not in cur: - return default - cur = cur[tok] - return cur - - -def fixup_output(cfg, mode): - (outfmt, errfmt) = get_output_cfg(cfg, mode) - redirect_output(outfmt, errfmt) - return (outfmt, errfmt) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -# -# if _CLOUD_INIT_SAVE_STDOUT is set in environment to a non empty and true -# value then output input will not be closed (useful for debugging). -# -def redirect_output(outfmt, errfmt, o_out=None, o_err=None): - - if is_true(os.environ.get("_CLOUD_INIT_SAVE_STDOUT")): - LOG.debug("Not redirecting output due to _CLOUD_INIT_SAVE_STDOUT") - return - - if not o_out: - o_out = sys.stdout - if not o_err: - o_err = sys.stderr - - if outfmt: - LOG.debug("Redirecting %s to %s", o_out, outfmt) - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("Invalid type for output format: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - - if errfmt == outfmt: - LOG.debug("Redirecting %s to %s", o_err, outfmt) - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - LOG.debug("Redirecting %s to %s", o_err, errfmt) - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("Invalid type for error format: %s" % errfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - - -def make_url(scheme, host, port=None, - path='', params='', query='', fragment=''): - - pieces = [] - pieces.append(scheme or '') - - netloc = '' - if host: - netloc = str(host) - - if port is not None: - netloc += ":" + "%s" % (port) - - pieces.append(netloc or '') - pieces.append(path or '') - pieces.append(params or '') - pieces.append(query or '') - pieces.append(fragment or '') - - return urlparse.urlunparse(pieces) - - -def mergemanydict(srcs, reverse=False): - if reverse: - srcs = reversed(srcs) - merged_cfg = {} - for cfg in srcs: - if cfg: - # Figure out which mergers to apply... - mergers_to_apply = mergers.dict_extract_mergers(cfg) - if not mergers_to_apply: - mergers_to_apply = mergers.default_mergers() - merger = mergers.construct(mergers_to_apply) - merged_cfg = merger.merge(merged_cfg, cfg) - return merged_cfg - - -@contextlib.contextmanager -def chdir(ndir): - curr = os.getcwd() - try: - os.chdir(ndir) - yield ndir - finally: - os.chdir(curr) - - -@contextlib.contextmanager -def umask(n_msk): - old = os.umask(n_msk) - try: - yield old - finally: - os.umask(old) - - -@contextlib.contextmanager -def tempdir(**kwargs): - # This seems like it was only added in python 3.2 - # Make it since its useful... - # See: http://bugs.python.org/file12970/tempdir.patch - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - del_dir(tdir) - - -def center(text, fill, max_len): - return '{0:{fill}{align}{size}}'.format(text, fill=fill, - align="^", size=max_len) - - -def del_dir(path): - LOG.debug("Recursively deleting %s", path) - shutil.rmtree(path) - - -def runparts(dirp, skip_no_exist=True, exe_prefix=None): - if skip_no_exist and not os.path.isdir(dirp): - return - - failed = [] - attempted = [] - - if exe_prefix is None: - prefix = [] - elif isinstance(exe_prefix, str): - prefix = [str(exe_prefix)] - elif isinstance(exe_prefix, list): - prefix = exe_prefix - else: - raise TypeError("exe_prefix must be None, str, or list") - - for exe_name in sorted(os.listdir(dirp)): - exe_path = os.path.join(dirp, exe_name) - if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): - attempted.append(exe_path) - try: - subp(prefix + [exe_path], capture=False) - except ProcessExecutionError as e: - logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) - failed.append(e) - - if failed and attempted: - raise RuntimeError('Runparts: %s failures in %s attempted commands' - % (len(failed), len(attempted))) - - -# read_optional_seed -# returns boolean indicating success or failure (presense of files) -# if files are present, populates 'fill' dictionary with 'user-data' and -# 'meta-data' entries -def read_optional_seed(fill, base="", ext="", timeout=5): - try: - (md, ud) = read_seeded(base, ext, timeout) - fill['user-data'] = ud - fill['meta-data'] = md - return True - except url_helper.UrlError as e: - if e.code == url_helper.NOT_FOUND: - return False - raise - - -def fetch_ssl_details(paths=None): - ssl_details = {} - # Lookup in these locations for ssl key/cert files - ssl_cert_paths = [ - '/var/lib/cloud/data/ssl', - '/var/lib/cloud/instance/data/ssl', - ] - if paths: - ssl_cert_paths.extend([ - os.path.join(paths.get_ipath_cur('data'), 'ssl'), - os.path.join(paths.get_cpath('data'), 'ssl'), - ]) - ssl_cert_paths = uniq_merge(ssl_cert_paths) - ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] - cert_file = None - for d in ssl_cert_paths: - if os.path.isfile(os.path.join(d, 'cert.pem')): - cert_file = os.path.join(d, 'cert.pem') - break - key_file = None - for d in ssl_cert_paths: - if os.path.isfile(os.path.join(d, 'key.pem')): - key_file = os.path.join(d, 'key.pem') - break - if cert_file and key_file: - ssl_details['cert_file'] = cert_file - ssl_details['key_file'] = key_file - elif cert_file: - ssl_details['cert_file'] = cert_file - return ssl_details - - -def read_file_or_url(url, timeout=5, retries=10, - headers=None, data=None, sec_between=1, ssl_details=None, - headers_cb=None, exception_cb=None): - url = url.lstrip() - if url.startswith("/"): - url = "file://%s" % url - if url.lower().startswith("file://"): - if data: - LOG.warn("Unable to post data to file resource %s", url) - file_path = url[len("file://"):] - try: - contents = load_file(file_path, decode=False) - except IOError as e: - code = e.errno - if e.errno == errno.ENOENT: - code = url_helper.NOT_FOUND - raise url_helper.UrlError(cause=e, code=code, headers=None, - url=url) - return url_helper.FileResponse(file_path, contents=contents) - else: - return url_helper.readurl(url, - timeout=timeout, - retries=retries, - headers=headers, - headers_cb=headers_cb, - data=data, - sec_between=sec_between, - ssl_details=ssl_details, - exception_cb=exception_cb) - - -def load_yaml(blob, default=None, allowed=(dict,)): - loaded = default - blob = decode_binary(blob) - try: - LOG.debug("Attempting to load yaml from string " - "of length %s with allowed root types %s", - len(blob), allowed) - converted = safeyaml.load(blob) - if not isinstance(converted, allowed): - # Yes this will just be caught, but thats ok for now... - raise TypeError(("Yaml load allows %s root types," - " but got %s instead") % - (allowed, type_utils.obj_name(converted))) - loaded = converted - except (yaml.YAMLError, TypeError, ValueError): - if len(blob) == 0: - LOG.debug("load_yaml given empty string, returning default") - else: - logexc(LOG, "Failed loading yaml blob") - return loaded - - -def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): - if base.startswith("/"): - base = "file://%s" % base - - # default retries for file is 0. for network is 10 - if base.startswith("file://"): - retries = file_retries - - if base.find("%s") >= 0: - ud_url = base % ("user-data" + ext) - md_url = base % ("meta-data" + ext) - else: - ud_url = "%s%s%s" % (base, "user-data", ext) - md_url = "%s%s%s" % (base, "meta-data", ext) - - md_resp = read_file_or_url(md_url, timeout, retries, file_retries) - md = None - if md_resp.ok(): - md = load_yaml(decode_binary(md_resp.contents), default={}) - - ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) - ud = None - if ud_resp.ok(): - ud = ud_resp.contents - - return (md, ud) - - -def read_conf_d(confd): - # Get reverse sorted list (later trumps newer) - confs = sorted(os.listdir(confd), reverse=True) - - # Remove anything not ending in '.cfg' - confs = [f for f in confs if f.endswith(".cfg")] - - # Remove anything not a file - confs = [f for f in confs - if os.path.isfile(os.path.join(confd, f))] - - # Load them all so that they can be merged - cfgs = [] - for fn in confs: - cfgs.append(read_conf(os.path.join(confd, fn))) - - return mergemanydict(cfgs) - - -def read_conf_with_confd(cfgfile): - cfg = read_conf(cfgfile) - - confd = False - if "conf_d" in cfg: - confd = cfg['conf_d'] - if confd: - if not isinstance(confd, six.string_types): - raise TypeError(("Config file %s contains 'conf_d' " - "with non-string type %s") % - (cfgfile, type_utils.obj_name(confd))) - else: - confd = str(confd).strip() - elif os.path.isdir("%s.d" % cfgfile): - confd = "%s.d" % cfgfile - - if not confd or not os.path.isdir(confd): - return cfg - - # Conf.d settings override input configuration - confd_cfg = read_conf_d(confd) - return mergemanydict([confd_cfg, cfg]) - - -def read_cc_from_cmdline(cmdline=None): - # this should support reading cloud-config information from - # the kernel command line. It is intended to support content of the - # format: - # cc: [end_cc] - # this would include: - # cc: ssh_import_id: [smoser, kirkland]\\n - # cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc - # cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc - if cmdline is None: - cmdline = get_cmdline() - - tag_begin = "cc:" - tag_end = "end_cc" - begin_l = len(tag_begin) - end_l = len(tag_end) - clen = len(cmdline) - tokens = [] - begin = cmdline.find(tag_begin) - while begin >= 0: - end = cmdline.find(tag_end, begin + begin_l) - if end < 0: - end = clen - tokens.append(cmdline[begin + begin_l:end].lstrip().replace("\\n", - "\n")) - - begin = cmdline.find(tag_begin, end + end_l) - - return '\n'.join(tokens) - - -def dos2unix(contents): - # find first end of line - pos = contents.find('\n') - if pos <= 0 or contents[pos - 1] != '\r': - return contents - return contents.replace('\r\n', '\n') - - -def get_hostname_fqdn(cfg, cloud): - # return the hostname and fqdn from 'cfg'. If not found in cfg, - # then fall back to data from cloud - if "fqdn" in cfg: - # user specified a fqdn. Default hostname then is based off that - fqdn = cfg['fqdn'] - hostname = get_cfg_option_str(cfg, "hostname", fqdn.split('.')[0]) - else: - if "hostname" in cfg and cfg['hostname'].find('.') > 0: - # user specified hostname, and it had '.' in it - # be nice to them. set fqdn and hostname from that - fqdn = cfg['hostname'] - hostname = cfg['hostname'][:fqdn.find('.')] - else: - # no fqdn set, get fqdn from cloud. - # get hostname from cfg if available otherwise cloud - fqdn = cloud.get_hostname(fqdn=True) - if "hostname" in cfg: - hostname = cfg['hostname'] - else: - hostname = cloud.get_hostname() - return (hostname, fqdn) - - -def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): - """ - For each host a single line should be present with - the following information: - - IP_address canonical_hostname [aliases...] - - Fields of the entry are separated by any number of blanks and/or tab - characters. Text from a "#" character until the end of the line is a - comment, and is ignored. Host names may contain only alphanumeric - characters, minus signs ("-"), and periods ("."). They must begin with - an alphabetic character and end with an alphanumeric character. - Optional aliases provide for name changes, alternate spellings, shorter - hostnames, or generic hostnames (for example, localhost). - """ - fqdn = None - try: - for line in load_file(filename).splitlines(): - hashpos = line.find("#") - if hashpos >= 0: - line = line[0:hashpos] - line = line.strip() - if not line: - continue - - # If there there is less than 3 entries - # (IP_address, canonical_hostname, alias) - # then ignore this line - toks = line.split() - if len(toks) < 3: - continue - - if hostname in toks[2:]: - fqdn = toks[1] - break - except IOError: - pass - return fqdn - - -def get_cmdline_url(names=('cloud-config-url', 'url'), - starts=b"#cloud-config", cmdline=None): - if cmdline is None: - cmdline = get_cmdline() - - data = keyval_str_to_dict(cmdline) - url = None - key = None - for key in names: - if key in data: - url = data[key] - break - - if not url: - return (None, None, None) - - resp = read_file_or_url(url) - # allow callers to pass starts as text when comparing to bytes contents - starts = encode_text(starts) - if resp.ok() and resp.contents.startswith(starts): - return (key, url, resp.contents) - - return (key, url, None) - - -def is_resolvable(name): - """determine if a url is resolvable, return a boolean - This also attempts to be resilent against dns redirection. - - Note, that normal nsswitch resolution is used here. So in order - to avoid any utilization of 'search' entries in /etc/resolv.conf - we have to append '.'. - - The top level 'invalid' domain is invalid per RFC. And example.com - should also not exist. The random entry will be resolved inside - the search list. - """ - global _DNS_REDIRECT_IP - if _DNS_REDIRECT_IP is None: - badips = set() - badnames = ("does-not-exist.example.com.", "example.invalid.", - rand_str()) - badresults = {} - for iname in badnames: - try: - result = socket.getaddrinfo(iname, None, 0, 0, - socket.SOCK_STREAM, - socket.AI_CANONNAME) - badresults[iname] = [] - for (_fam, _stype, _proto, cname, sockaddr) in result: - badresults[iname].append("%s: %s" % (cname, sockaddr[0])) - badips.add(sockaddr[0]) - except (socket.gaierror, socket.error): - pass - _DNS_REDIRECT_IP = badips - if badresults: - LOG.debug("detected dns redirection: %s", badresults) - - try: - result = socket.getaddrinfo(name, None) - # check first result's sockaddr field - addr = result[0][4][0] - if addr in _DNS_REDIRECT_IP: - return False - return True - except (socket.gaierror, socket.error): - return False - - -def get_hostname(): - hostname = socket.gethostname() - return hostname - - -def gethostbyaddr(ip): - try: - return socket.gethostbyaddr(ip)[0] - except socket.herror: - return None - - -def is_resolvable_url(url): - """determine if this url is resolvable (existing or ip).""" - return is_resolvable(urlparse.urlparse(url).hostname) - - -def search_for_mirror(candidates): - """ - Search through a list of mirror urls for one that works - This needs to return quickly. - """ - for cand in candidates: - try: - if is_resolvable_url(cand): - return cand - except Exception: - pass - return None - - -def close_stdin(): - """ - reopen stdin as /dev/null so even subprocesses or other os level things get - /dev/null as input. - - if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty and true - value then input will not be closed (useful for debugging). - """ - if is_true(os.environ.get("_CLOUD_INIT_SAVE_STDIN")): - return - with open(os.devnull) as fp: - os.dup2(fp.fileno(), sys.stdin.fileno()) - - -def find_devs_with(criteria=None, oformat='device', - tag=None, no_cache=False, path=None): - """ - find devices matching given criteria (via blkid) - criteria can be *one* of: - TYPE= - LABEL=