diff options
322 files changed, 30459 insertions, 1440 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a6668557d..57d855544d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ Ansible Changes By Release in progress, details pending +* Add a clone parameter to git module that allows you to get information about a remote repo even if it doesn't exist locally. +* Safety changes: several modules have force parameters that defaulted to true. + These have been changed to default to false so as not to accidentally lose + work. Playbooks that depended on the former behaviour simply to add + force=True to the task that needs it. Affected modules: + + * bzr: When local modifications exist in a checkout, the bzr module used to + default to temoving the modifications on any operation. Now the module + will not remove the modifications unless force=yes is specified. + Operations that depend on a clean working tree may fail unless force=yes is + added. + * git: When local modifications exist in a checkout, the git module will now + fail unless force is explictly specified. Specifying force will allow the + module to revert and overwrite local modifications to make git actions + succeed. + * hg: When local modifications exist in a checkout, the hg module used to + default to removing the modifications on any operation. Now the module + will not remove the modifications unless force=yes is specified. + * subversion: When updating a checkout with local modifications, you now need + to add force so the module will revert the modifications before updating. + ## 1.8.1 "You Really Got Me" - Nov 26, 2014 * Various bug fixes in postgresql and mysql modules. @@ -10,7 +10,7 @@ # make deb-src -------------- produce a DEB source # make deb ------------------ produce a DEB # make docs ----------------- rebuild the manpages (results are checked in) -# make tests ---------------- run the tests +# make tests ---------------- run the tests (see test/README.md for requirements) # make pyflakes, make pep8 -- source code checks ######################################################## @@ -49,8 +49,7 @@ Branch Info Authors ======= -Ansible was created by [Michael DeHaan](https://github.com/mpdehaan) (michael@ansible.com) and has contributions from over -900 users (and growing). Thanks everyone! +Ansible was created by [Michael DeHaan](https://github.com/mpdehaan) (michael.dehaan/gmail/com) and has contributions from over 900 users (and growing). Thanks everyone! Ansible is sponsored by [Ansible, Inc](http://ansible.com) diff --git a/bin/ansible b/bin/ansible index b82a18d3d3..7ba615dbc0 100755 --- a/bin/ansible +++ b/bin/ansible @@ -19,6 +19,17 @@ ######################################################## +__requires__ = ['ansible'] +try: + import pkg_resources +except Exception: + # Use pkg_resources to find the correct versions of libraries and set + # sys.path appropriately when there are multiversion installs. But we + # have code that better expresses the errors in the places where the code + # is actually used (the deps are optional for many code paths) so we don't + # want to fail here. + pass + import os import sys diff --git a/bin/ansible-doc b/bin/ansible-doc index 0ba84b9a30..4420036677 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -34,7 +34,7 @@ import traceback MODULEDIR = C.DEFAULT_MODULE_PATH BLACKLIST_EXTS = ('.pyc', '.swp', '.bak', '~', '.rpm') -IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README" ] +IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README", "VERSION"] _ITALIC = re.compile(r"I\(([^)]+)\)") _BOLD = re.compile(r"B\(([^)]+)\)") @@ -71,7 +71,7 @@ def pager(text): pager_print(text) else: pager_pipe(text, os.environ['PAGER']) - elif hasattr(os, 'system') and os.system('(less) 2> /dev/null') == 0: + elif subprocess.call('(less --version) 2> /dev/null', shell = True) == 0: pager_pipe(text, 'less') else: pager_print(text) @@ -165,7 +165,10 @@ def get_snippet_text(doc): return "\n".join(text) def get_module_list_text(module_list): - columns = max(60, int(os.popen('stty size', 'r').read().split()[1])) + tty_size = 0 + if os.isatty(0): + tty_size = int(os.popen('stty size', 'r').read().split()[1]) + columns = max(60, tty_size) displace = max(len(x) for x in module_list) linelimit = columns - displace - 5 text = [] diff --git a/bin/ansible-playbook b/bin/ansible-playbook index 96e87de3eb..c3ceab9b27 100755 --- a/bin/ansible-playbook +++ b/bin/ansible-playbook @@ -18,8 +18,16 @@ ####################################################### -#__requires__ = ['ansible'] -#import pkg_resources +__requires__ = ['ansible'] +try: + import pkg_resources +except Exception: + # Use pkg_resources to find the correct versions of libraries and set + # sys.path appropriately when there are multiversion installs. But we + # have code that better expresses the errors in the places where the code + # is actually used (the deps are optional for many code paths) so we don't + # want to fail here. + pass import sys import os @@ -39,6 +47,7 @@ import ansible.utils.template from ansible import errors from ansible import callbacks from ansible import utils +from ansible.utils import to_unicode from ansible.color import ANSIBLE_COLOR, stringc from ansible.callbacks import display @@ -136,10 +145,11 @@ def main(args): extra_vars = {} for extra_vars_opt in options.extra_vars: - if extra_vars_opt.startswith("@"): + extra_vars_opt = to_unicode(extra_vars_opt) + if extra_vars_opt.startswith(u"@"): # Argument is a YAML file (JSON is a subset of YAML) extra_vars = utils.combine_vars(extra_vars, utils.parse_yaml_from_file(extra_vars_opt[1:], vault_password=vault_pass)) - elif extra_vars_opt and extra_vars_opt[0] in '[{': + elif extra_vars_opt and extra_vars_opt[0] in u'[{': # Arguments as YAML extra_vars = utils.combine_vars(extra_vars, utils.parse_yaml(extra_vars_opt)) else: @@ -158,9 +168,23 @@ def main(args): raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook) inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass) - inventory.subset(options.subset) + + # Note: slightly wrong, this is written so that implicit localhost + # (which is not returned in list_hosts()) is taken into account for + # warning if inventory is empty. But it can't be taken into account for + # checking if limit doesn't match any hosts. Instead we don't worry about + # limit if only implicit localhost was in inventory to start with. + # + # Fix this in v2 + no_hosts = False if len(inventory.list_hosts()) == 0: - raise errors.AnsibleError("provided hosts list is empty") + # Empty inventory + utils.warning("provided hosts list is empty, only localhost is available") + no_hosts = True + inventory.subset(options.subset) + if len(inventory.list_hosts()) == 0 and no_hosts is False: + # Invalid limit + raise errors.AnsibleError("Specified --limit does not match any hosts") # run all playbooks specified on the command line for playbook in args: @@ -276,7 +300,7 @@ def main(args): retries = failed_hosts + unreachable_hosts - if len(retries) > 0: + if C.RETRY_FILES_ENABLED and len(retries) > 0: filename = pb.generate_retry_inventory(retries) if filename: display(" to retry, use: --limit @%s\n" % filename) diff --git a/bin/ansible-pull b/bin/ansible-pull index 2d91324315..0cea19fb35 100755 --- a/bin/ansible-pull +++ b/bin/ansible-pull @@ -40,7 +40,6 @@ import os import shutil -import subprocess import sys import datetime import socket @@ -137,6 +136,10 @@ def main(args): help='ask for sudo password') parser.add_option('-t', '--tags', dest='tags', default=False, help='only run plays and tasks tagged with these values') + parser.add_option('--accept-host-key', default=False, dest='accept_host_key', action='store_true', + help='adds the hostkey for the repo url if not already added') + parser.add_option('--key-file', dest='key_file', + help="Pass '-i <key_file>' to the SSH arguments used by git.") options, args = parser.parse_args(args) hostname = socket.getfqdn() @@ -151,7 +154,7 @@ def main(args): return 1 now = datetime.datetime.now() - print >>sys.stderr, now.strftime("Starting ansible-pull at %F %T") + print now.strftime("Starting ansible-pull at %F %T") # Attempt to use the inventory passed in as an argument # It might not yet have been downloaded so use localhost if note @@ -170,6 +173,15 @@ def main(args): if options.checkout: repo_opts += ' version=%s' % options.checkout + + # Only git module is supported + if options.module_name == DEFAULT_REPO_TYPE: + if options.accept_host_key: + repo_opts += ' accept_hostkey=yes' + + if options.key_file: + repo_opts += ' key_file=%s' % options.key_file + path = utils.plugins.module_finder.find_plugin(options.module_name) if path is None: sys.stderr.write("module '%s' not found.\n" % options.module_name) @@ -194,7 +206,7 @@ def main(args): if rc != 0: if options.force: - print "Unable to update repository. Continuing with (forced) run of playbook." + print >>sys.stderr, "Unable to update repository. Continuing with (forced) run of playbook." else: return rc elif options.ifchanged and '"changed": true' not in out: diff --git a/bin/ansible-vault b/bin/ansible-vault index 10da6a9ea7..22cfc0e148 100755 --- a/bin/ansible-vault +++ b/bin/ansible-vault @@ -18,8 +18,16 @@ # ansible-vault is a script that encrypts/decrypts YAML files. See # http://docs.ansible.com/playbooks_vault.html for more details. -#__requires__ = ['ansible'] -#import pkg_resources +__requires__ = ['ansible'] +try: + import pkg_resources +except Exception: + # Use pkg_resources to find the correct versions of libraries and set + # sys.path appropriately when there are multiversion installs. But we + # have code that better expresses the errors in the places where the code + # is actually used (the deps are optional for many code paths) so we don't + # want to fail here. + pass import os import sys diff --git a/docsite/rst/YAMLSyntax.rst b/docsite/rst/YAMLSyntax.rst index 3230a39f24..4b85ebac30 100644 --- a/docsite/rst/YAMLSyntax.rst +++ b/docsite/rst/YAMLSyntax.rst @@ -25,7 +25,7 @@ Ansible or not) should begin with ``---``. This is part of the YAML format and indicates the start of a document. All members of a list are lines beginning at the same indentation level starting -with a ``-`` (dash) character:: +with a ``"- "`` (dash and whitespace) character:: --- # A list of tasty fruits @@ -34,7 +34,7 @@ with a ``-`` (dash) character:: - Strawberry - Mango -A dictionary is represented in a simple ``key:`` and ``value`` form:: +A dictionary is represented in a simple ``key: `` (colon and whitespace) and ``value`` form:: --- # An employee record diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index decd5b305c..82edea9de8 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -441,12 +441,12 @@ Getting Your Module Into Ansible ```````````````````````````````` High-quality modules with minimal dependencies -can be included in the ansible, but modules (just due to the programming +can be included in Ansible, but modules (just due to the programming preferences of the developers) will need to be implemented in Python and use the AnsibleModule common code, and should generally use consistent arguments with the rest of the program. Stop by the mailing list to inquire about requirements if you like, and submit a github pull request to the `extras <https://github.com/ansible/ansible-modules-extras>`_ project. -Included modules will ship with ansible, and also have a change to be promoted to 'core' status, which +Included modules will ship with ansible, and also have a chance to be promoted to 'core' status, which gives them slightly higher development priority (though they'll work in exactly the same way). diff --git a/docsite/rst/developing_plugins.rst b/docsite/rst/developing_plugins.rst index e758644359..a54e8830f2 100644 --- a/docsite/rst/developing_plugins.rst +++ b/docsite/rst/developing_plugins.rst @@ -30,7 +30,7 @@ Lookup Plugins Language constructs like "with_fileglob" and "with_items" are implemented via lookup plugins. Just like other plugin types, you can write your own. -More documentation on writing connection plugins is pending, though you can jump into `lib/ansible/runner/lookup_plugins <https://github.com/ansible/ansible/tree/devel/lib/ansible/runner/lookup_plugins>`_ and figure +More documentation on writing lookup plugins is pending, though you can jump into `lib/ansible/runner/lookup_plugins <https://github.com/ansible/ansible/tree/devel/lib/ansible/runner/lookup_plugins>`_ and figure things out pretty easily. .. _developing_vars_plugins: @@ -42,7 +42,7 @@ Playbook constructs like 'host_vars' and 'group_vars' work via 'vars' plugins. data into ansible runs that did not come from an inventory, playbook, or command line. Note that variables can also be returned from inventory, so in most cases, you won't need to write or understand vars_plugins. -More documentation on writing connection plugins is pending, though you can jump into `lib/ansible/inventory/vars_plugins <https://github.com/ansible/ansible/tree/devel/lib/ansible/inventory/vars_plugins>`_ and figure +More documentation on writing vars plugins is pending, though you can jump into `lib/ansible/inventory/vars_plugins <https://github.com/ansible/ansible/tree/devel/lib/ansible/inventory/vars_plugins>`_ and figure things out pretty easily. If you find yourself wanting to write a vars_plugin, it's more likely you should write an inventory script instead. diff --git a/docsite/rst/guide_aws.rst b/docsite/rst/guide_aws.rst index 623a80fe40..7cfffc218d 100644 --- a/docsite/rst/guide_aws.rst +++ b/docsite/rst/guide_aws.rst @@ -56,7 +56,7 @@ In the example below, the "exact_count" of instances is set to 5. This means if be terminated. What is being counted is specified by the "count_tag" parameter. The parameter "instance_tags" is used to apply tags to the newly created -instance. +instance.:: # demo_setup.yml @@ -82,7 +82,7 @@ instance. The data about what instances are created is being saved by the "register" keyword in the variable named "ec2". -From this, we'll use the add_host module to dynamically create a host group consisting of these new instances. This facilitates performing configuration actions on the hosts immediately in a subsequent task:: +From this, we'll use the add_host module to dynamically create a host group consisting of these new instances. This facilitates performing configuration actions on the hosts immediately in a subsequent task.:: # demo_setup.yml @@ -104,6 +104,7 @@ From this, we'll use the add_host module to dynamically create a host group cons Name: Demo instance_tags: Name: Demo + register: ec2 - name: Add all instance public IPs to host group add_host: hostname={{ item.public_ip }} groupname=ec2hosts @@ -223,7 +224,7 @@ Generally speaking, we find most users using Packer. If you do not want to adopt Packer at this time, configuring a base-image with Ansible after provisioning (as shown above) is acceptable. -.. aws_next_steps:: +.. _aws_next_steps: Next Steps: Explore Modules ``````````````````````````` diff --git a/docsite/rst/guide_gce.rst b/docsite/rst/guide_gce.rst index 0cf413c58f..c0f90f13e2 100644 --- a/docsite/rst/guide_gce.rst +++ b/docsite/rst/guide_gce.rst @@ -133,7 +133,7 @@ For the following use case, let's use this small shell script as a wrapper. .. code-block:: bash - #!/bin/bash + #!/usr/bin/env bash PLAYBOOK="$1" if [[ -z $PLAYBOOK ]]; then diff --git a/docsite/rst/guide_rax.rst b/docsite/rst/guide_rax.rst index d00a090fa3..2a2f415e69 100644 --- a/docsite/rst/guide_rax.rst +++ b/docsite/rst/guide_rax.rst @@ -163,7 +163,7 @@ In Ansible it is quite possible to use multiple dynamic inventory plugins along rax.py ++++++ -To use the rackspace dynamic inventory script, copy ``rax.py`` into your inventory directory and make it executable. You can specify a credentails file for ``rax.py`` utilizing the ``RAX_CREDS_FILE`` environment variable. +To use the rackspace dynamic inventory script, copy ``rax.py`` into your inventory directory and make it executable. You can specify a credentials file for ``rax.py`` utilizing the ``RAX_CREDS_FILE`` environment variable. .. note:: Dynamic inventory scripts (like ``rax.py``) are saved in ``/usr/share/ansible/inventory`` if Ansible has been installed globally. If installed to a virtualenv, the inventory scripts are installed to ``$VIRTUALENV/share/inventory``. diff --git a/docsite/rst/guide_vagrant.rst b/docsite/rst/guide_vagrant.rst index 8dc8d10b44..f61fd84feb 100644 --- a/docsite/rst/guide_vagrant.rst +++ b/docsite/rst/guide_vagrant.rst @@ -107,14 +107,16 @@ inventory file may look something like this: If you want to run Ansible manually, you will want to make sure to pass ``ansible`` or ``ansible-playbook`` commands the correct arguments for the -username (usually ``vagrant``) and the SSH key (usually -``~/.vagrant.d/insecure_private_key``), and the autogenerated inventory file. +username (usually ``vagrant``) and the SSH key (since Vagrant 1.7.0, this will be something like +``.vagrant/machines/[machine name]/[provider]/private_key``), and the autogenerated inventory file. Here is an example: .. code-block:: bash - $ ansible-playbook -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory --private-key=~/.vagrant.d/insecure_private_key -u vagrant playbook.yml + $ ansible-playbook -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory --private-key=.vagrant/machines/default/virtualbox/private_key -u vagrant playbook.yml + +Note: Vagrant versions prior to 1.7.0 will use the private key located at ``~/.vagrant.d/insecure_private_key.`` .. seealso:: diff --git a/docsite/rst/index.rst b/docsite/rst/index.rst index 996d324fc9..c8d263d01a 100644 --- a/docsite/rst/index.rst +++ b/docsite/rst/index.rst @@ -16,7 +16,7 @@ We believe simplicity is relevant to all sizes of environments and design for bu Ansible manages machines in an agentless manner. There is never a question of how to upgrade remote daemons or the problem of not being able to manage systems because daemons are uninstalled. As OpenSSH is one of the most peer reviewed open source components, the security exposure of using the tool is greatly reduced. Ansible is decentralized -- it relies on your existing OS credentials to control access to remote machines; if needed it can easily connect with Kerberos, LDAP, and other centralized authentication management systems. -This documentation covers the current released version of Ansible (1.7.2) and also some development version features (1.8). For recent features, in each section, the version of Ansible where the feature is added is indicated. Ansible, Inc releases a new major release of Ansible approximately every 2 months. The core application evolves somewhat conservatively, valuing simplicity in language design and setup, while the community around new modules and plugins being developed and contributed moves very very quickly, typically adding 20 or so new modules in each release. +This documentation covers the current released version of Ansible (1.8.2) and also some development version features (1.9). For recent features, in each section, the version of Ansible where the feature is added is indicated. Ansible, Inc releases a new major release of Ansible approximately every 2 months. The core application evolves somewhat conservatively, valuing simplicity in language design and setup, while the community around new modules and plugins being developed and contributed moves very very quickly, typically adding 20 or so new modules in each release. .. _an_introduction: diff --git a/docsite/rst/intro_configuration.rst b/docsite/rst/intro_configuration.rst index a9f50f804f..a857ec90cd 100644 --- a/docsite/rst/intro_configuration.rst +++ b/docsite/rst/intro_configuration.rst @@ -289,10 +289,7 @@ The valid values are either 'replace' (the default) or 'merge'. hostfile ======== -This is the default location of the inventory file, script, or directory that Ansible will use to determine what hosts it has available -to talk to:: - - hostfile = /etc/ansible/hosts +This is a deprecated setting since 1.9, please look at :ref:`inventory` for the new setting. .. _host_key_checking: @@ -304,6 +301,18 @@ implications and wish to disable it, you may do so here by setting the value to host_key_checking=True +.. _inventory: + +inventory +========= + +This is the default location of the inventory file, script, or directory that Ansible will use to determine what hosts it has available +to talk to:: + + inventory = /etc/ansible/hosts + +It used to be called hostfile in Ansible before 1.9 + .. _jinja2_extensions: jinja2_extensions @@ -496,7 +505,7 @@ sudo_flags ========== Additional flags to pass to sudo when engaging sudo support. The default is '-H' which preserves the environment -of the original user. In some situations you may wish to add or remote flags, but in general most users +of the original user. In some situations you may wish to add or remove flags, but in general most users will not need to change this setting:: sudo_flags=-H diff --git a/docsite/rst/intro_dynamic_inventory.rst b/docsite/rst/intro_dynamic_inventory.rst index e6743c100e..ddb452e775 100644 --- a/docsite/rst/intro_dynamic_inventory.rst +++ b/docsite/rst/intro_dynamic_inventory.rst @@ -225,6 +225,26 @@ If the location given to -i in Ansible is a directory (or as so configured in an at the same time. When doing so, it is possible to mix both dynamic and statically managed inventory sources in the same ansible run. Instant hybrid cloud! +.. _static_groups_of_dynamic: + +Static Groups of Dynamic Groups +``````````````````````````````` + +When defining groups of groups in the static inventory file, the child groups +must also be defined in the static inventory file, or ansible will return an +error. If you want to define a static group of dynamic child groups, define +the dynamic groups as empty in the static inventory file. For example:: + + [tag_Name_staging_foo] + + [tag_Name_staging_bar] + + [staging:children] + tag_Name_staging_foo + tag_Name_staging_bar + + + .. seealso:: :doc:`intro_inventory` diff --git a/docsite/rst/intro_installation.rst b/docsite/rst/intro_installation.rst index 7f41851800..07e0c501c7 100644 --- a/docsite/rst/intro_installation.rst +++ b/docsite/rst/intro_installation.rst @@ -111,6 +111,10 @@ To install from source. $ cd ./ansible $ source ./hacking/env-setup +If you want to suppress spurious warnings/errors, use: + + $ source ./hacking/env-setup -q + If you don't have pip installed in your version of Python, install pip:: $ sudo easy_install pip @@ -202,6 +206,24 @@ You may also wish to run from source to get the latest, which is covered above. .. _from_pkg: +Latest Releases Via Portage (Gentoo) +++++++++++++++++++++++++++++++++++++ + +.. code-block:: bash + + $ emerge -av app-admin/ansible + +To install the newest version, you may need to unmask the ansible package prior to emerging: + +.. code-block:: bash + + $ echo 'app-admin/ansible' >> /etc/portage/package.accept_keywords + +.. note:: + + If you have Python 3 as a default Python slot on your Gentoo nodes (default setting), then you + must set ``ansible_python_interpreter = /usr/bin/python2`` in your group or inventory variables. + Latest Releases Via pkg (FreeBSD) +++++++++++++++++++++++++++++++++ @@ -227,6 +249,18 @@ To install on a Mac, make sure you have Homebrew, then run: $ brew update $ brew install ansible +.. _from_pkgutil: + +Latest Releases Via OpenCSW (Solaris) ++++++++++++++++++++++++++++++++++++++ + +Ansible is available for Solaris as `SysV package from OpenCSW <https://www.opencsw.org/packages/ansible/>`_. + +.. code-block:: bash + + # pkgadd -d http://get.opencsw.org/now + # /opt/csw/bin/pkgutil -i ansible + .. _from_pip: Latest Releases Via Pip diff --git a/docsite/rst/intro_inventory.rst b/docsite/rst/intro_inventory.rst index 5b409e8e65..5c38372e76 100644 --- a/docsite/rst/intro_inventory.rst +++ b/docsite/rst/intro_inventory.rst @@ -19,7 +19,7 @@ pull inventory from dynamic or cloud sources, as described in :doc:`intro_dynami Hosts and Groups ++++++++++++++++ -The format for /etc/ansible/hosts is an INI format and looks like this:: +The format for /etc/ansible/hosts is an INI-like format and looks like this:: mail.example.com @@ -184,7 +184,7 @@ variables. Note that this only works on Ansible 1.4 or later. Tip: In Ansible 1.2 or later the group_vars/ and host_vars/ directories can exist in either the playbook directory OR the inventory directory. If both paths exist, variables in the playbook -directory will be loaded second. +directory will override variables set in the inventory directory. Tip: Keeping your inventory file and variables in a git repo (or other version control) is an excellent way to track changes to your inventory and host variables. @@ -205,6 +205,8 @@ mentioned:: The default ssh user name to use. ansible_ssh_pass The ssh password to use (this is insecure, we strongly recommend using --ask-pass or SSH keys) + ansible_sudo + The boolean to decide if sudo should be used for this host. Defaults to false. ansible_sudo_pass The sudo password to use (this is insecure, we strongly recommend using --ask-sudo-pass) ansible_sudo_exe (new in version 1.8) diff --git a/docsite/rst/playbooks_best_practices.rst b/docsite/rst/playbooks_best_practices.rst index 2eaa8e7736..43c642d583 100644 --- a/docsite/rst/playbooks_best_practices.rst +++ b/docsite/rst/playbooks_best_practices.rst @@ -66,9 +66,7 @@ The top level of the directory would contain files and directories like so:: monitoring/ # "" fooapp/ # "" -.. note: If you find yourself having too many top level playbooks (for instance you have a playbook you wrote for a specific hotfix, etc), it may -make sense to have a playbooks/ directory instead. This can be a good idea as you get larger. If you do this, -configure your roles_path in ansible.cfg to find your roles location. +.. note: If you find yourself having too many top level playbooks (for instance you have a playbook you wrote for a specific hotfix, etc), it may make sense to have a playbooks/ directory instead. This can be a good idea as you get larger. If you do this, configure your roles_path in ansible.cfg to find your roles location. .. _use_dynamic_inventory_with_clouds: @@ -77,7 +75,7 @@ Use Dynamic Inventory With Clouds If you are using a cloud provider, you should not be managing your inventory in a static file. See :doc:`intro_dynamic_inventory`. -This does not just apply to clouds -- If you have another system maintaing a canonical list of systems +This does not just apply to clouds -- If you have another system maintaining a canonical list of systems in your infrastructure, usage of dynamic inventory is a great idea in general. .. _stage_vs_prod: @@ -367,7 +365,7 @@ If group-specific settings are needed, this can also be done. For example:: In the above example, CentOS machines get the value of '42' for asdf, but other machines get '10'. This can be used not only to set variables, but also to apply certain roles to only certain systems. -Alternatively, if only variables are needed: +Alternatively, if only variables are needed:: - hosts: all tasks: diff --git a/docsite/rst/playbooks_conditionals.rst b/docsite/rst/playbooks_conditionals.rst index a00ec916c4..d71a0d3c7a 100644 --- a/docsite/rst/playbooks_conditionals.rst +++ b/docsite/rst/playbooks_conditionals.rst @@ -166,11 +166,11 @@ To use this conditional import feature, you'll need facter or ohai installed pri you can of course push this out with Ansible if you like:: # for facter - ansible -m yum -a "pkg=facter ensure=installed" - ansible -m yum -a "pkg=ruby-json ensure=installed" + ansible -m yum -a "pkg=facter state=present" + ansible -m yum -a "pkg=ruby-json state=present" # for ohai - ansible -m yum -a "pkg=ohai ensure=installed" + ansible -m yum -a "pkg=ohai state=present" Ansible's approach to configuration -- separating variables from tasks, keeps your playbooks from turning into arbitrary code with ugly nested ifs, conditionals, and so on - and results diff --git a/docsite/rst/playbooks_intro.rst b/docsite/rst/playbooks_intro.rst index 4bc3bccf2d..ecf8d46de1 100644 --- a/docsite/rst/playbooks_intro.rst +++ b/docsite/rst/playbooks_intro.rst @@ -151,8 +151,8 @@ Just `Control-C` to kill it and run it again with `-K`. These are deleted immediately after the command is executed. This only occurs when sudoing from a user like 'bob' to 'timmy', not when going from 'bob' to 'root', or logging in directly as 'bob' or - 'root'. If this concerns you that this data is briefly readable - (not writable), avoid transferring uncrypted passwords with + 'root'. If it concerns you that this data is briefly readable + (not writable), avoid transferring unencrypted passwords with `sudo_user` set. In other cases, '/tmp' is not used and this does not come into play. Ansible also takes care to not log password parameters. diff --git a/docsite/rst/playbooks_lookups.rst b/docsite/rst/playbooks_lookups.rst index 429a1a1232..f33e21a3c5 100644 --- a/docsite/rst/playbooks_lookups.rst +++ b/docsite/rst/playbooks_lookups.rst @@ -121,12 +121,17 @@ Here are some examples:: - debug: msg="{{ lookup('redis_kv', 'redis://localhost:6379,somekey') }} is value in Redis for somekey" + # dnstxt lookup requires the Python dnspython package - debug: msg="{{ lookup('dnstxt', 'example.com') }} is a DNS TXT record for example.com" - debug: msg="{{ lookup('template', './some_template.j2') }} is a value from evaluation of this template" - debug: msg="{{ lookup('etcd', 'foo') }} is a value from a locally running etcd" + - debug: msg="{{item}}" + with_url: + - 'http://github.com/gremlin.keys' + As an alternative you can also assign lookup plugins to variables or use them elsewhere. This macros are evaluated each time they are used in a task (or template):: diff --git a/docsite/rst/playbooks_startnstep.rst b/docsite/rst/playbooks_startnstep.rst index ac06962cf2..1067c3e121 100644 --- a/docsite/rst/playbooks_startnstep.rst +++ b/docsite/rst/playbooks_startnstep.rst @@ -4,7 +4,7 @@ Start and Step This shows a few alternative ways to run playbooks. These modes are very useful for testing new plays or debugging. -.. _start_at_task +.. _start_at_task: Start-at-task ````````````` @@ -15,7 +15,7 @@ If you want to start executing your playbook at a particular task, you can do so The above will start executing your playbook at a task named "install packages". -.. _step +.. _step: Step ```` diff --git a/docsite/rst/playbooks_variables.rst b/docsite/rst/playbooks_variables.rst index 3a52261360..2e3d67373e 100644 --- a/docsite/rst/playbooks_variables.rst +++ b/docsite/rst/playbooks_variables.rst @@ -311,6 +311,74 @@ To get a random list from an existing list:: note that when used with a non 'listable' item it is a noop, otherwise it always returns a list + +.. _math_stuff: + +Math +-------------------- +.. versionadded:: 1.9 + + +To see if something is actually a number:: + + {{ myvar | isnan }} + +Get the logarithm (default is e):: + + {{ myvar | log }} + +Get the base 10 logarithm:: + + {{ myvar | log(10) }} + +Give me the power of 2! (or 5):: + + {{ myvar | pow(2) }} + {{ myvar | pow(5) }} + +Square root, or the 5th:: + + {{ myvar | root }} + {{ myvar | root(5) }} + +Note that jinja2 already provides some like abs() and round(). + + +.. _hash_filters: + +Hashing filters +-------------------- +.. versionadded:: 1.9 + +To get the sha1 hash of a string:: + + {{ 'test1'|hash('sha1') }} + +To get the md5 hash of a string:: + + {{ 'test1'|hash('md5') }} + +Get a string checksum:: + + {{ 'test2'|checksum }} + +Other hashes (platform dependant):: + + {{ 'test2'|hash('blowfish') }} + +To get a sha512 password hash (random salt):: + + {{ 'passwordsaresecret'|password_hash('sha512') }} + +To get a sha256 password hash with a specific salt:: + + {{ 'secretpassword'|password_hash('sha256', 'mysecretsalt') }} + + +Hash types available depend on the master system running ansible, +'hash' depends on hashlib password_hash depends on crypt. + + .. _other_useful_filters: Other Useful Filters @@ -341,9 +409,9 @@ To work with Base64 encoded strings:: {{ encoded | b64decode }} {{ decoded | b64encode }} -To take a sha1sum of a filename:: +To create a UUID from a string (new in version 1.9):: - {{ filename | sha1 }} + {{ hostname | to_uuid }} To cast values as certain types, such as when you input a string as "True" from a vars_prompt and the system doesn't know it is a boolean value:: @@ -758,11 +826,14 @@ the fact that they have not been communicated with in the current execution of / To configure fact caching, enable it in ansible.cfg as follows:: [defaults] + gathering = smart fact_caching = redis fact_caching_timeout = 86400 # seconds -At the time of writing, Redis is the only supported fact caching engine. +You might also want to change the 'gathering' setting to 'smart' or 'explicit' or set gather_facts to False in most plays. + +At the time of writing, Redis is the only supported fact caching engine. To get redis up and running, perform the equivalent OS commands:: yum install redis @@ -911,7 +982,7 @@ The contents of each variables file is a simple YAML dictionary, like this:: .. note:: It's also possible to keep per-host and per-group variables in very - similar files, this is covered in :doc:`intro_patterns`. + similar files, this is covered in :ref:`splitting_out_vars`. .. _passing_variables_on_the_command_line: diff --git a/examples/ansible.cfg b/examples/ansible.cfg index a89fa47664..67aa039608 100644 --- a/examples/ansible.cfg +++ b/examples/ansible.cfg @@ -11,8 +11,8 @@ # some basic default values... -hostfile = /etc/ansible/hosts -# library_path = /usr/share/my_modules/ +inventory = /etc/ansible/hosts +#library = /usr/share/my_modules/ remote_tmp = $HOME/.ansible/tmp pattern = * forks = 5 @@ -21,7 +21,7 @@ sudo_user = root #ask_sudo_pass = True #ask_pass = True transport = smart -remote_port = 22 +#remote_port = 22 module_lang = C # plays will gather facts by default, which contain information about @@ -154,6 +154,11 @@ filter_plugins = /usr/share/ansible_plugins/filter_plugins # current IP information. fact_caching = memory + +# retry files +#retry_files_enabled = False +#retry_files_save_path = ~/.ansible-retry + [paramiko_connection] # uncomment this line to cause the paramiko connection plugin to not record new host diff --git a/examples/scripts/ConfigureRemotingForAnsible.ps1 b/examples/scripts/ConfigureRemotingForAnsible.ps1 index 39601d2a76..1b45ce442b 100644 --- a/examples/scripts/ConfigureRemotingForAnsible.ps1 +++ b/examples/scripts/ConfigureRemotingForAnsible.ps1 @@ -98,13 +98,7 @@ ElseIf ((Get-Service "WinRM").Status -ne "Running") If (!(Get-PSSessionConfiguration -Verbose:$false) -or (!(Get-ChildItem WSMan:\localhost\Listener))) { Write-Verbose "Enabling PS Remoting." - Try - { - Enable-PSRemoting -Force -ErrorAction SilentlyContinue - } - Catch - { - } + Enable-PSRemoting -Force -ErrorAction Stop } Else { diff --git a/hacking/env-setup b/hacking/env-setup index 4fed169097..10adad5099 100755..100644 --- a/hacking/env-setup +++ b/hacking/env-setup @@ -1,42 +1,76 @@ -#!/bin/bash -# usage: source ./hacking/env-setup [-q] +# usage: source hacking/env-setup [-q] # modifies environment for running Ansible from checkout +# Default values for shell variables we use +PYTHONPATH=${PYTHONPATH-""} +PATH=${PATH-""} +MANPATH=${MANPATH-""} +verbosity=${1-info} # Defaults to `info' if unspecified + +if [ "$verbosity" = -q ]; then + verbosity=silent +fi + # When run using source as directed, $0 gets set to bash, so we must use $BASH_SOURCE if [ -n "$BASH_SOURCE" ] ; then - HACKING_DIR=`dirname $BASH_SOURCE` -elif [ $(basename $0) = "env-setup" ]; then - HACKING_DIR=`dirname $0` + HACKING_DIR=$(dirname "$BASH_SOURCE") +elif [ $(basename -- "$0") = "env-setup" ]; then + HACKING_DIR=$(dirname "$0") else HACKING_DIR="$PWD/hacking" fi # The below is an alternative to readlink -fn which doesn't exist on OS X # Source: http://stackoverflow.com/a/1678636 -FULL_PATH=`python -c "import os; print(os.path.realpath('$HACKING_DIR'))"` -ANSIBLE_HOME=`dirname "$FULL_PATH"` +FULL_PATH=$(python -c "import os; print(os.path.realpath('$HACKING_DIR'))") +ANSIBLE_HOME=$(dirname "$FULL_PATH") PREFIX_PYTHONPATH="$ANSIBLE_HOME/lib" PREFIX_PATH="$ANSIBLE_HOME/bin" PREFIX_MANPATH="$ANSIBLE_HOME/docs/man" -[[ $PYTHONPATH != ${PREFIX_PYTHONPATH}* ]] && export PYTHONPATH=$PREFIX_PYTHONPATH:$PYTHONPATH -[[ $PATH != ${PREFIX_PATH}* ]] && export PATH=$PREFIX_PATH:$PATH -[[ $MANPATH != ${PREFIX_MANPATH}* ]] && export MANPATH=$PREFIX_MANPATH:$MANPATH - -# Print out values unless -q is set - -if [ $# -eq 0 -o "$1" != "-q" ] ; then - echo "" - echo "Setting up Ansible to run out of checkout..." - echo "" - echo "PATH=$PATH" - echo "PYTHONPATH=$PYTHONPATH" - echo "MANPATH=$MANPATH" - echo "" - - echo "Remember, you may wish to specify your host file with -i" - echo "" - echo "Done!" - echo "" +expr "$PYTHONPATH" : "${PREFIX_PYTHONPATH}.*" > /dev/null || export PYTHONPATH="$PREFIX_PYTHONPATH:$PYTHONPATH" +expr "$PATH" : "${PREFIX_PATH}.*" > /dev/null || export PATH="$PREFIX_PATH:$PATH" +expr "$MANPATH" : "${PREFIX_MANPATH}.*" > /dev/null || export MANPATH="$PREFIX_MANPATH:$MANPATH" + +# +# Generate egg_info so that pkg_resources works +# + +# Do the work in a function so we don't repeat ourselves later +gen_egg_info() +{ + python setup.py egg_info + if [ -e "$PREFIX_PYTHONPATH/ansible.egg-info" ] ; then + rm -r "$PREFIX_PYTHONPATH/ansible.egg-info" + fi + mv "ansible.egg-info" "$PREFIX_PYTHONPATH" +} + +if [ "$ANSIBLE_HOME" != "$PWD" ] ; then + current_dir="$PWD" +else + current_dir="$ANSIBLE_HOME" fi +cd "$ANSIBLE_HOME" +if [ "$verbosity" = silent ] ; then + gen_egg_info > /dev/null 2>&1 +else + gen_egg_info +fi +cd "$current_dir" +if [ "$verbosity" != silent ] ; then + cat <<- EOF + + Setting up Ansible to run out of checkout... + + PATH=$PATH + PYTHONPATH=$PYTHONPATH + MANPATH=$MANPATH + + Remember, you may wish to specify your host file with -i + + Done! + + EOF +fi diff --git a/hacking/env-setup.fish b/hacking/env-setup.fish index 05fb60672d..1b872f4dc0 100644 --- a/hacking/env-setup.fish +++ b/hacking/env-setup.fish @@ -36,6 +36,16 @@ end set -gx ANSIBLE_LIBRARY $ANSIBLE_HOME/library +# Generate egg_info so that pkg_resources works +pushd $ANSIBLE_HOME +python setup.py egg_info +if test -e $PREFIX_PYTHONPATH/ansible*.egg-info + rm -r $PREFIX_PYTHONPATH/ansible*.egg-info +end +mv ansible*egg-info $PREFIX_PYTHONPATH +popd + + if set -q argv switch $argv case '-q' '--quiet' diff --git a/hacking/update.sh b/hacking/update.sh new file mode 100755 index 0000000000..5979dd0ab2 --- /dev/null +++ b/hacking/update.sh @@ -0,0 +1,3 @@ +#!/bin/sh +git pull --rebase +git submodule update --init --recursive diff --git a/lib/ansible/cache/jsonfile.py b/lib/ansible/cache/jsonfile.py index 8b4c892a40..b7d72c8d2e 100644 --- a/lib/ansible/cache/jsonfile.py +++ b/lib/ansible/cache/jsonfile.py @@ -17,14 +17,17 @@ import os import time -import json import errno +try: + import simplejson as json +except ImportError: + import json + from ansible import constants as C from ansible import utils from ansible.cache.base import BaseCacheModule - class CacheModule(BaseCacheModule): """ A caching module backed by json files. @@ -34,6 +37,8 @@ class CacheModule(BaseCacheModule): self._timeout = float(C.CACHE_PLUGIN_TIMEOUT) self._cache = {} self._cache_dir = C.CACHE_PLUGIN_CONNECTION # expects a dir path + if not self._cache_dir: + utils.exit("error, fact_caching_connection is not set, cannot use fact cache") if not os.path.exists(self._cache_dir): try: @@ -68,12 +73,11 @@ class CacheModule(BaseCacheModule): cachefile = "%s/%s" % (self._cache_dir, key) try: - #TODO: check if valid keys can have invalid FS chars, base32? f = open(cachefile, 'w') except (OSError,IOError), e: utils.warning("error while trying to read %s : %s" % (cachefile, str(e))) else: - json.dump(value, f, ensure_ascii=False) + f.write(utils.jsonify(value)) finally: f.close() diff --git a/lib/ansible/cache/redis.py b/lib/ansible/cache/redis.py index c55b74469d..7ae5ef74c1 100644 --- a/lib/ansible/cache/redis.py +++ b/lib/ansible/cache/redis.py @@ -20,9 +20,14 @@ import collections # FIXME: can we store these as something else before we ship it? import sys import time -import json + +try: + import simplejson as json +except ImportError: + import json from ansible import constants as C +from ansible.utils import jsonify from ansible.cache.base import BaseCacheModule try: @@ -65,7 +70,7 @@ class CacheModule(BaseCacheModule): return json.loads(value) def set(self, key, value): - value2 = json.dumps(value) + value2 = jsonify(value) if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire' self._cache.setex(self._make_key(key), int(self._timeout), value2) else: diff --git a/lib/ansible/callbacks.py b/lib/ansible/callbacks.py index a4b62fb005..3c0a4fe52e 100644 --- a/lib/ansible/callbacks.py +++ b/lib/ansible/callbacks.py @@ -27,6 +27,7 @@ import fcntl import constants import locale from ansible.color import stringc +from ansible.module_utils import basic import logging if constants.DEFAULT_LOG_PATH != '': @@ -456,6 +457,12 @@ class PlaybookRunnerCallbacks(DefaultRunnerCallbacks): item = None if type(results) == dict: item = results.get('item', None) + if isinstance(item, unicode): + item = utils.to_bytes(item) + results = basic.json_dict_unicode_to_bytes(results) + else: + results = utils.to_bytes(results) + host = utils.to_bytes(host) if item: msg = "fatal: [%s] => (item=%s) => %s" % (host, item, results) else: @@ -622,7 +629,13 @@ class PlaybookCallbacks(object): if hasattr(self, 'start_at'): # we still have start_at so skip the task self.skip_task = True elif hasattr(self, 'step') and self.step: - msg = ('Perform task: %s (y/n/c): ' % name).encode(sys.stdout.encoding) + if isinstance(name, str): + name = utils.to_unicode(name) + msg = u'Perform task: %s (y/n/c): ' % name + if sys.stdout.encoding: + msg = msg.encode(sys.stdout.encoding, errors='replace') + else: + msg = msg.encode('utf-8') resp = raw_input(msg) if resp.lower() in ['y','yes']: self.skip_task = False @@ -672,7 +685,7 @@ class PlaybookCallbacks(object): result = prompt(msg, private) # if result is false and default is not None - if not result and default: + if not result and default is not None: result = default diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 861dd5325c..b7aca6f36f 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -98,7 +98,7 @@ YAML_FILENAME_EXTENSIONS = [ "", ".yml", ".yaml", ".json" ] DEFAULTS='defaults' # configurable things -DEFAULT_HOST_LIST = shell_expand_path(get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts')) +DEFAULT_HOST_LIST = shell_expand_path(get_config(p, DEFAULTS, 'inventory', 'ANSIBLE_INVENTORY', get_config(p, DEFAULTS,'hostfile','ANSIBLE_HOSTS', '/etc/ansible/hosts'))) DEFAULT_MODULE_PATH = get_config(p, DEFAULTS, 'library', 'ANSIBLE_LIBRARY', None) DEFAULT_ROLES_PATH = shell_expand_path(get_config(p, DEFAULTS, 'roles_path', 'ANSIBLE_ROLES_PATH', '/etc/ansible/roles')) DEFAULT_REMOTE_TMP = get_config(p, DEFAULTS, 'remote_tmp', 'ANSIBLE_REMOTE_TEMP', '$HOME/.ansible/tmp') @@ -161,6 +161,9 @@ DEFAULT_CALLABLE_WHITELIST = get_config(p, DEFAULTS, 'callable_whitelist', ' COMMAND_WARNINGS = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', False, boolean=True) DEFAULT_LOAD_CALLBACK_PLUGINS = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True) +RETRY_FILES_ENABLED = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True) +RETRY_FILES_SAVE_PATH = get_config(p, DEFAULTS, 'retry_files_save_path', 'ANSIBLE_RETRY_FILES_SAVE_PATH', '~/') + # CONNECTION RELATED ANSIBLE_SSH_ARGS = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None) ANSIBLE_SSH_CONTROL_PATH = get_config(p, 'ssh_connection', 'control_path', 'ANSIBLE_SSH_CONTROL_PATH', "%(directory)s/ansible-ssh-%%h-%%p-%%r") diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index 7d279b7b4d..2048046d3c 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -420,7 +420,7 @@ class Inventory(object): group = self.get_group(groupname) if group is None: - raise Exception("group not found: %s" % groupname) + raise errors.AnsibleError("group not found: %s" % groupname) vars = {} @@ -439,7 +439,7 @@ class Inventory(object): host = self.get_host(hostname) if not host: - raise Exception("host not found: %s" % hostname) + raise errors.AnsibleError("host not found: %s" % hostname) return host.get_variables() def get_host_variables(self, hostname, update_cached=False, vault_password=None): diff --git a/lib/ansible/inventory/ini.py b/lib/ansible/inventory/ini.py index 3848696006..2c05253bb3 100644 --- a/lib/ansible/inventory/ini.py +++ b/lib/ansible/inventory/ini.py @@ -36,6 +36,7 @@ class InventoryParser(object): def __init__(self, filename=C.DEFAULT_HOST_LIST): with open(filename) as fh: + self.filename = filename self.lines = fh.readlines() self.groups = {} self.hosts = {} @@ -87,8 +88,8 @@ class InventoryParser(object): self.groups = dict(all=all, ungrouped=ungrouped) active_group_name = 'ungrouped' - for line in self.lines: - line = utils.before_comment(line).strip() + for lineno in range(len(self.lines)): + line = utils.before_comment(self.lines[lineno]).strip() if line.startswith("[") and line.endswith("]"): active_group_name = line.replace("[","").replace("]","") if ":vars" in line or ":children" in line: @@ -142,7 +143,7 @@ class InventoryParser(object): try: (k,v) = t.split("=", 1) except ValueError, e: - raise errors.AnsibleError("Invalid ini entry: %s - %s" % (t, str(e))) + raise errors.AnsibleError("%s:%s: Invalid ini entry: %s - %s" % (self.filename, lineno + 1, t, str(e))) host.set_variable(k, self._parse_value(v)) self.groups[active_group_name].add_host(host) @@ -153,8 +154,8 @@ class InventoryParser(object): def _parse_group_children(self): group = None - for line in self.lines: - line = line.strip() + for lineno in range(len(self.lines)): + line = self.lines[lineno].strip() if line is None or line == '': continue if line.startswith("[") and ":children]" in line: @@ -169,7 +170,7 @@ class InventoryParser(object): elif group: kid_group = self.groups.get(line, None) if kid_group is None: - raise errors.AnsibleError("child group is not defined: (%s)" % line) + raise errors.AnsibleError("%s:%d: child group is not defined: (%s)" % (self.filename, lineno + 1, line)) else: group.add_child_group(kid_group) @@ -180,13 +181,13 @@ class InventoryParser(object): def _parse_group_variables(self): group = None - for line in self.lines: - line = line.strip() + for lineno in range(len(self.lines)): + line = self.lines[lineno].strip() if line.startswith("[") and ":vars]" in line: line = line.replace("[","").replace(":vars]","") group = self.groups.get(line, None) if group is None: - raise errors.AnsibleError("can't add vars to undefined group: %s" % line) + raise errors.AnsibleError("%s:%d: can't add vars to undefined group: %s" % (self.filename, lineno + 1, line)) elif line.startswith("#") or line.startswith(";"): pass elif line.startswith("["): @@ -195,7 +196,7 @@ class InventoryParser(object): pass elif group: if "=" not in line: - raise errors.AnsibleError("variables assigned to group must be in key=value form") + raise errors.AnsibleError("%s:%d: variables assigned to group must be in key=value form" % (self.filename, lineno + 1)) else: (k, v) = [e.strip() for e in line.split("=", 1)] group.set_variable(k, self._parse_value(v)) diff --git a/lib/ansible/inventory/script.py b/lib/ansible/inventory/script.py index 6239be0140..9b8d72de41 100644 --- a/lib/ansible/inventory/script.py +++ b/lib/ansible/inventory/script.py @@ -22,7 +22,7 @@ import subprocess import ansible.constants as C from ansible.inventory.host import Host from ansible.inventory.group import Group -from ansible.module_utils.basic import json_dict_unicode_to_bytes +from ansible.module_utils.basic import json_dict_bytes_to_unicode from ansible import utils from ansible import errors import sys @@ -43,6 +43,10 @@ class InventoryScript(object): except OSError, e: raise errors.AnsibleError("problem running %s (%s)" % (' '.join(cmd), e)) (stdout, stderr) = sp.communicate() + + if sp.returncode != 0: + raise errors.AnsibleError("Inventory script (%s) had an execution error: %s " % (filename,stderr)) + self.data = stdout # see comment about _meta below self.host_vars_from_top = None @@ -55,7 +59,7 @@ class InventoryScript(object): # not passing from_remote because data from CMDB is trusted self.raw = utils.parse_json(self.data) - self.raw = json_dict_unicode_to_bytes(self.raw) + self.raw = json_dict_bytes_to_unicode(self.raw) all = Group('all') groups = dict(all=all) @@ -144,7 +148,7 @@ class InventoryScript(object): if out.strip() == '': return dict() try: - return json_dict_unicode_to_bytes(utils.parse_json(out)) + return json_dict_bytes_to_unicode(utils.parse_json(out)) except ValueError: raise errors.AnsibleError("could not parse post variable response: %s, %s" % (cmd, out)) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index cee6510f34..8603976c5a 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -251,6 +251,24 @@ def json_dict_unicode_to_bytes(d): else: return d +def json_dict_bytes_to_unicode(d): + ''' Recursively convert dict keys and values to byte str + + Specialized for json return because this only handles, lists, tuples, + and dict container types (the containers that the json module returns) + ''' + + if isinstance(d, str): + return unicode(d, 'utf-8') + elif isinstance(d, dict): + return dict(map(json_dict_bytes_to_unicode, d.iteritems())) + elif isinstance(d, list): + return list(map(json_dict_bytes_to_unicode, d)) + elif isinstance(d, tuple): + return tuple(map(json_dict_bytes_to_unicode, d)) + else: + return d + class AnsibleModule(object): @@ -1104,12 +1122,11 @@ class AnsibleModule(object): msg = msg.encode('utf-8') if (has_journal): - journal_args = ["MESSAGE=%s %s" % (module, msg)] - journal_args.append("MODULE=%s" % os.path.basename(__file__)) + journal_args = [("MODULE", os.path.basename(__file__))] for arg in log_args: - journal_args.append(arg.upper() + "=" + str(log_args[arg])) + journal_args.append((arg.upper(), str(log_args[arg]))) try: - journal.sendv(*journal_args) + journal.send("%s %s" % (module, msg), **dict(journal_args)) except IOError, e: # fall back to syslog since logging to journal failed syslog.openlog(str(module), 0, syslog.LOG_USER) diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index 417e1b9521..0f08fead18 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -39,6 +39,7 @@ AWS_REGIONS = [ 'cn-north-1', 'eu-central-1', 'eu-west-1', + 'eu-central-1', 'sa-east-1', 'us-east-1', 'us-west-1', @@ -165,6 +166,11 @@ def boto_fix_security_token_in_profile(conn, profile_name): def connect_to_aws(aws_module, region, **params): conn = aws_module.connect_to_region(region, **params) + if not conn: + if region not in [aws_module_region.name for aws_module_region in aws_module.regions()]: + raise StandardError("Region %s does not seem to be available for aws module %s. If the region definitely exists, you may need to upgrade boto" % (region, aws_module.__name__)) + else: + raise StandardError("Unknown problem connecting to region %s for aws module %s." % (region, aws_module.__name__)) if params.get('profile_name'): conn = boto_fix_security_token_in_profile(conn, params['profile_name']) return conn @@ -180,13 +186,13 @@ def ec2_connect(module): if region: try: ec2 = connect_to_aws(boto.ec2, region, **boto_params) - except boto.exception.NoAuthHandlerFound, e: + except (boto.exception.NoAuthHandlerFound, StandardError), e: module.fail_json(msg=str(e)) # Otherwise, no region so we fallback to the old connection method elif ec2_url: try: ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params) - except boto.exception.NoAuthHandlerFound, e: + except (boto.exception.NoAuthHandlerFound, StandardError), e: module.fail_json(msg=str(e)) else: module.fail_json(msg="Either region or ec2_url must be specified") diff --git a/lib/ansible/module_utils/facts.py b/lib/ansible/module_utils/facts.py index 38082fe854..c2d7b652e1 100644 --- a/lib/ansible/module_utils/facts.py +++ b/lib/ansible/module_utils/facts.py @@ -29,6 +29,7 @@ import socket import struct import datetime import getpass +import pwd import ConfigParser import StringIO @@ -46,7 +47,7 @@ except ImportError: import simplejson as json # -------------------------------------------------------------- -# timeout function to make sure some fact gathering +# timeout function to make sure some fact gathering # steps do not exceed a time limit class TimeoutError(Exception): @@ -82,7 +83,8 @@ class Facts(object): subclass Facts. """ - _I386RE = re.compile(r'i[3456]86') + # i86pc is a Solaris and derivatives-ism + _I386RE = re.compile(r'i([3456]86|86pc)') # For the most part, we assume that platform.dist() will tell the truth. # This is the fallback to handle unknowns or exceptions OSDIST_LIST = ( ('/etc/redhat-release', 'RedHat'), @@ -274,84 +276,115 @@ class Facts(object): self.facts['distribution_release'] = dist[2] or 'NA' # Try to handle the exceptions now ... for (path, name) in Facts.OSDIST_LIST: - if os.path.exists(path) and os.path.getsize(path) > 0: - if self.facts['distribution'] in ('Fedora', ): - # Once we determine the value is one of these distros - # we trust the values are always correct - break - elif name == 'RedHat': - data = get_file_content(path) - if 'Red Hat' in data: - self.facts['distribution'] = name - else: - self.facts['distribution'] = data.split()[0] - break - elif name == 'OtherLinux': - data = get_file_content(path) - if 'Amazon' in data: - self.facts['distribution'] = 'Amazon' - self.facts['distribution_version'] = data.split()[-1] + if os.path.exists(path): + if os.path.getsize(path) > 0: + if self.facts['distribution'] in ('Fedora', ): + # Once we determine the value is one of these distros + # we trust the values are always correct break - elif name == 'OpenWrt': - data = get_file_content(path) - if 'OpenWrt' in data: - self.facts['distribution'] = name - version = re.search('DISTRIB_RELEASE="(.*)"', data) - if version: - self.facts['distribution_version'] = version.groups()[0] - release = re.search('DISTRIB_CODENAME="(.*)"', data) - if release: - self.facts['distribution_release'] = release.groups()[0] + elif name == 'RedHat': + data = get_file_content(path) + if 'Red Hat' in data: + self.facts['distribution'] = name + else: + self.facts['distribution'] = data.split()[0] break - elif name == 'Alpine': - data = get_file_content(path) - self.facts['distribution'] = name - self.facts['distribution_version'] = data - break - elif name == 'Solaris': - data = get_file_content(path).split('\n')[0] - if 'Solaris' in data: - ora_prefix = '' - if 'Oracle Solaris' in data: - data = data.replace('Oracle ','') - ora_prefix = 'Oracle ' - self.facts['distribution'] = data.split()[0] - self.facts['distribution_version'] = data.split()[1] - self.facts['distribution_release'] = ora_prefix + data - break - elif name == 'SuSE': - data = get_file_content(path) - if 'suse' in data.lower(): - if path == '/etc/os-release': - release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) + elif name == 'OtherLinux': + data = get_file_content(path) + if 'Amazon' in data: + self.facts['distribution'] = 'Amazon' + self.facts['distribution_version'] = data.split()[-1] + break + elif name == 'OpenWrt': + data = get_file_content(path) + if 'OpenWrt' in data: + self.facts['distribution'] = name + version = re.search('DISTRIB_RELEASE="(.*)"', data) + if version: + self.facts['distribution_version'] = version.groups()[0] + release = re.search('DISTRIB_CODENAME="(.*)"', data) if release: self.facts['distribution_release'] = release.groups()[0] - break - elif path == '/etc/SuSE-release': - data = data.splitlines() - for line in data: - release = re.search('CODENAME *= *([^\n]+)', line) - if release: - self.facts['distribution_release'] = release.groups()[0].strip() - break - elif name == 'Debian': - data = get_file_content(path) - if 'Debian' in data: - release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) - if release: - self.facts['distribution_release'] = release.groups()[0] - break - elif name == 'Mandriva': - data = get_file_content(path) - if 'Mandriva' in data: - version = re.search('DISTRIB_RELEASE="(.*)"', data) - if version: - self.facts['distribution_version'] = version.groups()[0] - release = re.search('DISTRIB_CODENAME="(.*)"', data) - if release: - self.facts['distribution_release'] = release.groups()[0] + break + elif name == 'Alpine': + data = get_file_content(path) self.facts['distribution'] = name + self.facts['distribution_version'] = data break + elif name == 'Solaris': + data = get_file_content(path).split('\n')[0] + if 'Solaris' in data: + ora_prefix = '' + if 'Oracle Solaris' in data: + data = data.replace('Oracle ','') + ora_prefix = 'Oracle ' + self.facts['distribution'] = data.split()[0] + self.facts['distribution_version'] = data.split()[1] + self.facts['distribution_release'] = ora_prefix + data + break + + uname_rc, uname_out, uname_err = module.run_command(['uname', '-v']) + distribution_version = None + if 'SmartOS' in data: + self.facts['distribution'] = 'SmartOS' + if os.path.exists('/etc/product'): + product_data = dict([l.split(': ', 1) for l in get_file_content('/etc/product').split('\n') if ': ' in l]) + if 'Image' in product_data: + distribution_version = product_data.get('Image').split()[-1] + elif 'OpenIndiana' in data: + self.facts['distribution'] = 'OpenIndiana' + elif 'OmniOS' in data: + self.facts['distribution'] = 'OmniOS' + distribution_version = data.split()[-1] + elif uname_rc == 0 and 'NexentaOS_' in uname_out: + self.facts['distribution'] = 'Nexenta' + distribution_version = data.split()[-1].lstrip('v') + + if self.facts['distribution'] in ('SmartOS', 'OpenIndiana', 'OmniOS', 'Nexenta'): + self.facts['distribution_release'] = data.strip() + if distribution_version is not None: + self.facts['distribution_version'] = distribution_version + elif uname_rc == 0: + self.facts['distribution_version'] = uname_out.split('\n')[0].strip() + break + + elif name == 'SuSE': + data = get_file_content(path) + if 'suse' in data.lower(): + if path == '/etc/os-release': + release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) + distdata = get_file_content(path).split('\n')[0] + self.facts['distribution'] = distdata.split('=')[1] + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif path == '/etc/SuSE-release': + data = data.splitlines() + distdata = get_file_content(path).split('\n')[0] + self.facts['distribution'] = distdata.split()[0] + for line in data: + release = re.search('CODENAME *= *([^\n]+)', line) + if release: + self.facts['distribution_release'] = release.groups()[0].strip() + break + elif name == 'Debian': + data = get_file_content(path) + if 'Debian' in data: + release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif name == 'Mandriva': + data = get_file_content(path) + if 'Mandriva' in data: + version = re.search('DISTRIB_RELEASE="(.*)"', data) + if version: + self.facts['distribution_version'] = version.groups()[0] + release = re.search('DISTRIB_CODENAME="(.*)"', data) + if release: + self.facts['distribution_release'] = release.groups()[0] + self.facts['distribution'] = name + break else: self.facts['distribution'] = name @@ -519,6 +552,12 @@ class Facts(object): # User def get_user_facts(self): self.facts['user_id'] = getpass.getuser() + pwent = pwd.getpwnam(getpass.getuser()) + self.facts['user_uid'] = pwent.pw_uid + self.facts['user_gid'] = pwent.pw_gid + self.facts['user_gecos'] = pwent.pw_gecos + self.facts['user_dir'] = pwent.pw_dir + self.facts['user_shell'] = pwent.pw_shell def get_env_facts(self): self.facts['env'] = {} @@ -570,7 +609,11 @@ class LinuxHardware(Hardware): """ platform = 'Linux' - MEMORY_FACTS = ['MemTotal', 'SwapTotal', 'MemFree', 'SwapFree'] + + # Originally only had these four as toplevelfacts + ORIGINAL_MEMORY_FACTS = frozenset(('MemTotal', 'SwapTotal', 'MemFree', 'SwapFree')) + # Now we have all of these in a dict structure + MEMORY_FACTS = ORIGINAL_MEMORY_FACTS.union(('Buffers', 'Cached', 'SwapCached')) def __init__(self): Hardware.__init__(self) @@ -589,31 +632,91 @@ class LinuxHardware(Hardware): def get_memory_facts(self): if not os.access("/proc/meminfo", os.R_OK): return + + memstats = {} for line in open("/proc/meminfo").readlines(): data = line.split(":", 1) key = data[0] - if key in LinuxHardware.MEMORY_FACTS: + if key in self.ORIGINAL_MEMORY_FACTS: val = data[1].strip().split(' ')[0] self.facts["%s_mb" % key.lower()] = long(val) / 1024 + if key in self.MEMORY_FACTS: + val = data[1].strip().split(' ')[0] + memstats[key.lower()] = long(val) / 1024 + + if None not in (memstats.get('memtotal'), memstats.get('memfree')): + memstats['real:used'] = memstats['memtotal'] - memstats['memfree'] + if None not in (memstats.get('cached'), memstats.get('memfree'), memstats.get('buffers')): + memstats['nocache:free'] = memstats['cached'] + memstats['memfree'] + memstats['buffers'] + if None not in (memstats.get('memtotal'), memstats.get('nocache:free')): + memstats['nocache:used'] = memstats['memtotal'] - memstats['nocache:free'] + if None not in (memstats.get('swaptotal'), memstats.get('swapfree')): + memstats['swap:used'] = memstats['swaptotal'] - memstats['swapfree'] + + self.facts['memory_mb'] = { + 'real' : { + 'total': memstats.get('memtotal'), + 'used': memstats.get('real:used'), + 'free': memstats.get('memfree'), + }, + 'nocache' : { + 'free': memstats.get('nocache:free'), + 'used': memstats.get('nocache:used'), + }, + 'swap' : { + 'total': memstats.get('swaptotal'), + 'free': memstats.get('swapfree'), + 'used': memstats.get('swap:used'), + 'cached': memstats.get('swapcached'), + }, + } + def get_cpu_facts(self): i = 0 + vendor_id_occurrence = 0 + model_name_occurrence = 0 physid = 0 coreid = 0 sockets = {} cores = {} + + xen = False + xen_paravirt = False + try: + if os.path.exists('/proc/xen'): + xen = True + elif open('/sys/hypervisor/type').readline().strip() == 'xen': + xen = True + except IOError: + pass + if not os.access("/proc/cpuinfo", os.R_OK): return self.facts['processor'] = [] for line in open("/proc/cpuinfo").readlines(): data = line.split(":", 1) key = data[0].strip() + + if xen: + if key == 'flags': + # Check for vme cpu flag, Xen paravirt does not expose this. + # Need to detect Xen paravirt because it exposes cpuinfo + # differently than Xen HVM or KVM and causes reporting of + # only a single cpu core. + if 'vme' not in data: + xen_paravirt = True + # model name is for Intel arch, Processor (mind the uppercase P) # works for some ARM devices, like the Sheevaplug. if key == 'model name' or key == 'Processor' or key == 'vendor_id': if 'processor' not in self.facts: self.facts['processor'] = [] self.facts['processor'].append(data[1].strip()) + if key == 'vendor_id': + vendor_id_occurrence += 1 + if key == 'model name': + model_name_occurrence += 1 i += 1 elif key == 'physical id': physid = data[1].strip() @@ -629,13 +732,23 @@ class LinuxHardware(Hardware): cores[coreid] = int(data[1].strip()) elif key == '# processors': self.facts['processor_cores'] = int(data[1].strip()) + + if vendor_id_occurrence == model_name_occurrence: + i = vendor_id_occurrence + if self.facts['architecture'] != 's390x': - self.facts['processor_count'] = sockets and len(sockets) or i - self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1 - self.facts['processor_threads_per_core'] = ((cores.values() and - cores.values()[0] or 1) / self.facts['processor_cores']) - self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] * - self.facts['processor_count'] * self.facts['processor_cores']) + if xen_paravirt: + self.facts['processor_count'] = i + self.facts['processor_cores'] = i + self.facts['processor_threads_per_core'] = 1 + self.facts['processor_vcpus'] = i + else: + self.facts['processor_count'] = sockets and len(sockets) or i + self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1 + self.facts['processor_threads_per_core'] = ((cores.values() and + cores.values()[0] or 1) / self.facts['processor_cores']) + self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] * + self.facts['processor_count'] * self.facts['processor_cores']) def get_dmi_facts(self): ''' learn dmi facts from system @@ -726,6 +839,13 @@ class LinuxHardware(Hardware): size_available = statvfs_result.f_bsize * (statvfs_result.f_bavail) except OSError, e: continue + lsblkPath = module.get_bin_path("lsblk") + rc, out, err = module.run_command("%s -ln --output UUID %s" % (lsblkPath, fields[0]), use_unsafe_shell=True) + + if rc == 0: + uuid = out.strip() + else: + uuid = 'NA' self.facts['mounts'].append( {'mount': fields[1], @@ -735,6 +855,7 @@ class LinuxHardware(Hardware): # statvfs data 'size_total': size_total, 'size_available': size_available, + 'uuid': uuid, }) def get_device_facts(self): diff --git a/lib/ansible/module_utils/powershell.ps1 b/lib/ansible/module_utils/powershell.ps1 index c097c69768..ee7d3ddeca 100644 --- a/lib/ansible/module_utils/powershell.ps1 +++ b/lib/ansible/module_utils/powershell.ps1 @@ -142,3 +142,25 @@ Function ConvertTo-Bool return } +# Helper function to calculate a hash of a file in a way which powershell 3 +# and above can handle: +Function Get-FileChecksum($path) +{ + $hash = "" + If (Test-Path -PathType Leaf $path) + { + $sp = new-object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider; + $fp = [System.IO.File]::Open($path, [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read); + [System.BitConverter]::ToString($sp.ComputeHash($fp)).Replace("-", "").ToLower(); + $fp.Dispose(); + } + ElseIf (Test-Path -PathType Container $path) + { + $hash= "3"; + } + Else + { + $hash = "1"; + } + return $hash +} diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index c2d87c27bc..962b868ee0 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -252,9 +252,33 @@ class SSLValidationHandler(urllib2.BaseHandler): except: self.module.fail_json(msg='Connection to proxy failed') + def detect_no_proxy(self, url): + ''' + Detect if the 'no_proxy' environment variable is set and honor those locations. + ''' + env_no_proxy = os.environ.get('no_proxy') + if env_no_proxy: + env_no_proxy = env_no_proxy.split(',') + netloc = urlparse.urlparse(url).netloc + + for host in env_no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # Our requested URL matches something in no_proxy, so don't + # use the proxy for this + return False + return True + def http_request(self, req): tmp_ca_cert_path, paths_checked = self.get_ca_certs() https_proxy = os.environ.get('https_proxy') + + # Detect if 'no_proxy' environment variable is set and if our URL is included + use_proxy = self.detect_no_proxy(req.get_full_url()) + + if not use_proxy: + # ignore proxy settings for this host request + return req + try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if https_proxy: diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core -Subproject 467ad65f735ddb33b6302cf0968074c22d15356 +Subproject 1394920cd3e440f5806463d0c1cfbe4a4b94f42 diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras -Subproject b8071a8d5eebe405250774a0b7c6c74451bc953 +Subproject ed35fc69bf3cf280cdc3d272d2aec419e47a07b diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py index d3c0aa5300..7f4fd8cb1f 100644 --- a/lib/ansible/playbook/__init__.py +++ b/lib/ansible/playbook/__init__.py @@ -643,19 +643,28 @@ class PlayBook(object): buf = StringIO.StringIO() for x in replay_hosts: buf.write("%s\n" % x) - basedir = self.inventory.basedir() + basedir = C.shell_expand_path(C.RETRY_FILES_SAVE_PATH) filename = "%s.retry" % os.path.basename(self.filename) filename = filename.replace(".yml","") - filename = os.path.join(os.path.expandvars('$HOME/'), filename) + filename = os.path.join(basedir, filename) try: + if not os.path.exists(basedir): + os.makedirs(basedir) + fd = open(filename, 'w') fd.write(buf.getvalue()) fd.close() - return filename except: - pass - return None + ansible.callbacks.display( + "\nERROR: could not create retry file. Check the value of \n" + + "the configuration variable 'retry_files_save_path' or set \n" + + "'retry_files_enabled' to False to avoid this message.\n", + color='red' + ) + return None + + return filename # ***************************************************** diff --git a/lib/ansible/playbook/play.py b/lib/ansible/playbook/play.py index 6e7cc0fc94..b551baf6b3 100644 --- a/lib/ansible/playbook/play.py +++ b/lib/ansible/playbook/play.py @@ -38,7 +38,7 @@ class Play(object): 'accelerate_port', 'accelerate_ipv6', 'sudo', 'sudo_user', 'transport', 'playbook', 'tags', 'gather_facts', 'serial', '_ds', '_handlers', '_tasks', 'basedir', 'any_errors_fatal', 'roles', 'max_fail_pct', '_play_hosts', 'su', 'su_user', - 'vault_password', 'no_log', + 'vault_password', 'no_log', 'environment', ] # to catch typos and so forth -- these are userland names @@ -48,7 +48,7 @@ class Play(object): 'tasks', 'handlers', 'remote_user', 'user', 'port', 'include', 'accelerate', 'accelerate_port', 'accelerate_ipv6', 'sudo', 'sudo_user', 'connection', 'tags', 'gather_facts', 'serial', 'any_errors_fatal', 'roles', 'role_names', 'pre_tasks', 'post_tasks', 'max_fail_percentage', - 'su', 'su_user', 'vault_password', 'no_log', + 'su', 'su_user', 'vault_password', 'no_log', 'environment', ] # ************************************************* @@ -71,6 +71,7 @@ class Play(object): self.roles = ds.get('roles', None) self.tags = ds.get('tags', None) self.vault_password = vault_password + self.environment = ds.get('environment', {}) if self.tags is None: self.tags = [] diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index 783f488fa1..bdffba5527 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -133,7 +133,7 @@ class Task(object): self.register = ds.get('register', None) self.sudo = utils.boolean(ds.get('sudo', play.sudo)) self.su = utils.boolean(ds.get('su', play.su)) - self.environment = ds.get('environment', {}) + self.environment = ds.get('environment', play.environment) self.role_name = role_name self.no_log = utils.boolean(ds.get('no_log', "false")) or self.play.no_log self.run_once = utils.boolean(ds.get('run_once', 'false')) diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index 7912d23462..052ae642e2 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -394,20 +394,20 @@ class Runner(object): actual_user = inject.get('ansible_ssh_user', self.remote_user) thisuser = None - if host in inject['hostvars']: - if inject['hostvars'][host].get('ansible_ssh_user'): - # user for delegate host in inventory - thisuser = inject['hostvars'][host].get('ansible_ssh_user') - else: - # look up the variables for the host directly from inventory - try: - host_vars = self.inventory.get_variables(host, vault_password=self.vault_pass) - if 'ansible_ssh_user' in host_vars: - thisuser = host_vars['ansible_ssh_user'] - except Exception, e: - # the hostname was not found in the inventory, so - # we just ignore this and try the next method - pass + try: + if host in inject['hostvars']: + if inject['hostvars'][host].get('ansible_ssh_user'): + # user for delegate host in inventory + thisuser = inject['hostvars'][host].get('ansible_ssh_user') + else: + # look up the variables for the host directly from inventory + host_vars = self.inventory.get_variables(host, vault_password=self.vault_pass) + if 'ansible_ssh_user' in host_vars: + thisuser = host_vars['ansible_ssh_user'] + except errors.AnsibleError, e: + # the hostname was not found in the inventory, so + # we just ignore this and try the next method + pass if thisuser is None and self.remote_user: # user defined by play/runner @@ -817,6 +817,10 @@ class Runner(object): port, complex_args=complex_args ) + + if 'stdout' in result.result and 'stdout_lines' not in result.result: + result.result['stdout_lines'] = result.result['stdout'].splitlines() + results.append(result.result) if result.comm_ok == False: all_comm_ok = False @@ -1461,9 +1465,15 @@ class Runner(object): # Expose the current hostgroup to the bypassing plugins self.host_set = hosts # We aren't iterating over all the hosts in this - # group. So, just pick the first host in our group to + # group. So, just choose the "delegate_to" host if that is defined and is + # one of the targeted hosts, otherwise pick the first host in our group to # construct the conn object with. - result_data = self._executor(hosts[0], None).result + if self.delegate_to is not None and self.delegate_to in hosts: + host = self.delegate_to + else: + host = hosts[0] + + result_data = self._executor(host, None).result # Create a ResultData item for each host in this group # using the returned result. If we didn't do this we would # get false reports of dark hosts. diff --git a/lib/ansible/runner/action_plugins/debug.py b/lib/ansible/runner/action_plugins/debug.py index 75613b9919..eaf1364c3f 100644 --- a/lib/ansible/runner/action_plugins/debug.py +++ b/lib/ansible/runner/action_plugins/debug.py @@ -52,7 +52,7 @@ class ActionModule(object): result = dict(msg=args['msg']) elif 'var' in args and not utils.LOOKUP_REGEX.search(args['var']): results = template.template(self.basedir, args['var'], inject, convert_bare=True) - result[args['var']] = results + result['var'] = { args['var']: results } # force flag to make debug output module always verbose result['verbose_always'] = True diff --git a/lib/ansible/runner/action_plugins/unarchive.py b/lib/ansible/runner/action_plugins/unarchive.py index cfcaf454bd..572cd03d5e 100644 --- a/lib/ansible/runner/action_plugins/unarchive.py +++ b/lib/ansible/runner/action_plugins/unarchive.py @@ -59,13 +59,14 @@ class ActionModule(object): # do not run the command if the line contains creates=filename # and the filename already exists. This allows idempotence # of command executions. - module_args_tmp = "path=%s" % creates + module_args_tmp = "" + complex_args_tmp = dict(path=creates, get_md5=False, get_checksum=False) module_return = self.runner._execute_module(conn, tmp, 'stat', module_args_tmp, inject=inject, - complex_args=complex_args, persist_files=True) + complex_args=complex_args_tmp, persist_files=True) stat = module_return.result.get('stat', None) if stat and stat.get('exists', False): return ReturnData( - conn=conn, + conn=conn, comm_ok=True, result=dict( skipped=True, diff --git a/lib/ansible/runner/action_plugins/win_copy.py b/lib/ansible/runner/action_plugins/win_copy.py new file mode 100644 index 0000000000..28362195c9 --- /dev/null +++ b/lib/ansible/runner/action_plugins/win_copy.py @@ -0,0 +1,377 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os + +from ansible import utils +import ansible.constants as C +import ansible.utils.template as template +from ansible import errors +from ansible.runner.return_data import ReturnData +import base64 +import json +import stat +import tempfile +import pipes + +## fixes https://github.com/ansible/ansible/issues/3518 +# http://mypy.pythonblogs.com/12_mypy/archive/1253_workaround_for_python_bug_ascii_codec_cant_encode_character_uxa0_in_position_111_ordinal_not_in_range128.html +import sys +reload(sys) +sys.setdefaultencoding("utf8") + + +class ActionModule(object): + + def __init__(self, runner): + self.runner = runner + + def run(self, conn, tmp_path, module_name, module_args, inject, complex_args=None, **kwargs): + ''' handler for file transfer operations ''' + + # load up options + options = {} + if complex_args: + options.update(complex_args) + options.update(utils.parse_kv(module_args)) + source = options.get('src', None) + content = options.get('content', None) + dest = options.get('dest', None) + raw = utils.boolean(options.get('raw', 'no')) + force = utils.boolean(options.get('force', 'yes')) + + # content with newlines is going to be escaped to safely load in yaml + # now we need to unescape it so that the newlines are evaluated properly + # when writing the file to disk + if content: + if isinstance(content, unicode): + try: + content = content.decode('unicode-escape') + except UnicodeDecodeError: + pass + + if (source is None and content is None and not 'first_available_file' in inject) or dest is None: + result=dict(failed=True, msg="src (or content) and dest are required") + return ReturnData(conn=conn, result=result) + elif (source is not None or 'first_available_file' in inject) and content is not None: + result=dict(failed=True, msg="src and content are mutually exclusive") + return ReturnData(conn=conn, result=result) + + # Check if the source ends with a "/" + source_trailing_slash = False + if source: + source_trailing_slash = source.endswith("/") + + # Define content_tempfile in case we set it after finding content populated. + content_tempfile = None + + # If content is defined make a temp file and write the content into it. + if content is not None: + try: + # If content comes to us as a dict it should be decoded json. + # We need to encode it back into a string to write it out. + if type(content) is dict: + content_tempfile = self._create_content_tempfile(json.dumps(content)) + else: + content_tempfile = self._create_content_tempfile(content) + source = content_tempfile + except Exception, err: + result = dict(failed=True, msg="could not write content temp file: %s" % err) + return ReturnData(conn=conn, result=result) + # if we have first_available_file in our vars + # look up the files and use the first one we find as src + elif 'first_available_file' in inject: + found = False + for fn in inject.get('first_available_file'): + fn_orig = fn + fnt = template.template(self.runner.basedir, fn, inject) + fnd = utils.path_dwim(self.runner.basedir, fnt) + if not os.path.exists(fnd) and '_original_file' in inject: + fnd = utils.path_dwim_relative(inject['_original_file'], 'files', fnt, self.runner.basedir, check=False) + if os.path.exists(fnd): + source = fnd + found = True + break + if not found: + results = dict(failed=True, msg="could not find src in first_available_file list") + return ReturnData(conn=conn, result=results) + else: + source = template.template(self.runner.basedir, source, inject) + if '_original_file' in inject: + source = utils.path_dwim_relative(inject['_original_file'], 'files', source, self.runner.basedir) + else: + source = utils.path_dwim(self.runner.basedir, source) + + # A list of source file tuples (full_path, relative_path) which will try to copy to the destination + source_files = [] + + # If source is a directory populate our list else source is a file and translate it to a tuple. + if os.path.isdir(source): + # Get the amount of spaces to remove to get the relative path. + if source_trailing_slash: + sz = len(source) + 1 + else: + sz = len(source.rsplit('/', 1)[0]) + 1 + + # Walk the directory and append the file tuples to source_files. + for base_path, sub_folders, files in os.walk(source): + for file in files: + full_path = os.path.join(base_path, file) + rel_path = full_path[sz:] + source_files.append((full_path, rel_path)) + + # If it's recursive copy, destination is always a dir, + # explicitly mark it so (note - copy module relies on this). + if not conn.shell.path_has_trailing_slash(dest): + dest = conn.shell.join_path(dest, '') + else: + source_files.append((source, os.path.basename(source))) + + changed = False + diffs = [] + module_result = {"changed": False} + + # A register for if we executed a module. + # Used to cut down on command calls when not recursive. + module_executed = False + + # Tell _execute_module to delete the file if there is one file. + delete_remote_tmp = (len(source_files) == 1) + + # If this is a recursive action create a tmp_path that we can share as the _exec_module create is too late. + if not delete_remote_tmp: + if "-tmp-" not in tmp_path: + tmp_path = self.runner._make_tmp_path(conn) + + # expand any user home dir specifier + dest = self.runner._remote_expand_user(conn, dest, tmp_path) + + for source_full, source_rel in source_files: + # Generate a hash of the local file. + local_checksum = utils.checksum(source_full) + + # If local_checksum is not defined we can't find the file so we should fail out. + if local_checksum is None: + result = dict(failed=True, msg="could not find src=%s" % source_full) + return ReturnData(conn=conn, result=result) + + # This is kind of optimization - if user told us destination is + # dir, do path manipulation right away, otherwise we still check + # for dest being a dir via remote call below. + if conn.shell.path_has_trailing_slash(dest): + dest_file = conn.shell.join_path(dest, source_rel) + else: + dest_file = conn.shell.join_path(dest) + + # Attempt to get the remote checksum + remote_checksum = self.runner._remote_checksum(conn, tmp_path, dest_file, inject) + + if remote_checksum == '3': + # The remote_checksum was executed on a directory. + if content is not None: + # If source was defined as content remove the temporary file and fail out. + self._remove_tempfile_if_content_defined(content, content_tempfile) + result = dict(failed=True, msg="can not use content with a dir as dest") + return ReturnData(conn=conn, result=result) + else: + # Append the relative source location to the destination and retry remote_checksum. + dest_file = conn.shell.join_path(dest, source_rel) + remote_checksum = self.runner._remote_checksum(conn, tmp_path, dest_file, inject) + + if remote_checksum != '1' and not force: + # remote_file does not exist so continue to next iteration. + continue + + if local_checksum != remote_checksum: + # The checksums don't match and we will change or error out. + changed = True + + # Create a tmp_path if missing only if this is not recursive. + # If this is recursive we already have a tmp_path. + if delete_remote_tmp: + if "-tmp-" not in tmp_path: + tmp_path = self.runner._make_tmp_path(conn) + + if self.runner.diff and not raw: + diff = self._get_diff_data(conn, tmp_path, inject, dest_file, source_full) + else: + diff = {} + + if self.runner.noop_on_check(inject): + self._remove_tempfile_if_content_defined(content, content_tempfile) + diffs.append(diff) + changed = True + module_result = dict(changed=True) + continue + + # Define a remote directory that we will copy the file to. + tmp_src = tmp_path + 'source' + + if not raw: + conn.put_file(source_full, tmp_src) + else: + conn.put_file(source_full, dest_file) + + # We have copied the file remotely and no longer require our content_tempfile + self._remove_tempfile_if_content_defined(content, content_tempfile) + + # fix file permissions when the copy is done as a different user + if (self.runner.sudo and self.runner.sudo_user != 'root' or self.runner.su and self.runner.su_user != 'root') and not raw: + self.runner._remote_chmod(conn, 'a+r', tmp_src, tmp_path) + + if raw: + # Continue to next iteration if raw is defined. + continue + + # Run the copy module + + # src and dest here come after original and override them + # we pass dest only to make sure it includes trailing slash in case of recursive copy + new_module_args = dict( + src=tmp_src, + dest=dest, + original_basename=source_rel + ) + if self.runner.noop_on_check(inject): + new_module_args['CHECKMODE'] = True + if self.runner.no_log: + new_module_args['NO_LOG'] = True + + module_args_tmp = utils.merge_module_args(module_args, new_module_args) + + module_return = self.runner._execute_module(conn, tmp_path, 'win_copy', module_args_tmp, inject=inject, complex_args=complex_args, delete_remote_tmp=delete_remote_tmp) + module_executed = True + + else: + # no need to transfer the file, already correct md5, but still need to call + # the file module in case we want to change attributes + self._remove_tempfile_if_content_defined(content, content_tempfile) + + if raw: + # Continue to next iteration if raw is defined. + # self.runner._remove_tmp_path(conn, tmp_path) + continue + + tmp_src = tmp_path + source_rel + + # Build temporary module_args. + new_module_args = dict( + src=tmp_src, + dest=dest, + original_basename=source_rel + ) + if self.runner.noop_on_check(inject): + new_module_args['CHECKMODE'] = True + if self.runner.no_log: + new_module_args['NO_LOG'] = True + + module_args_tmp = utils.merge_module_args(module_args, new_module_args) + + # Execute the file module. + module_return = self.runner._execute_module(conn, tmp_path, 'win_file', module_args_tmp, inject=inject, complex_args=complex_args, delete_remote_tmp=delete_remote_tmp) + module_executed = True + + module_result = module_return.result + if not module_result.get('checksum'): + module_result['checksum'] = local_checksum + if module_result.get('failed') == True: + return module_return + if module_result.get('changed') == True: + changed = True + + # Delete tmp_path if we were recursive or if we did not execute a module. + if (not C.DEFAULT_KEEP_REMOTE_FILES and not delete_remote_tmp) \ + or (not C.DEFAULT_KEEP_REMOTE_FILES and delete_remote_tmp and not module_executed): + self.runner._remove_tmp_path(conn, tmp_path) + + # the file module returns the file path as 'path', but + # the copy module uses 'dest', so add it if it's not there + if 'path' in module_result and 'dest' not in module_result: + module_result['dest'] = module_result['path'] + + # TODO: Support detailed status/diff for multiple files + if len(source_files) == 1: + result = module_result + else: + result = dict(dest=dest, src=source, changed=changed) + if len(diffs) == 1: + return ReturnData(conn=conn, result=result, diff=diffs[0]) + else: + return ReturnData(conn=conn, result=result) + + def _create_content_tempfile(self, content): + ''' Create a tempfile containing defined content ''' + fd, content_tempfile = tempfile.mkstemp() + f = os.fdopen(fd, 'w') + try: + f.write(content) + except Exception, err: + os.remove(content_tempfile) + raise Exception(err) + finally: + f.close() + return content_tempfile + + def _get_diff_data(self, conn, tmp, inject, destination, source): + peek_result = self.runner._execute_module(conn, tmp, 'win_file', "path=%s diff_peek=1" % destination, inject=inject, persist_files=True) + + if not peek_result.is_successful(): + return {} + + diff = {} + if peek_result.result['state'] == 'absent': + diff['before'] = '' + elif peek_result.result['appears_binary']: + diff['dst_binary'] = 1 + elif peek_result.result['size'] > utils.MAX_FILE_SIZE_FOR_DIFF: + diff['dst_larger'] = utils.MAX_FILE_SIZE_FOR_DIFF + else: + dest_result = self.runner._execute_module(conn, tmp, 'slurp', "path=%s" % destination, inject=inject, persist_files=True) + if 'content' in dest_result.result: + dest_contents = dest_result.result['content'] + if dest_result.result['encoding'] == 'base64': + dest_contents = base64.b64decode(dest_contents) + else: + raise Exception("unknown encoding, failed: %s" % dest_result.result) + diff['before_header'] = destination + diff['before'] = dest_contents + + src = open(source) + src_contents = src.read(8192) + st = os.stat(source) + if "\x00" in src_contents: + diff['src_binary'] = 1 + elif st[stat.ST_SIZE] > utils.MAX_FILE_SIZE_FOR_DIFF: + diff['src_larger'] = utils.MAX_FILE_SIZE_FOR_DIFF + else: + src.seek(0) + diff['after_header'] = source + diff['after'] = src.read() + + return diff + + def _remove_tempfile_if_content_defined(self, content, content_tempfile): + if content is not None: + os.remove(content_tempfile) + + + def _result_key_merge(self, options, results): + # add keys to file module results to mimic copy + if 'path' in results.result and 'dest' not in results.result: + results.result['dest'] = results.result['path'] + del results.result['path'] + return results diff --git a/lib/ansible/runner/action_plugins/win_template.py b/lib/ansible/runner/action_plugins/win_template.py new file mode 100644 index 0000000000..e32a5806c4 --- /dev/null +++ b/lib/ansible/runner/action_plugins/win_template.py @@ -0,0 +1,146 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os +import pipes +from ansible.utils import template +from ansible import utils +from ansible import errors +from ansible.runner.return_data import ReturnData +import base64 + +class ActionModule(object): + + TRANSFERS_FILES = True + + def __init__(self, runner): + self.runner = runner + + def run(self, conn, tmp, module_name, module_args, inject, complex_args=None, **kwargs): + ''' handler for template operations ''' + + if not self.runner.is_playbook: + raise errors.AnsibleError("in current versions of ansible, templates are only usable in playbooks") + + # load up options + options = {} + if complex_args: + options.update(complex_args) + options.update(utils.parse_kv(module_args)) + + source = options.get('src', None) + dest = options.get('dest', None) + + if (source is None and 'first_available_file' not in inject) or dest is None: + result = dict(failed=True, msg="src and dest are required") + return ReturnData(conn=conn, comm_ok=False, result=result) + + # if we have first_available_file in our vars + # look up the files and use the first one we find as src + + if 'first_available_file' in inject: + found = False + for fn in self.runner.module_vars.get('first_available_file'): + fn_orig = fn + fnt = template.template(self.runner.basedir, fn, inject) + fnd = utils.path_dwim(self.runner.basedir, fnt) + if not os.path.exists(fnd) and '_original_file' in inject: + fnd = utils.path_dwim_relative(inject['_original_file'], 'templates', fnt, self.runner.basedir, check=False) + if os.path.exists(fnd): + source = fnd + found = True + break + if not found: + result = dict(failed=True, msg="could not find src in first_available_file list") + return ReturnData(conn=conn, comm_ok=False, result=result) + else: + source = template.template(self.runner.basedir, source, inject) + + if '_original_file' in inject: + source = utils.path_dwim_relative(inject['_original_file'], 'templates', source, self.runner.basedir) + else: + source = utils.path_dwim(self.runner.basedir, source) + + if conn.shell.path_has_trailing_slash(dest): + base = os.path.basename(source) + dest = conn.shell.join_path(dest, base) + + # template the source data locally & get ready to transfer + try: + resultant = template.template_from_file(self.runner.basedir, source, inject, vault_password=self.runner.vault_pass) + except Exception, e: + result = dict(failed=True, msg=type(e).__name__ + ": " + str(e)) + return ReturnData(conn=conn, comm_ok=False, result=result) + + local_checksum = utils.checksum_s(resultant) + remote_checksum = self.runner._remote_checksum(conn, tmp, dest, inject) + + if local_checksum != remote_checksum: + + # template is different from the remote value + + # if showing diffs, we need to get the remote value + dest_contents = '' + + if self.runner.diff: + # using persist_files to keep the temp directory around to avoid needing to grab another + dest_result = self.runner._execute_module(conn, tmp, 'slurp', "path=%s" % dest, inject=inject, persist_files=True) + if 'content' in dest_result.result: + dest_contents = dest_result.result['content'] + if dest_result.result['encoding'] == 'base64': + dest_contents = base64.b64decode(dest_contents) + else: + raise Exception("unknown encoding, failed: %s" % dest_result.result) + + xfered = self.runner._transfer_str(conn, tmp, 'source', resultant) + + # fix file permissions when the copy is done as a different user + if self.runner.sudo and self.runner.sudo_user != 'root' or self.runner.su and self.runner.su_user != 'root': + self.runner._remote_chmod(conn, 'a+r', xfered, tmp) + + # run the copy module + new_module_args = dict( + src=xfered, + dest=dest, + original_basename=os.path.basename(source), + follow=True, + ) + module_args_tmp = utils.merge_module_args(module_args, new_module_args) + + if self.runner.noop_on_check(inject): + return ReturnData(conn=conn, comm_ok=True, result=dict(changed=True), diff=dict(before_header=dest, after_header=source, before=dest_contents, after=resultant)) + else: + res = self.runner._execute_module(conn, tmp, 'win_copy', module_args_tmp, inject=inject, complex_args=complex_args) + if res.result.get('changed', False): + res.diff = dict(before=dest_contents, after=resultant) + return res + else: + # when running the file module based on the template data, we do + # not want the source filename (the name of the template) to be used, + # since this would mess up links, so we clear the src param and tell + # the module to follow links + new_module_args = dict( + src=None, + follow=True, + ) + # be sure to inject the check mode param into the module args and + # rely on the file module to report its changed status + if self.runner.noop_on_check(inject): + new_module_args['CHECKMODE'] = True + module_args = utils.merge_module_args(module_args, new_module_args) + return self.runner._execute_module(conn, tmp, 'win_file', module_args, inject=inject, complex_args=complex_args) + diff --git a/lib/ansible/runner/connection_plugins/ssh.py b/lib/ansible/runner/connection_plugins/ssh.py index c2fd9666eb..104f60fd35 100644 --- a/lib/ansible/runner/connection_plugins/ssh.py +++ b/lib/ansible/runner/connection_plugins/ssh.py @@ -230,6 +230,7 @@ class Connection(object): host_fh.close() for line in data.split("\n"): + line = line.strip() if line is None or " " not in line: continue tokens = line.split() diff --git a/lib/ansible/runner/filter_plugins/core.py b/lib/ansible/runner/filter_plugins/core.py index 966d3876d8..22ce73970c 100644 --- a/lib/ansible/runner/filter_plugins/core.py +++ b/lib/ansible/runner/filter_plugins/core.py @@ -15,21 +15,33 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. +import sys import base64 import json import os.path -import yaml import types import pipes import glob import re import collections +import crypt +import hashlib +import string +from functools import partial import operator as py_operator -from ansible import errors -from ansible.utils import md5s, checksum_s -from distutils.version import LooseVersion, StrictVersion from random import SystemRandom, shuffle +import uuid + +import yaml from jinja2.filters import environmentfilter +from distutils.version import LooseVersion, StrictVersion + +from ansible import errors +from ansible.utils.hashing import md5s, checksum_s +from ansible.utils.unicode import unicode_wrap + + +UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E') def to_nice_yaml(*a, **kw): @@ -42,6 +54,22 @@ def to_json(a, *args, **kw): def to_nice_json(a, *args, **kw): '''Make verbose, human readable JSON''' + # python-2.6's json encoder is buggy (can't encode hostvars) + if sys.version_info < (2, 7): + try: + import simplejson + except ImportError: + pass + else: + try: + major = int(simplejson.__version__.split('.')[0]) + except: + pass + else: + if major >= 2: + return simplejson.dumps(a, indent=4, sort_keys=True, *args, **kw) + # Fallback to the to_json filter + return to_json(a, *args, **kw) return json.dumps(a, indent=4, sort_keys=True, *args, **kw) def failed(*a, **kw): @@ -243,14 +271,52 @@ def randomize_list(mylist): pass return mylist +def get_hash(data, hashtype='sha1'): + + try: # see if hash is supported + h = hashlib.new(hashtype) + except: + return None + + h.update(data) + return h.hexdigest() + +def get_encrypted_password(password, hashtype='sha512', salt=None): + + # TODO: find a way to construct dynamically from system + cryptmethod= { + 'md5': '1', + 'blowfish': '2a', + 'sha256': '5', + 'sha512': '6', + } + + hastype = hashtype.lower() + if hashtype in cryptmethod: + if salt is None: + r = SystemRandom() + salt = ''.join([r.choice(string.ascii_letters + string.digits) for _ in range(16)]) + + saltstring = "$%s$%s" % (cryptmethod[hashtype],salt) + encrypted = crypt.crypt(password,saltstring) + return encrypted + + return None + +def to_uuid(string): + return str(uuid.uuid5(UUID_NAMESPACE_ANSIBLE, str(string))) + class FilterModule(object): ''' Ansible core jinja2 filters ''' def filters(self): return { # base 64 - 'b64decode': base64.b64decode, - 'b64encode': base64.b64encode, + 'b64decode': partial(unicode_wrap, base64.b64decode), + 'b64encode': partial(unicode_wrap, base64.b64encode), + + # uuid + 'to_uuid': to_uuid, # json 'to_json': to_json, @@ -263,11 +329,11 @@ class FilterModule(object): 'from_yaml': yaml.safe_load, # path - 'basename': os.path.basename, - 'dirname': os.path.dirname, - 'expanduser': os.path.expanduser, - 'realpath': os.path.realpath, - 'relpath': os.path.relpath, + 'basename': partial(unicode_wrap, os.path.basename), + 'dirname': partial(unicode_wrap, os.path.dirname), + 'expanduser': partial(unicode_wrap, os.path.expanduser), + 'realpath': partial(unicode_wrap, os.path.realpath), + 'relpath': partial(unicode_wrap, os.path.relpath), # failure testing 'failed' : failed, @@ -295,6 +361,9 @@ class FilterModule(object): 'sha1': checksum_s, # checksum of string as used by ansible for checksuming files 'checksum': checksum_s, + # generic hashing + 'password_hash': get_encrypted_password, + 'hash': get_hash, # file glob 'fileglob': fileglob, diff --git a/lib/ansible/runner/filter_plugins/ipaddr.py b/lib/ansible/runner/filter_plugins/ipaddr.py new file mode 100644 index 0000000000..bcb19b16fd --- /dev/null +++ b/lib/ansible/runner/filter_plugins/ipaddr.py @@ -0,0 +1,626 @@ +# (c) 2014, Maciej Delmanowski <drybjed@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from functools import partial + +try: + import netaddr +except ImportError: + # in this case, we'll make the filters return error messages (see bottom) + netaddr = None +else: + class mac_linux(netaddr.mac_unix): + pass + mac_linux.word_fmt = '%.2x' + +from ansible import errors + + +# ---- IP address and network query helpers ---- + +def _empty_ipaddr_query(v, vtype): + # We don't have any query to process, so just check what type the user + # expects, and return the IP address in a correct format + if v: + if vtype == 'address': + return str(v.ip) + elif vtype == 'network': + return str(v) + +def _6to4_query(v, vtype, value): + if v.version == 4: + + if v.size == 1: + ipconv = str(v.ip) + elif v.size > 1: + if v.ip != v.network: + ipconv = str(v.ip) + else: + ipconv = False + + if ipaddr(ipconv, 'public'): + numbers = list(map(int, ipconv.split('.'))) + + try: + return '2002:{:02x}{:02x}:{:02x}{:02x}::1/48'.format(*numbers) + except: + return False + + elif v.version == 6: + if vtype == 'address': + if ipaddr(str(v), '2002::/16'): + return value + elif vtype == 'network': + if v.ip != v.network: + if ipaddr(str(v.ip), '2002::/16'): + return value + else: + return False + +def _ip_query(v): + if v.size == 1: + return str(v.ip) + if v.size > 1: + if v.ip != v.network: + return str(v.ip) + +def _gateway_query(v): + if v.size > 1: + if v.ip != v.network: + return str(v.ip) + '/' + str(v.prefixlen) + +def _bool_ipaddr_query(v): + if v: + return True + +def _broadcast_query(v): + if v.size > 1: + return str(v.broadcast) + +def _cidr_query(v): + return str(v) + +def _cidr_lookup_query(v, iplist, value): + try: + if v in iplist: + return value + except: + return False + +def _host_query(v): + if v.size == 1: + return str(v) + elif v.size > 1: + if v.ip != v.network: + return str(v.ip) + '/' + str(v.prefixlen) + +def _hostmask_query(v): + return str(v.hostmask) + +def _int_query(v, vtype): + if vtype == 'address': + return int(v.ip) + elif vtype == 'network': + return str(int(v.ip)) + '/' + str(int(v.prefixlen)) + +def _ipv4_query(v, value): + if v.version == 6: + try: + return str(v.ipv4()) + except: + return False + else: + return value + +def _ipv6_query(v, value): + if v.version == 4: + return str(v.ipv6()) + else: + return value + +def _link_local_query(v, value): + v_ip = netaddr.IPAddress(str(v.ip)) + if v.version == 4: + if ipaddr(str(v_ip), '169.254.0.0/24'): + return value + + elif v.version == 6: + if ipaddr(str(v_ip), 'fe80::/10'): + return value + +def _loopback_query(v, value): + v_ip = netaddr.IPAddress(str(v.ip)) + if v_ip.is_loopback(): + return value + +def _multicast_query(v, value): + if v.is_multicast(): + return value + +def _net_query(v): + if v.size > 1: + if v.ip == v.network: + return str(v.network) + '/' + str(v.prefixlen) + +def _netmask_query(v): + if v.size > 1: + return str(v.netmask) + +def _network_query(v): + if v.size > 1: + return str(v.network) + +def _prefix_query(v): + return int(v.prefixlen) + +def _private_query(v, value): + if v.is_private(): + return value + +def _public_query(v, value): + v_ip = netaddr.IPAddress(str(v.ip)) + if v_ip.is_unicast() and not v_ip.is_private() and \ + not v_ip.is_loopback() and not v_ip.is_netmask() and \ + not v_ip.is_hostmask(): + return value + +def _revdns_query(v): + v_ip = netaddr.IPAddress(str(v.ip)) + return v_ip.reverse_dns + +def _size_query(v): + return v.size + +def _subnet_query(v): + return str(v.cidr) + +def _type_query(v): + if v.size == 1: + return 'address' + if v.size > 1: + if v.ip != v.network: + return 'address' + else: + return 'network' + +def _unicast_query(v, value): + if v.is_unicast(): + return value + +def _version_query(v): + return v.version + +def _wrap_query(v, vtype, value): + if v.version == 6: + if vtype == 'address': + return '[' + str(v.ip) + ']' + elif vtype == 'network': + return '[' + str(v.ip) + ']/' + str(v.prefixlen) + else: + return value + + +# ---- HWaddr query helpers ---- +def _bare_query(v): + v.dialect = netaddr.mac_bare + return str(v) + +def _bool_hwaddr_query(v): + if v: + return True + +def _cisco_query(v): + v.dialect = netaddr.mac_cisco + return str(v) + +def _empty_hwaddr_query(v, value): + if v: + return value + +def _linux_query(v): + v.dialect = mac_linux + return str(v) + +def _postgresql_query(v): + v.dialect = netaddr.mac_pgsql + return str(v) + +def _unix_query(v): + v.dialect = netaddr.mac_unix + return str(v) + +def _win_query(v): + v.dialect = netaddr.mac_eui48 + return str(v) + + +# ---- IP address and network filters ---- + +def ipaddr(value, query = '', version = False, alias = 'ipaddr'): + ''' Check if string is an IP address or network and filter it ''' + + query_func_extra_args = { + '': ('vtype',), + '6to4': ('vtype', 'value'), + 'cidr_lookup': ('iplist', 'value'), + 'int': ('vtype',), + 'ipv4': ('value',), + 'ipv6': ('value',), + 'link-local': ('value',), + 'loopback': ('value',), + 'lo': ('value',), + 'multicast': ('value',), + 'private': ('value',), + 'public': ('value',), + 'unicast': ('value',), + 'wrap': ('vtype', 'value'), + } + query_func_map = { + '': _empty_ipaddr_query, + '6to4': _6to4_query, + 'address': _ip_query, + 'address/prefix': _gateway_query, + 'bool': _bool_ipaddr_query, + 'broadcast': _broadcast_query, + 'cidr': _cidr_query, + 'cidr_lookup': _cidr_lookup_query, + 'gateway': _gateway_query, + 'gw': _gateway_query, + 'host': _host_query, + 'host/prefix': _gateway_query, + 'hostmask': _hostmask_query, + 'hostnet': _gateway_query, + 'int': _int_query, + 'ip': _ip_query, + 'ipv4': _ipv4_query, + 'ipv6': _ipv6_query, + 'link-local': _link_local_query, + 'lo': _loopback_query, + 'loopback': _loopback_query, + 'multicast': _multicast_query, + 'net': _net_query, + 'netmask': _netmask_query, + 'network': _network_query, + 'prefix': _prefix_query, + 'private': _private_query, + 'public': _public_query, + 'revdns': _revdns_query, + 'router': _gateway_query, + 'size': _size_query, + 'subnet': _subnet_query, + 'type': _type_query, + 'unicast': _unicast_query, + 'v4': _ipv4_query, + 'v6': _ipv6_query, + 'version': _version_query, + 'wrap': _wrap_query, + } + + vtype = None + + if not value: + return False + + elif value == True: + return False + + # Check if value is a list and parse each element + elif isinstance(value, (list, tuple)): + + _ret = [] + for element in value: + if ipaddr(element, str(query), version): + _ret.append(ipaddr(element, str(query), version)) + + if _ret: + return _ret + else: + return list() + + # Check if value is a number and convert it to an IP address + elif str(value).isdigit(): + + # We don't know what IP version to assume, so let's check IPv4 first, + # then IPv6 + try: + if ((not version) or (version and version == 4)): + v = netaddr.IPNetwork('0.0.0.0/0') + v.value = int(value) + v.prefixlen = 32 + elif version and version == 6: + v = netaddr.IPNetwork('::/0') + v.value = int(value) + v.prefixlen = 128 + + # IPv4 didn't work the first time, so it definitely has to be IPv6 + except: + try: + v = netaddr.IPNetwork('::/0') + v.value = int(value) + v.prefixlen = 128 + + # The value is too big for IPv6. Are you a nanobot? + except: + return False + + # We got an IP address, let's mark it as such + value = str(v) + vtype = 'address' + + # value has not been recognized, check if it's a valid IP string + else: + try: + v = netaddr.IPNetwork(value) + + # value is a valid IP string, check if user specified + # CIDR prefix or just an IP address, this will indicate default + # output format + try: + address, prefix = value.split('/') + vtype = 'network' + except: + vtype = 'address' + + # value hasn't been recognized, maybe it's a numerical CIDR? + except: + try: + address, prefix = value.split('/') + address.isdigit() + address = int(address) + prefix.isdigit() + prefix = int(prefix) + + # It's not numerical CIDR, give up + except: + return False + + # It is something, so let's try and build a CIDR from the parts + try: + v = netaddr.IPNetwork('0.0.0.0/0') + v.value = address + v.prefixlen = prefix + + # It's not a valid IPv4 CIDR + except: + try: + v = netaddr.IPNetwork('::/0') + v.value = address + v.prefixlen = prefix + + # It's not a valid IPv6 CIDR. Give up. + except: + return False + + # We have a valid CIDR, so let's write it in correct format + value = str(v) + vtype = 'network' + + # We have a query string but it's not in the known query types. Check if + # that string is a valid subnet, if so, we can check later if given IP + # address/network is inside that specific subnet + try: + ### ?? 6to4 and link-local were True here before. Should they still? + if query and (query not in query_func_map or query == 'cidr_lookup') and ipaddr(query, 'network'): + iplist = netaddr.IPSet([netaddr.IPNetwork(query)]) + query = 'cidr_lookup' + except: + pass + + # This code checks if value maches the IP version the user wants, ie. if + # it's any version ("ipaddr()"), IPv4 ("ipv4()") or IPv6 ("ipv6()") + # If version does not match, return False + if version and v.version != version: + return False + + extras = [] + for arg in query_func_extra_args.get(query, tuple()): + extras.append(locals()[arg]) + try: + return query_func_map[query](v, *extras) + except KeyError: + try: + float(query) + if v.size == 1: + if vtype == 'address': + return str(v.ip) + elif vtype == 'network': + return str(v) + + elif v.size > 1: + try: + return str(v[query]) + '/' + str(v.prefixlen) + except: + return False + + else: + return value + + except: + raise errors.AnsibleFilterError(alias + ': unknown filter type: %s' % query) + + return False + + +def ipwrap(value, query = ''): + try: + if isinstance(value, (list, tuple)): + _ret = [] + for element in value: + if ipaddr(element, query, version = False, alias = 'ipwrap'): + _ret.append(ipaddr(element, 'wrap')) + else: + _ret.append(element) + + return _ret + else: + _ret = ipaddr(value, query, version = False, alias = 'ipwrap') + if _ret: + return ipaddr(_ret, 'wrap') + else: + return value + + except: + return value + + +def ipv4(value, query = ''): + return ipaddr(value, query, version = 4, alias = 'ipv4') + + +def ipv6(value, query = ''): + return ipaddr(value, query, version = 6, alias = 'ipv6') + + +# Split given subnet into smaller subnets or find out the biggest subnet of +# a given IP address with given CIDR prefix +# Usage: +# +# - address or address/prefix | ipsubnet +# returns CIDR subnet of a given input +# +# - address/prefix | ipsubnet(cidr) +# returns number of possible subnets for given CIDR prefix +# +# - address/prefix | ipsubnet(cidr, index) +# returns new subnet with given CIDR prefix +# +# - address | ipsubnet(cidr) +# returns biggest subnet with given CIDR prefix that address belongs to +# +# - address | ipsubnet(cidr, index) +# returns next indexed subnet which contains given address +def ipsubnet(value, query = '', index = 'x'): + ''' Manipulate IPv4/IPv6 subnets ''' + + try: + vtype = ipaddr(value, 'type') + if vtype == 'address': + v = ipaddr(value, 'cidr') + elif vtype == 'network': + v = ipaddr(value, 'subnet') + + value = netaddr.IPNetwork(v) + except: + return False + + if not query: + return str(value) + + elif str(query).isdigit(): + vsize = ipaddr(v, 'size') + query = int(query) + + try: + float(index) + index = int(index) + + if vsize > 1: + try: + return str(list(value.subnet(query))[index]) + except: + return False + + elif vsize == 1: + try: + return str(value.supernet(query)[index]) + except: + return False + + except: + if vsize > 1: + try: + return str(len(list(value.subnet(query)))) + except: + return False + + elif vsize == 1: + try: + return str(value.supernet(query)[0]) + except: + return False + + return False + + +# ---- HWaddr / MAC address filters ---- + +def hwaddr(value, query = '', alias = 'hwaddr'): + ''' Check if string is a HW/MAC address and filter it ''' + + query_func_extra_args = { + '': ('value',), + } + query_func_map = { + '': _empty_hwaddr_query, + 'bare': _bare_query, + 'bool': _bool_hwaddr_query, + 'cisco': _cisco_query, + 'eui48': _win_query, + 'linux': _linux_query, + 'pgsql': _postgresql_query, + 'postgresql': _postgresql_query, + 'psql': _postgresql_query, + 'unix': _unix_query, + 'win': _win_query, + } + + try: + v = netaddr.EUI(value) + except: + if query and query != 'bool': + raise errors.AnsibleFilterError(alias + ': not a hardware address: %s' % value) + + extras = [] + for arg in query_func_extra_args.get(query, tuple()): + extras.append(locals()[arg]) + try: + return query_func_map[query](v, *extras) + except KeyError: + raise errors.AnsibleFilterError(alias + ': unknown filter type: %s' % query) + + return False + +def macaddr(value, query = ''): + return hwaddr(value, query, alias = 'macaddr') + +def _need_netaddr(f_name, *args, **kwargs): + raise errors.AnsibleFilterError('The {0} filter requires python-netaddr be' + ' installed on the ansible controller'.format(f_name)) + +# ---- Ansible filters ---- + +class FilterModule(object): + ''' IP address and network manipulation filters ''' + filter_map = { + # IP addresses and networks + 'ipaddr': ipaddr, + 'ipwrap': ipwrap, + 'ipv4': ipv4, + 'ipv6': ipv6, + 'ipsubnet': ipsubnet, + + # MAC / HW addresses + 'hwaddr': hwaddr, + 'macaddr': macaddr + } + + def filters(self): + if netaddr: + return self.filter_map + else: + # Need to install python-netaddr for these filters to work + return dict((f, partial(_need_netaddr, f)) for f in self.filter_map) diff --git a/lib/ansible/runner/filter_plugins/math.py b/lib/ansible/runner/filter_plugins/math.py new file mode 100644 index 0000000000..7f6cc19555 --- /dev/null +++ b/lib/ansible/runner/filter_plugins/math.py @@ -0,0 +1,69 @@ +# (c) 2014, Brian Coca <bcoca@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import absolute_import + +import math +from ansible import errors + +def isnotanumber(x): + try: + return math.isnan(x) + except TypeError: + return False + + +def logarithm(x, base=math.e): + try: + if base == 10: + return math.log10(x) + else: + return math.log(x, base) + except TypeError, e: + raise errors.AnsibleFilterError('log() can only be used on numbers: %s' % str(e)) + + +def power(x, y): + try: + return math.pow(x, y) + except TypeError, e: + raise errors.AnsibleFilterError('pow() can only be used on numbers: %s' % str(e)) + + +def inversepower(x, base=2): + try: + if base == 2: + return math.sqrt(x) + else: + return math.pow(x, 1.0/float(base)) + except TypeError, e: + raise errors.AnsibleFilterError('root() can only be used on numbers: %s' % str(e)) + + +class FilterModule(object): + ''' Ansible math jinja2 filters ''' + + def filters(self): + return { + # general math + 'isnan': isnotanumber, + + # exponents and logarithms + 'log': logarithm, + 'pow': power, + 'root': inversepower, + } diff --git a/lib/ansible/runner/lookup_plugins/url.py b/lib/ansible/runner/lookup_plugins/url.py new file mode 100644 index 0000000000..37a1df6c7a --- /dev/null +++ b/lib/ansible/runner/lookup_plugins/url.py @@ -0,0 +1,48 @@ +# (c) 2015, Brian Coca <bcoca@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible import utils +import urllib2 + +class LookupModule(object): + + def __init__(self, basedir=None, **kwargs): + self.basedir = basedir + + def run(self, terms, inject=None, **kwargs): + + terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + + if isinstance(terms, basestring): + terms = [ terms ] + + ret = [] + for term in terms: + try: + r = urllib2.Request(term) + response = urllib2.urlopen(r) + except URLError, e: + utils.warnings("Failed lookup url for %s : %s" % (term, str(e))) + continue + except HTTPError, e: + utils.warnings("Recieved HTTP error for %s : %s" % (term, str(e))) + continue + + for line in response.read().splitlines(): + ret.append(line) + + return ret diff --git a/lib/ansible/runner/shell_plugins/sh.py b/lib/ansible/runner/shell_plugins/sh.py index 95d48e9e7d..5462429743 100644 --- a/lib/ansible/runner/shell_plugins/sh.py +++ b/lib/ansible/runner/shell_plugins/sh.py @@ -79,7 +79,6 @@ class ShellModule(object): return 'echo %s' % user_home_path def checksum(self, path, python_interp): - path = pipes.quote(path) # The following test needs to be SH-compliant. BASH-isms will # not work if /bin/sh points to a non-BASH shell. # @@ -97,14 +96,14 @@ class ShellModule(object): # 0. This logic is added to the end of the cmd at the bottom of this # function. - test = "rc=flag; [ -r \"%(p)s\" ] || rc=2; [ -f \"%(p)s\" ] || rc=1; [ -d \"%(p)s\" ] && rc=3; %(i)s -V 2>/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] && echo \"${rc} %(p)s\" && exit 0" % dict(p=path, i=python_interp) + test = "rc=flag; [ -r \'%(p)s\' ] || rc=2; [ -f \'%(p)s\' ] || rc=1; [ -d \'%(p)s\' ] && rc=3; %(i)s -V 2>/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] && echo \"${rc}\"\' %(p)s\' && exit 0" % dict(p=path, i=python_interp) csums = [ - "(%s -c 'import hashlib; print(hashlib.sha1(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path), # Python > 2.4 (including python3) - "(%s -c 'import sha; print(sha.sha(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path), # Python == 2.4 + "(%s -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();\nafile = open(\"%s\", \"rb\")\nbuf = afile.read(BLOCKSIZE)\nwhile len(buf) > 0:\n\thasher.update(buf)\n\tbuf = afile.read(BLOCKSIZE)\nafile.close()\nprint(hasher.hexdigest())' 2>/dev/null)" % (python_interp, path), # Python > 2.4 (including python3) + "(%s -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha1();\nafile = open(\"%s\", \"rb\")\nbuf = afile.read(BLOCKSIZE)\nwhile len(buf) > 0:\n\thasher.update(buf)\n\tbuf = afile.read(BLOCKSIZE)\nafile.close()\nprint(hasher.hexdigest())' 2>/dev/null)" % (python_interp, path), # Python == 2.4 ] cmd = " || ".join(csums) - cmd = "%s; %s || (echo \"0 %s\")" % (test, cmd, path) + cmd = "%s; %s || (echo \'0 %s\')" % (test, cmd, path) return cmd def build_module_command(self, env_string, shebang, cmd, rm_tmp=None): diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index a735e9c0b0..7c4d82914b 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -29,6 +29,7 @@ from ansible import __version__ from ansible.utils.display_functions import * from ansible.utils.plugins import * from ansible.utils.su_prompts import * +from ansible.utils.hashing import secure_hash, secure_hash_s, checksum, checksum_s, md5, md5s from ansible.callbacks import display from ansible.module_utils.splitter import split_args, unquote import ansible.constants as C @@ -45,7 +46,6 @@ import warnings import traceback import getpass import sys -import json import subprocess import contextlib @@ -63,26 +63,10 @@ CODE_REGEX = re.compile(r'(?:{%|%})') try: - import json -except ImportError: + # simplejson can be much faster if it's available import simplejson as json - -# Note, sha1 is the only hash algorithm compatible with python2.4 and with -# FIPS-140 mode (as of 11-2014) -try: - from hashlib import sha1 as sha1 except ImportError: - from sha import sha as sha1 - -# Backwards compat only -try: - from hashlib import md5 as _md5 -except ImportError: - try: - from md5 import md5 as _md5 - except ImportError: - # Assume we're running in FIPS mode here - _md5 = None + import json PASSLIB_AVAILABLE = False try: @@ -832,56 +816,6 @@ def merge_hash(a, b): return result -def secure_hash_s(data, hash_func=sha1): - ''' Return a secure hash hex digest of data. ''' - - digest = hash_func() - try: - digest.update(data) - except UnicodeEncodeError: - digest.update(data.encode('utf-8')) - return digest.hexdigest() - -def secure_hash(filename, hash_func=sha1): - ''' Return a secure hash hex digest of local file, None if file is not present or a directory. ''' - - if not os.path.exists(filename) or os.path.isdir(filename): - return None - digest = hash_func() - blocksize = 64 * 1024 - try: - infile = open(filename, 'rb') - block = infile.read(blocksize) - while block: - digest.update(block) - block = infile.read(blocksize) - infile.close() - except IOError, e: - raise errors.AnsibleError("error while accessing the file %s, error was: %s" % (filename, e)) - return digest.hexdigest() - -# The checksum algorithm must match with the algorithm in ShellModule.checksum() method -checksum = secure_hash -checksum_s = secure_hash_s - -# Backwards compat. Some modules include md5s in their return values -# Continue to support that for now. As of ansible-1.8, all of those modules -# should also return "checksum" (sha1 for now) -# Do not use m5 unless it is needed for: -# 1) Optional backwards compatibility -# 2) Compliance with a third party protocol -# -# MD5 will not work on systems which are FIPS-140-2 compliant. -def md5s(data): - if not _md5: - raise ValueError('MD5 not available. Possibly running in FIPS mode') - return secure_hash_s(data, _md5) - -def md5(filename): - if not _md5: - raise ValueError('MD5 not available. Possibly running in FIPS mode') - return secure_hash(filename, _md5) - def default(value, function): ''' syntactic sugar around lazy evaluation of defaults ''' if value is None: diff --git a/lib/ansible/utils/hashing.py b/lib/ansible/utils/hashing.py new file mode 100644 index 0000000000..a7d142e5bd --- /dev/null +++ b/lib/ansible/utils/hashing.py @@ -0,0 +1,91 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +# Note, sha1 is the only hash algorithm compatible with python2.4 and with +# FIPS-140 mode (as of 11-2014) +try: + from hashlib import sha1 as sha1 +except ImportError: + from sha import sha as sha1 + +# Backwards compat only +try: + from hashlib import md5 as _md5 +except ImportError: + try: + from md5 import md5 as _md5 + except ImportError: + # Assume we're running in FIPS mode here + _md5 = None + +def secure_hash_s(data, hash_func=sha1): + ''' Return a secure hash hex digest of data. ''' + + digest = hash_func() + try: + digest.update(data) + except UnicodeEncodeError: + digest.update(data.encode('utf-8')) + return digest.hexdigest() + +def secure_hash(filename, hash_func=sha1): + ''' Return a secure hash hex digest of local file, None if file is not present or a directory. ''' + + if not os.path.exists(filename) or os.path.isdir(filename): + return None + digest = hash_func() + blocksize = 64 * 1024 + try: + infile = open(filename, 'rb') + block = infile.read(blocksize) + while block: + digest.update(block) + block = infile.read(blocksize) + infile.close() + except IOError, e: + raise errors.AnsibleError("error while accessing the file %s, error was: %s" % (filename, e)) + return digest.hexdigest() + +# The checksum algorithm must match with the algorithm in ShellModule.checksum() method +checksum = secure_hash +checksum_s = secure_hash_s + +# Backwards compat functions. Some modules include md5s in their return values +# Continue to support that for now. As of ansible-1.8, all of those modules +# should also return "checksum" (sha1 for now) +# Do not use md5 unless it is needed for: +# 1) Optional backwards compatibility +# 2) Compliance with a third party protocol +# +# MD5 will not work on systems which are FIPS-140-2 compliant. + +def md5s(data): + if not _md5: + raise ValueError('MD5 not available. Possibly running in FIPS mode') + return secure_hash_s(data, _md5) + +def md5(filename): + if not _md5: + raise ValueError('MD5 not available. Possibly running in FIPS mode') + return secure_hash(filename, _md5) + diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index 3e7f5e4d81..9556b8fcea 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -33,6 +33,7 @@ import ast import traceback from ansible.utils.string_functions import count_newlines_from_end +from ansible.utils import to_bytes class Globals(object): @@ -93,6 +94,8 @@ def lookup(name, *args, **kwargs): ran = instance.run(*args, inject=tvars, **kwargs) except errors.AnsibleError: raise + except jinja2.exceptions.UndefinedError, e: + raise errors.AnsibleUndefinedVariable("One or more undefined variables: %s" % str(e)) except Exception, e: raise errors.AnsibleError('Unexpected error in during lookup: %s' % e) if ran: @@ -270,7 +273,7 @@ def template_from_file(basedir, path, vars, vault_password=None): managed_str = managed_default.format( host = vars['template_host'], uid = vars['template_uid'], - file = vars['template_path'] + file = to_bytes(vars['template_path']) ) vars['ansible_managed'] = time.strftime( managed_str, diff --git a/lib/ansible/utils/unicode.py b/lib/ansible/utils/unicode.py new file mode 100644 index 0000000000..b2fcf65161 --- /dev/null +++ b/lib/ansible/utils/unicode.py @@ -0,0 +1,248 @@ +# (c) 2012-2014, Toshio Kuraotmi <a.badger@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# to_bytes and to_unicode were written by Toshio Kuratomi for the +# python-kitchen library https://pypi.python.org/pypi/kitchen +# They are licensed in kitchen under the terms of the GPLv2+ +# They were copied and modified for use in ansible by Toshio in Jan 2015 +# (simply removing the deprecated features) + +#: Aliases for the utf-8 codec +_UTF8_ALIASES = frozenset(('utf-8', 'UTF-8', 'utf8', 'UTF8', 'utf_8', 'UTF_8', + 'utf', 'UTF', 'u8', 'U8')) +#: Aliases for the latin-1 codec +_LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1', + 'latin', 'LATIN', 'l1', 'L1', 'cp819', 'CP819', '8859', 'iso8859-1', + 'ISO8859-1', 'iso-8859-1', 'ISO-8859-1')) + +# EXCEPTION_CONVERTERS is defined below due to using to_unicode + +def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None): + '''Convert an object into a :class:`unicode` string + + :arg obj: Object to convert to a :class:`unicode` string. This should + normally be a byte :class:`str` + :kwarg encoding: What encoding to try converting the byte :class:`str` as. + Defaults to :term:`utf-8` + :kwarg errors: If errors are found while decoding, perform this action. + Defaults to ``replace`` which replaces the invalid bytes with + a character that means the bytes were unable to be decoded. Other + values are the same as the error handling schemes in the `codec base + classes + <http://docs.python.org/library/codecs.html#codec-base-classes>`_. + For instance ``strict`` which raises an exception and ``ignore`` which + simply omits the non-decodable characters. + :kwarg nonstring: How to treat nonstring values. Possible values are: + + :simplerepr: Attempt to call the object's "simple representation" + method and return that value. Python-2.3+ has two methods that + try to return a simple representation: :meth:`object.__unicode__` + and :meth:`object.__str__`. We first try to get a usable value + from :meth:`object.__unicode__`. If that fails we try the same + with :meth:`object.__str__`. + :empty: Return an empty :class:`unicode` string + :strict: Raise a :exc:`TypeError` + :passthru: Return the object unchanged + :repr: Attempt to return a :class:`unicode` string of the repr of the + object + + Default is ``simplerepr`` + + :raises TypeError: if :attr:`nonstring` is ``strict`` and + a non-:class:`basestring` object is passed in or if :attr:`nonstring` + is set to an unknown value + :raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and + :attr:`obj` is not decodable using the given encoding + :returns: :class:`unicode` string or the original object depending on the + value of :attr:`nonstring`. + + Usually this should be used on a byte :class:`str` but it can take both + byte :class:`str` and :class:`unicode` strings intelligently. Nonstring + objects are handled in different ways depending on the setting of the + :attr:`nonstring` parameter. + + The default values of this function are set so as to always return + a :class:`unicode` string and never raise an error when converting from + a byte :class:`str` to a :class:`unicode` string. However, when you do + not pass validly encoded text (or a nonstring object), you may end up with + output that you don't expect. Be sure you understand the requirements of + your data, not just ignore errors by passing it through this function. + ''' + # Could use isbasestring/isunicode here but we want this code to be as + # fast as possible + if isinstance(obj, basestring): + if isinstance(obj, unicode): + return obj + if encoding in _UTF8_ALIASES: + return unicode(obj, 'utf-8', errors) + if encoding in _LATIN1_ALIASES: + return unicode(obj, 'latin-1', errors) + return obj.decode(encoding, errors) + + if not nonstring: + nonstring = 'simplerepr' + if nonstring == 'empty': + return u'' + elif nonstring == 'passthru': + return obj + elif nonstring == 'simplerepr': + try: + simple = obj.__unicode__() + except (AttributeError, UnicodeError): + simple = None + if not simple: + try: + simple = str(obj) + except UnicodeError: + try: + simple = obj.__str__() + except (UnicodeError, AttributeError): + simple = u'' + if isbytestring(simple): + return unicode(simple, encoding, errors) + return simple + elif nonstring in ('repr', 'strict'): + obj_repr = repr(obj) + if isbytestring(obj_repr): + obj_repr = unicode(obj_repr, encoding, errors) + if nonstring == 'repr': + return obj_repr + raise TypeError('to_unicode was given "%(obj)s" which is neither' + ' a byte string (str) or a unicode string' % + {'obj': obj_repr.encode(encoding, 'replace')}) + + raise TypeError('nonstring value, %(param)s, is not set to a valid' + ' action' % {'param': nonstring}) + +def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None): + '''Convert an object into a byte :class:`str` + + :arg obj: Object to convert to a byte :class:`str`. This should normally + be a :class:`unicode` string. + :kwarg encoding: Encoding to use to convert the :class:`unicode` string + into a byte :class:`str`. Defaults to :term:`utf-8`. + :kwarg errors: If errors are found while encoding, perform this action. + Defaults to ``replace`` which replaces the invalid bytes with + a character that means the bytes were unable to be encoded. Other + values are the same as the error handling schemes in the `codec base + classes + <http://docs.python.org/library/codecs.html#codec-base-classes>`_. + For instance ``strict`` which raises an exception and ``ignore`` which + simply omits the non-encodable characters. + :kwarg nonstring: How to treat nonstring values. Possible values are: + + :simplerepr: Attempt to call the object's "simple representation" + method and return that value. Python-2.3+ has two methods that + try to return a simple representation: :meth:`object.__unicode__` + and :meth:`object.__str__`. We first try to get a usable value + from :meth:`object.__str__`. If that fails we try the same + with :meth:`object.__unicode__`. + :empty: Return an empty byte :class:`str` + :strict: Raise a :exc:`TypeError` + :passthru: Return the object unchanged + :repr: Attempt to return a byte :class:`str` of the :func:`repr` of the + object + + Default is ``simplerepr``. + + :raises TypeError: if :attr:`nonstring` is ``strict`` and + a non-:class:`basestring` object is passed in or if :attr:`nonstring` + is set to an unknown value. + :raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the + bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`. + :returns: byte :class:`str` or the original object depending on the value + of :attr:`nonstring`. + + .. warning:: + + If you pass a byte :class:`str` into this function the byte + :class:`str` is returned unmodified. It is **not** re-encoded with + the specified :attr:`encoding`. The easiest way to achieve that is:: + + to_bytes(to_unicode(text), encoding='utf-8') + + The initial :func:`to_unicode` call will ensure text is + a :class:`unicode` string. Then, :func:`to_bytes` will turn that into + a byte :class:`str` with the specified encoding. + + Usually, this should be used on a :class:`unicode` string but it can take + either a byte :class:`str` or a :class:`unicode` string intelligently. + Nonstring objects are handled in different ways depending on the setting + of the :attr:`nonstring` parameter. + + The default values of this function are set so as to always return a byte + :class:`str` and never raise an error when converting from unicode to + bytes. However, when you do not pass an encoding that can validly encode + the object (or a non-string object), you may end up with output that you + don't expect. Be sure you understand the requirements of your data, not + just ignore errors by passing it through this function. + ''' + # Could use isbasestring, isbytestring here but we want this to be as fast + # as possible + if isinstance(obj, basestring): + if isinstance(obj, str): + return obj + return obj.encode(encoding, errors) + if not nonstring: + nonstring = 'simplerepr' + + if nonstring == 'empty': + return '' + elif nonstring == 'passthru': + return obj + elif nonstring == 'simplerepr': + try: + simple = str(obj) + except UnicodeError: + try: + simple = obj.__str__() + except (AttributeError, UnicodeError): + simple = None + if not simple: + try: + simple = obj.__unicode__() + except (AttributeError, UnicodeError): + simple = '' + if isunicodestring(simple): + simple = simple.encode(encoding, 'replace') + return simple + elif nonstring in ('repr', 'strict'): + try: + obj_repr = obj.__repr__() + except (AttributeError, UnicodeError): + obj_repr = '' + if isunicodestring(obj_repr): + obj_repr = obj_repr.encode(encoding, errors) + else: + obj_repr = str(obj_repr) + if nonstring == 'repr': + return obj_repr + raise TypeError('to_bytes was given "%(obj)s" which is neither' + ' a unicode string or a byte string (str)' % {'obj': obj_repr}) + + raise TypeError('nonstring value, %(param)s, is not set to a valid' + ' action' % {'param': nonstring}) + + +# force the return value of a function to be unicode. Use with partial to +# ensure that a filter will return unicode values. +def unicode_wrap(func, *args, **kwargs): + return to_unicode(func(*args, **kwargs), nonstring='passthru') diff --git a/plugins/inventory/cobbler.py b/plugins/inventory/cobbler.py index f607da2dc2..f352c8cf9d 100755 --- a/plugins/inventory/cobbler.py +++ b/plugins/inventory/cobbler.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ Cobbler external inventory script diff --git a/plugins/inventory/ec2.ini b/plugins/inventory/ec2.ini index c66bf309b1..66f65a69d2 100644 --- a/plugins/inventory/ec2.ini +++ b/plugins/inventory/ec2.ini @@ -24,14 +24,17 @@ regions_exclude = us-gov-west-1,cn-north-1 # This is the normal destination variable to use. If you are running Ansible # from outside EC2, then 'public_dns_name' makes the most sense. If you are # running Ansible from within EC2, then perhaps you want to use the internal -# address, and should set this to 'private_dns_name'. +# address, and should set this to 'private_dns_name'. The key of an EC2 tag +# may optionally be used; however the boto instance variables hold precedence +# in the event of a collision. destination_variable = public_dns_name # For server inside a VPC, using DNS names may not make sense. When an instance # has 'subnet_id' set, this variable is used. If the subnet is public, setting # this to 'ip_address' will return the public IP address. For instances in a # private subnet, this should be set to 'private_ip_address', and Ansible must -# be run from with EC2. +# be run from with EC2. The key of an EC2 tag may optionally be used; however +# the boto instance variables hold precedence in the event of a collision. vpc_destination_variable = ip_address # To tag instances on EC2 with the resource records that point to them from diff --git a/plugins/inventory/ec2.py b/plugins/inventory/ec2.py index 9d2dec38d3..1b9944d108 100755 --- a/plugins/inventory/ec2.py +++ b/plugins/inventory/ec2.py @@ -385,9 +385,13 @@ class Ec2Inventory(object): # Select the best destination address if instance.subnet_id: - dest = getattr(instance, self.vpc_destination_variable) + dest = getattr(instance, self.vpc_destination_variable, None) + if dest is None: + dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None) else: - dest = getattr(instance, self.destination_variable) + dest = getattr(instance, self.destination_variable, None) + if dest is None: + dest = getattr(instance, 'tags').get(self.destination_variable, None) if not dest: # Skip instances we cannot address (e.g. private VPC subnet) @@ -432,7 +436,11 @@ class Ec2Inventory(object): self.push(self.inventory, key_name, dest) if self.nested_groups: self.push_group(self.inventory, 'keys', key_name) - + + # Inventory: Group by VPC + if instance.vpc_id: + self.push(self.inventory, self.to_safe('vpc_id_' + instance.vpc_id), dest) + # Inventory: Group by security group try: for group in instance.groups: @@ -461,6 +469,10 @@ class Ec2Inventory(object): if self.nested_groups: self.push_group(self.inventory, 'route53', name) + # Global Tag: instances without tags + if len(instance.tags) == 0: + self.push(self.inventory, 'tag_none', dest) + # Global Tag: tag all EC2 instances self.push(self.inventory, 'ec2', dest) @@ -504,13 +516,13 @@ class Ec2Inventory(object): self.push(self.inventory, instance.availability_zone, dest) if self.nested_groups: self.push_group(self.inventory, region, instance.availability_zone) - + # Inventory: Group by instance type type_name = self.to_safe('type_' + instance.instance_class) self.push(self.inventory, type_name, dest) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) - + # Inventory: Group by security group try: if instance.security_group: @@ -644,7 +656,7 @@ class Ec2Inventory(object): # try updating the cache self.do_api_calls_update_cache() if not self.args.host in self.index: - # host migh not exist anymore + # host might not exist anymore return self.json_format_dict({}, True) (region, instance_id) = self.index[self.args.host] diff --git a/plugins/inventory/freeipa.py b/plugins/inventory/freeipa.py index ae4ad59fd2..caf336239c 100755 --- a/plugins/inventory/freeipa.py +++ b/plugins/inventory/freeipa.py @@ -1,19 +1,79 @@ -#!/usr/bin/python +#!/usr/bin/env python -import json +import argparse from ipalib import api -api.bootstrap(context='cli') -api.finalize() -api.Backend.xmlclient.connect() -inventory = {} -hostvars={} -meta={} -result =api.Command.hostgroup_find()['result'] -for hostgroup in result: - inventory[hostgroup['cn'][0]] = { 'hosts': [host for host in hostgroup['member_host']]} - for host in hostgroup['member_host']: - hostvars[host] = {} -inventory['_meta'] = {'hostvars': hostvars} -inv_string = json.dumps( inventory) -print inv_string +import json + +def initialize(): + ''' + This function initializes the FreeIPA/IPA API. This function requires + no arguments. A kerberos key must be present in the users keyring in + order for this to work. + ''' + + api.bootstrap(context='cli') + api.finalize() + api.Backend.xmlclient.connect() + + return api + +def list_groups(api): + ''' + This function returns a list of all host groups. This function requires + one argument, the FreeIPA/IPA API object. + ''' + + inventory = {} + hostvars={} + meta={} + + result = api.Command.hostgroup_find()['result'] + + for hostgroup in result: + inventory[hostgroup['cn'][0]] = { 'hosts': [host for host in hostgroup['member_host']]} + + for host in hostgroup['member_host']: + hostvars[host] = {} + + inventory['_meta'] = {'hostvars': hostvars} + inv_string = json.dumps(inventory, indent=1, sort_keys=True) + print inv_string + + return None + +def parse_args(): + ''' + This function parses the arguments that were passed in via the command line. + This function expects no arguments. + ''' + + parser = argparse.ArgumentParser(description='Ansible FreeIPA/IPA ' + 'inventory module') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--list', action='store_true', + help='List active servers') + group.add_argument('--host', help='List details about the specified host') + + return parser.parse_args() + +def print_host(host): + ''' + This function is really a stub, it could return variables to be used in + a playbook. However, at this point there are no variables stored in + FreeIPA/IPA. + + This function expects one string, this hostname to lookup variables for. + ''' + + print json.dumps({}) + + return None + +if __name__ == '__main__': + args = parse_args() + if args.host: + print_host(args.host) + elif args.list: + api = initialize() + list_groups(api) diff --git a/plugins/inventory/gce.py b/plugins/inventory/gce.py index 08a21f12f2..e77178c16b 100755 --- a/plugins/inventory/gce.py +++ b/plugins/inventory/gce.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2013 Google Inc. # # This file is part of Ansible @@ -9,8 +9,9 @@ from ansible import __version__, __author__ try: from setuptools import setup, find_packages except ImportError: - print "Ansible now needs setuptools in order to build. " \ - "Install it using your package manager (usually python-setuptools) or via pip (pip install setuptools)." + print("Ansible now needs setuptools in order to build. Install it using" + " your package manager (usually python-setuptools) or via pip (pip" + " install setuptools).") sys.exit(1) setup(name='ansible', diff --git a/test/integration/Makefile b/test/integration/Makefile index cf15c753cf..7ed2569be3 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -15,7 +15,7 @@ CREDENTIALS_ARG = endif # http://unix.stackexchange.com/questions/30091/fix-or-alternative-for-mktemp-in-os-x -TMPDIR = $(shell mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') +MYTMPDIR = $(shell mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') VAULT_PASSWORD_FILE = vault-password @@ -33,10 +33,13 @@ includes: ansible-playbook test_includes.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS) unicode: - ansible-playbook unicode.yml -i $(INVENTORY) -e @$(VARS_FILE) -v $(TEST_FLAGS) + ansible-playbook unicode.yml -i $(INVENTORY) -e @$(VARS_FILE) -v $(TEST_FLAGS) -e 'extra_var=café' # Test the start-at-task flag #9571 ansible-playbook unicode.yml -i $(INVENTORY) -e @$(VARS_FILE) -v --start-at-task '*¶' -e 'start_at_task=True' $(TEST_FLAGS) +mine: + ansible-playbook mine.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) + non_destructive: ansible-playbook non_destructive.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) @@ -91,8 +94,7 @@ gce_cleanup: python cleanup_gce.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" rackspace_cleanup: - @echo "FIXME - cleanup_rax.py not yet implemented" - @# python cleanup_rax.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" + python cleanup_rax.py -y --match="^$(CLOUD_RESOURCE_PREFIX)" $(CREDENTIALS_FILE): @echo "No credentials file found. A file named '$(CREDENTIALS_FILE)' is needed to provide credentials needed to run cloud tests. See sample 'credentials.template' file." @@ -120,7 +122,7 @@ rackspace: $(CREDENTIALS_FILE) test_galaxy: test_galaxy_spec test_galaxy_yaml test_galaxy_spec: - mytmpdir=$(TMPDIR) ; \ + mytmpdir=$(MYTMPDIR) ; \ ansible-galaxy install -r galaxy_rolesfile -p $$mytmpdir/roles ; \ cp galaxy_playbook.yml $$mytmpdir ; \ ansible-playbook -i $(INVENTORY) $$mytmpdir/galaxy_playbook.yml -v $(TEST_FLAGS) ; \ @@ -129,7 +131,7 @@ test_galaxy_spec: exit $$RC test_galaxy_yaml: - mytmpdir=$(TMPDIR) ; \ + mytmpdir=$(MYTMPDIR) ; \ ansible-galaxy install -r galaxy_roles.yml -p $$mytmpdir/roles ; \ cp galaxy_playbook.yml $$mytmpdir ; \ ansible-playbook -i $(INVENTORY) $$mytmpdir/galaxy_playbook.yml -v $(TEST_FLAGS) ; \ diff --git a/test/integration/cleanup_rax.py b/test/integration/cleanup_rax.py new file mode 100644 index 0000000000..5fdd15d8a1 --- /dev/null +++ b/test/integration/cleanup_rax.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import os +import yaml +import argparse + +try: + import pyrax + HAS_PYRAX = True +except ImportError: + HAS_PYRAX = False + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-y', '--yes', action='store_true', dest='assumeyes', + default=False, help="Don't prompt for confirmation") + parser.add_argument('--match', dest='match_re', + default='^ansible-testing', + help='Regular expression used to find resources ' + '(default: %(default)s)') + + return parser.parse_args() + + +def authenticate(): + try: + with open(os.path.realpath('./credentials.yml')) as f: + credentials = yaml.load(f) + except Exception as e: + raise SystemExit(e) + + try: + pyrax.set_credentials(credentials.get('rackspace_username'), + credentials.get('rackspace_api_key')) + except Exception as e: + raise SystemExit(e) + + +def prompt_and_delete(item, prompt, assumeyes): + if not assumeyes: + assumeyes = raw_input(prompt).lower() == 'y' + assert (hasattr(item, 'delete') or hasattr(item, 'terminate'), + "Class <%s> has no delete or terminate attribute" % item.__class__) + if assumeyes: + if hasattr(item, 'delete'): + item.delete() + print ("Deleted %s" % item) + if hasattr(item, 'terminate'): + item.terminate() + print ("Terminated %s" % item) + + +def delete_rax(args): + """Function for deleting CloudServers""" + for region in pyrax.identity.services.compute.regions: + cs = pyrax.connect_to_cloudservers(region=region) + servers = cs.servers.list(search_opts=dict(name='^%s' % args.match_re)) + for server in servers: + prompt_and_delete(server, + 'Delete matching %s? [y/n]: ' % server, + args.assumeyes) + + +def main(): + if not HAS_PYRAX: + raise SystemExit('The pyrax python module is required for this script') + + args = parse_args() + authenticate() + delete_rax(args) + + +if __name__ == '__main__': + main() diff --git a/test/integration/credentials.template b/test/integration/credentials.template index 12316254bb..4894f5827b 100644 --- a/test/integration/credentials.template +++ b/test/integration/credentials.template @@ -1,4 +1,9 @@ --- +# Rackspace Credentials +rackspace_username: +rackspace_api_key: +rackspace_region: + # AWS Credentials ec2_access_key: ec2_secret_key: diff --git a/test/integration/integration_config.yml b/test/integration/integration_config.yml index 4c2fb2a0a5..bf5d6db3de 100644 --- a/test/integration/integration_config.yml +++ b/test/integration/integration_config.yml @@ -1,4 +1,5 @@ --- +win_output_dir: 'C:/temp/' output_dir: ~/ansible_testing non_root_test_user: ansible pip_test_package: epdb diff --git a/test/integration/rackspace.yml b/test/integration/rackspace.yml index a6ba60c13e..fd3079e9de 100644 --- a/test/integration/rackspace.yml +++ b/test/integration/rackspace.yml @@ -1,4 +1,9 @@ -- hosts: testhost - gather_facts: True - roles: [] - +--- +- hosts: localhost + connection: local + gather_facts: false + tags: + - rackspace + roles: + - role: test_rax + tags: test_rax diff --git a/test/integration/roles/prepare_win_tests/tasks/main.yml b/test/integration/roles/prepare_win_tests/tasks/main.yml new file mode 100644 index 0000000000..756c977fb1 --- /dev/null +++ b/test/integration/roles/prepare_win_tests/tasks/main.yml @@ -0,0 +1,30 @@ +# test code for the windows versions of copy, file and template module +# originally +# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com> + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + + +- name: clean out the test directory + win_file: name={{win_output_dir|mandatory}} state=absent + tags: + - prepare + +- name: create the test directory + win_file: name={{win_output_dir}} state=directory + tags: + - prepare + diff --git a/test/integration/roles/setup_postgresql_db/tasks/main.yml b/test/integration/roles/setup_postgresql_db/tasks/main.yml index d306ac3b7a..fbcc9cab72 100644 --- a/test/integration/roles/setup_postgresql_db/tasks/main.yml +++ b/test/integration/roles/setup_postgresql_db/tasks/main.yml @@ -61,13 +61,21 @@ - name: Copy pg_hba into place copy: src=pg_hba.conf dest="{{ pg_hba_location }}" owner="postgres" group="root" mode="0644" -- name: Generate locale on Debian systems +- name: Generate pt_BR locale (Debian) command: locale-gen pt_BR when: ansible_os_family == 'Debian' -- name: Generate locale on Debian systems +- name: Generate es_MX locale (Debian) command: locale-gen es_MX when: ansible_os_family == 'Debian' +- name: Generate pt_BR locale (Red Hat) + command: localedef -f ISO-8859-1 -i pt_BR pt_BR + when: ansible_os_family == 'RedHat' + +- name: Generate es_MX locale (Red Hat) + command: localedef -f ISO-8859-1 -i es_MX es_MX + when: ansible_os_family == 'RedHat' + - name: restart postgresql service service: name={{ postgresql_service }} state=restarted diff --git a/test/integration/roles/test_apt/tasks/apt-builddep.yml b/test/integration/roles/test_apt/tasks/apt-builddep.yml new file mode 100644 index 0000000000..e3f9357b12 --- /dev/null +++ b/test/integration/roles/test_apt/tasks/apt-builddep.yml @@ -0,0 +1,55 @@ +# test installing build-deps using netcat and quilt as test victims. +# +# Deps can be discovered like so (taken from ubuntu 12.04) +# ==== +# root@localhost:~ # apt-rdepends --build-depends --follow=DEPENDS netcat +# Reading package lists... Done +# Building dependency tree +# Reading state information... Done +# netcat +# Build-Depends: debhelper (>= 8.0.0) +# Build-Depends: quilt +# root@localhost:~ # +# ==== +# Since many things depend on debhelper, let's just uninstall quilt, then +# install build-dep for netcat to get it back. build-dep doesn't have an +# uninstall, so we don't need to test for reverse actions (eg, uninstall +# build-dep and ensure things are clean) + +# uninstall quilt +- name: check quilt with dpkg + shell: dpkg -s quilt + register: dpkg_result + ignore_errors: true + tags: ['test_apt_builddep'] + +- name: uninstall quilt with apt + apt: pkg=quilt state=absent purge=yes + register: apt_result + when: dpkg_result|success + tags: ['test_apt_builddep'] + +# install build-dep for netcat +- name: install netcat build-dep with apt + apt: pkg=netcat state=build-dep + register: apt_result + tags: ['test_apt_builddep'] + +- name: verify build_dep of netcat + assert: + that: + - "'changed' in apt_result" + tags: ['test_apt_builddep'] + +# ensure debhelper and qilt are installed +- name: check build_deps with dpkg + shell: dpkg --get-selections | egrep '(debhelper|quilt)' + failed_when: False + register: dpkg_result + tags: ['test_apt_builddep'] + +- name: verify build_deps are really there + assert: + that: + - "dpkg_result.rc == 0" + tags: ['test_apt_builddep'] diff --git a/test/integration/roles/test_apt/tasks/main.yml b/test/integration/roles/test_apt/tasks/main.yml index 4f2215f57a..8976087371 100644 --- a/test/integration/roles/test_apt/tasks/main.yml +++ b/test/integration/roles/test_apt/tasks/main.yml @@ -19,3 +19,5 @@ - include: 'apt.yml' when: ansible_distribution in ('Ubuntu', 'Debian') +- include: 'apt-builddep.yml' + when: ansible_distribution in ('Ubuntu', 'Debian') diff --git a/test/integration/roles/test_binary/tasks/main.yml b/test/integration/roles/test_binary/tasks/main.yml index 7ae9f16dc9..486ee6d6b0 100644 --- a/test/integration/roles/test_binary/tasks/main.yml +++ b/test/integration/roles/test_binary/tasks/main.yml @@ -40,7 +40,7 @@ content: "{{ simple_accents }}\n" dest: "{{ output_dir }}/from_playbook.txt" -- name: Check that what was written matches +- name: Check that copying utf-8 content matches stat: path: "{{ output_dir }}/from_playbook.txt" register: results @@ -54,7 +54,7 @@ content: "{{ utf8_simple_accents|b64decode }}\n" dest: "{{ output_dir }}/b64_utf8.txt" -- name: Check that what was written matches +- name: Check that utf8 in a base64 string matches stat: path: "{{ output_dir }}/b64_utf8.txt" register: results @@ -68,7 +68,7 @@ content: "{{ latin1_simple_accents|b64decode }}\n" dest: "{{ output_dir }}/b64_latin1.txt" -- name: Check that what was written matches +- name: Check that latin1 in a base64 string matches stat: path: "{{ output_dir }}/b64_latin1.txt" register: results @@ -76,13 +76,16 @@ - assert: that: - 'results.stat.checksum == b64_latin1.stat.checksum' + # This one depends on being able to pass binary data through + # Might be a while before we find a solution for this + ignore_errors: True - name: Template with a unicode string from the playbook template: src: "from_playbook_template.j2" dest: "{{ output_dir }}/from_playbook_template.txt" -- name: Check that what was written matches +- name: Check that writing a template from a playbook var matches stat: path: "{{ output_dir }}/from_playbook_template.txt" register: results @@ -96,7 +99,7 @@ src: "b64_utf8_template.j2" dest: "{{ output_dir }}/b64_utf8_template.txt" -- name: Check that what was written matches +- name: Check that writing a template from a base64 encoded utf8 string matches stat: path: "{{ output_dir }}/b64_utf8_template.txt" register: results @@ -110,7 +113,7 @@ src: "b64_latin1_template.j2" dest: "{{ output_dir }}/b64_latin1_template.txt" -- name: Check that what was written matches +- name: Check that writing a template from a base64 encoded latin1 string matches stat: path: "{{ output_dir }}/b64_latin1_template.txt" register: results @@ -118,6 +121,9 @@ - assert: that: - 'results.stat.checksum == b64_latin1.stat.checksum' + # This one depends on being able to pass binary data through + # Might be a while before we find a solution for this + ignore_errors: True # These might give garbled output but none of them should traceback - debug: var=simple_accents diff --git a/test/integration/roles/test_conditionals/tasks/main.yml b/test/integration/roles/test_conditionals/tasks/main.yml index 8d794e497f..01a4f960d7 100644 --- a/test/integration/roles/test_conditionals/tasks/main.yml +++ b/test/integration/roles/test_conditionals/tasks/main.yml @@ -293,3 +293,7 @@ that: - result.results|length == 3 - result.results[1].skipped + +- name: test complex templated condition + debug: msg="it works" + when: vars_file_var in things1|union([vars_file_var]) diff --git a/test/integration/roles/test_file/tasks/main.yml b/test/integration/roles/test_file/tasks/main.yml index 2126587e6c..d03ded13b6 100644 --- a/test/integration/roles/test_file/tasks/main.yml +++ b/test/integration/roles/test_file/tasks/main.yml @@ -106,6 +106,15 @@ that: - "file6_result.changed == true" +- name: touch a hard link + file: src={{output_file}} dest={{output_dir}}/hard.txt state=touch + register: file6_touch_result + +- name: verify that the hard link was touched + assert: + that: + - "file6_touch_result.changed == true" + - name: create a directory file: path={{output_dir}}/foobar state=directory register: file7_result diff --git a/test/integration/roles/test_filters/files/foo.txt b/test/integration/roles/test_filters/files/foo.txt index 5ee5a5812c..c5af545d3a 100644 --- a/test/integration/roles/test_filters/files/foo.txt +++ b/test/integration/roles/test_filters/files/foo.txt @@ -1,17 +1,6 @@ This is a test of various filter plugins found in Ansible (ex: core.py), and not so much a test of the core filters in Jinja2. -Dumping a nested structure to JSON - -[ - "this is a list element", - { - "this": "is a hash element in a list", - "warp": 9, - "where": "endor" - } -] - Dumping the same structure to YAML - this is a list element diff --git a/test/integration/roles/test_filters/tasks/main.yml b/test/integration/roles/test_filters/tasks/main.yml index 985cbf8327..3d1ee322e3 100644 --- a/test/integration/roles/test_filters/tasks/main.yml +++ b/test/integration/roles/test_filters/tasks/main.yml @@ -22,6 +22,9 @@ - debug: var=some_registered_var +- name: Verify that we workaround a py26 json bug + template: src=py26json.j2 dest={{output_dir}}/py26json.templated mode=0644 + - name: fill in a basic template template: src=foo.j2 dest={{output_dir}}/foo.templated mode=0644 register: template_result diff --git a/test/integration/roles/test_filters/templates/foo.j2 b/test/integration/roles/test_filters/templates/foo.j2 index 6d1dde20c2..cf592f98cc 100644 --- a/test/integration/roles/test_filters/templates/foo.j2 +++ b/test/integration/roles/test_filters/templates/foo.j2 @@ -1,10 +1,6 @@ This is a test of various filter plugins found in Ansible (ex: core.py), and not so much a test of the core filters in Jinja2. -Dumping a nested structure to JSON - -{{ some_structure | to_nice_json }} - Dumping the same structure to YAML {{ some_structure | to_nice_yaml }} diff --git a/test/integration/roles/test_filters/templates/py26json.j2 b/test/integration/roles/test_filters/templates/py26json.j2 new file mode 100644 index 0000000000..b87d3c8090 --- /dev/null +++ b/test/integration/roles/test_filters/templates/py26json.j2 @@ -0,0 +1,2 @@ +Provoke a python2.6 json bug +{{ hostvars | to_nice_json }} diff --git a/test/integration/roles/test_git/tasks/main.yml b/test/integration/roles/test_git/tasks/main.yml index cbdd8f9556..4bdc1d8bd8 100644 --- a/test/integration/roles/test_git/tasks/main.yml +++ b/test/integration/roles/test_git/tasks/main.yml @@ -146,6 +146,7 @@ git: repo: '{{ repo_format1 }}' update: no + clone: no accept_hostkey: yes register: git_result @@ -205,6 +206,41 @@ that: - 'git_result.failed' +# Same as the previous test, but this time we specify which ref +# contains the SHA1 +- name: update to revision by specifying the refspec + git: + repo: https://github.com/ansible/ansible-examples.git + dest: '{{ checkout_dir }}' + version: 2cfde3668b8bb10fbe2b9d5cec486025ad8cc51b + refspec: refs/pull/7/merge + +- name: check HEAD after update with refspec + command: git rev-parse HEAD chdir="{{ checkout_dir }}" + register: git_result + +- assert: + that: + - 'git_result.stdout == "2cfde3668b8bb10fbe2b9d5cec486025ad8cc51b"' + +- name: clear checkout_dir + file: state=absent path={{ checkout_dir }} + +- name: clone to revision by specifying the refspec + git: + repo: https://github.com/ansible/ansible-examples.git + dest: '{{ checkout_dir }}' + version: 2cfde3668b8bb10fbe2b9d5cec486025ad8cc51b + refspec: refs/pull/7/merge + +- name: check HEAD after update with refspec + command: git rev-parse HEAD chdir="{{ checkout_dir }}" + register: git_result + +- assert: + that: + - 'git_result.stdout == "2cfde3668b8bb10fbe2b9d5cec486025ad8cc51b"' + # # Submodule tests # diff --git a/test/integration/roles/test_lookups/tasks/main.yml b/test/integration/roles/test_lookups/tasks/main.yml index 6480b18b35..8440ff5772 100644 --- a/test/integration/roles/test_lookups/tasks/main.yml +++ b/test/integration/roles/test_lookups/tasks/main.yml @@ -81,7 +81,7 @@ - "wc_result.stdout == '9'" - "cat_result.stdout == newpass" -# ENV LOOKUP +# ENV LOOKUP - name: get first environment var name shell: env | head -n1 | cut -d\= -f1 @@ -92,16 +92,16 @@ register: known_var_value - name: use env lookup to get known var - set_fact: + set_fact: test_val: "{{ lookup('env', known_var_name.stdout) }}" - debug: var=known_var_name.stdout - debug: var=known_var_value.stdout -- debug: var=test_val +- debug: var=test_val - name: compare values assert: - that: + that: - "test_val == known_var_value.stdout" @@ -109,11 +109,23 @@ # https://github.com/ansible/ansible/issues/6550 - name: confirm pipe lookup works with a single positional arg - debug: msg="{{ lookup('pipe', 'ls') }}" + debug: msg="{{ lookup('pipe', 'ls') }}" -# https://github.com/ansible/ansible/issues/6550 -- name: confirm pipe lookup works with multiple positional args - debug: msg="{{ lookup('pipe', 'ls -l /tmp') }}" +# LOOKUP TEMPLATING + +- name: use bare interpolation + debug: msg="got {{item}}" + with_items: things1 + register: bare_var + +- name: verify that list was interpolated + assert: + that: + - "bare_var.results[0].item == 1" + - "bare_var.results[1].item == 2" +- name: use list with undefined var in it + debug: msg={{item}} + with_items: things2 diff --git a/test/integration/roles/test_mysql_user/tasks/user_password_update_test.yml b/test/integration/roles/test_mysql_user/tasks/user_password_update_test.yml index d342d0a4ac..8dcc414fde 100644 --- a/test/integration/roles/test_mysql_user/tasks/user_password_update_test.yml +++ b/test/integration/roles/test_mysql_user/tasks/user_password_update_test.yml @@ -57,11 +57,12 @@ ignore_errors: true register: result +- debug: var=result.msg - name: assert output message that database not create with old password assert: that: - "result.failed == true" - - "'check login_user and login_password are correct' in result.msg" + - "'check login credentials (login_user, and login_password' in result.msg" - name: create database using user2 and new password mysql_db: name={{ db_name }} state=present login_user={{ user_name_2 }} login_password={{ user_password_1 }} diff --git a/test/integration/roles/test_rax/defaults/main.yml b/test/integration/roles/test_rax/defaults/main.yml new file mode 100644 index 0000000000..4854b645cf --- /dev/null +++ b/test/integration/roles/test_rax/defaults/main.yml @@ -0,0 +1,3 @@ +--- +rackspace_region: IAD +resource_prefix: ansible-testing diff --git a/test/integration/roles/test_rax/tasks/main.yml b/test/integration/roles/test_rax/tasks/main.yml new file mode 100644 index 0000000000..e4422d8859 --- /dev/null +++ b/test/integration/roles/test_rax/tasks/main.yml @@ -0,0 +1,801 @@ +--- +- name: Check for required variables + assert: + that: + - resource_prefix is defined and resource_prefix + - rackspace_username is defined and rackspace_username + - rackspace_api_key is defined and rackspace_api_key + - rackspace_region is defined and rackspace_region + +# ============================================================ +- name: Test rax with no args + rax: + ignore_errors: true + register: rax + +- name: Validate results of rax with no args + assert: + that: + - rax|failed + - rax.msg == 'No credentials supplied!' +# ============================================================ + + + +# ============================================================ +- name: Test rax with credentials + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + ignore_errors: true + register: rax + +- name: Validate results of rax with only creds + assert: + that: + - rax|failed + - rax.msg.startswith('None is not a valid region') +# ============================================================ + + + +# ============================================================ +- name: Test rax with creds and region + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + ignore_errors: true + register: rax + +- name: Validate rax creds and region + assert: + that: + - rax|failed + - rax.msg == 'image is required for the "rax" module' +# ============================================================ + + + +# ============================================================ +- name: Test rax with creds, region and image + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + ignore_errors: true + register: rax + +- name: Validate rax with creds, region and image + assert: + that: + - rax|failed + - rax.msg == 'flavor is required for the "rax" module' +# ============================================================ + + + +# ============================================================ +- name: Test rax with creds, region, image and flavor + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + ignore_errors: true + register: rax + +- name: Validate rax with creds, region, image and flavor + assert: + that: + - rax|failed + - rax.msg == 'name is required for the "rax" module' +# ============================================================ + + + +# ============================================================ +- name: Test rax with creds, region, image, flavor and name + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-1" + register: rax + +- name: Validate rax with creds, region, image, flavor and name + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 1 + - rax.instances[0].name == "{{ resource_prefix }}-1" + - rax.instances[0] == rax.success[0] + - rax.instances[0].rax_status == 'BUILD' + +- name: "Delete integration 1" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-1" + state: absent + wait: true + register: rax + +- name: "Validate delete integration 1" + assert: + that: + - rax|changed + - rax.action == 'delete' + - rax.success[0].name == "{{ resource_prefix }}-1" +# ============================================================ + + + +# ============================================================ +- name: Test rax basic idempotency 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-2" + wait: true + register: rax + +- name: Validate rax basic idepmpotency 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 1 + - rax.instances[0].name == "{{ resource_prefix }}-2" + - rax.instances[0] == rax.success[0] + - rax.instances[0].rax_status == 'ACTIVE' + +- name: Test rax basic idempotency 2 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-2" + wait: true + register: rax + +- name: Validate rax basic idempotency 2 + assert: + that: + - rax|success + - not rax|changed + - not rax.action + - rax.instances|length == 1 + - rax.instances[0].name == "{{ resource_prefix }}-2" + - not rax.success + +- name: "Delete integration 2" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-2" + state: absent + wait: true + register: rax + +- name: "Validate delete integration 2" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success[0].name == "{{ resource_prefix }}-2" + - rax.success[0].rax_status == "DELETED" +# ============================================================ + + + +# ============================================================ +- name: Test rax basic idempotency with meta 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-3" + meta: + foo: bar + wait: true + register: rax + +- name: Validate rax basic idepmpotency with meta 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 1 + - rax.instances[0].name == "{{ resource_prefix }}-3" + - rax.instances[0] == rax.success[0] + - rax.instances[0].rax_status == 'ACTIVE' + - rax.instances[0].rax_metadata.foo == 'bar' + +- name: Test rax basic idempotency with meta 2 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-3" + meta: + foo: bar + wait: true + register: rax + +- name: Validate rax basic idempotency with meta 2 + assert: + that: + - rax|success + - not rax|changed + - not rax.action + - rax.instances|length == 1 + - rax.instances[0].name == "{{ resource_prefix }}-3" + - not rax.success + +- name: "Delete integration 3" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-3" + state: absent + meta: + foo: bar + wait: true + register: rax + +- name: "Validate delete integration 3" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success[0].name == "{{ resource_prefix }}-3" + - rax.success[0].rax_status == "DELETED" +# ============================================================ + + + +# ============================================================ +- name: Test rax basic idempotency multi server 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-4" + count: 2 + wait: true + register: rax + +- name: Validate rax basic idepmpotency multi server 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + +- name: Test rax basic idempotency multi server 2 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-4" + count: 2 + wait: true + register: rax + +- name: Validate rax basic idempotency multi server 2 + assert: + that: + - rax|success + - not rax|changed + - not rax.action + - rax.instances|length == 2 + - not rax.success + +- name: Test rax basic idempotency multi server 3 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-4" + count: 3 + wait: true + register: rax + +- name: Validate rax basic idempotency multi server 3 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 3 + - rax.success|length == 1 + +- name: "Delete integration 4" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-4" + count: 3 + state: absent + wait: true + register: rax + +- name: "Validate delete integration 4" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 3 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group without exact_count 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-5-%02d" + count: 2 + group: "{{ resource_prefix }}-5" + wait: true + register: rax + +- name: Validate rax multi server group without exact_count 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|length == 2 + +- name: "Test delete integration 5" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-5-%02d" + count: 2 + group: "{{ resource_prefix }}-5" + wait: true + state: absent + register: rax + +- name: "Validate delete integration 5" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 2 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group without exact_count non-idempotency 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-6-%02d" + count: 2 + group: "{{ resource_prefix }}-6" + wait: true + register: rax + +- name: Validate rax multi server group without exact_count non-idempotency 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|length == 2 + +- name: Test rax multi server group without exact_count non-idempotency 2 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-6-%02d" + count: 2 + group: "{{ resource_prefix }}-6" + wait: true + register: rax + +- name: Validate rax multi server group without exact_count non-idempotency 2 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 4 + - rax.instances|map(attribute='rax_name')|unique|length == 4 + +- name: "Test delete integration 6" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-6-%02d" + count: 4 + group: "{{ resource_prefix }}-6" + wait: true + state: absent + register: rax + +- name: "Validate delete integration 6" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 4 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group with exact_count 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-7-%02d" + count: 2 + exact_count: true + group: "{{ resource_prefix }}-7" + wait: true + register: rax + +- name: Validate rax multi server group with exact_count 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|length == 2 + +- name: Test rax multi server group with exact_count 2 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-7-%02d" + count: 2 + exact_count: true + group: "{{ resource_prefix }}-7" + wait: true + register: rax + +- name: Validate rax multi server group with exact_count 2 + assert: + that: + - rax|success + - not rax|changed + - not rax.action + - rax.instances|length == 2 + - rax.instances|map(attribute='rax_name')|unique|length == 2 + +- name: Test rax multi server group with exact_count 3 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-7-%02d" + count: 4 + exact_count: true + group: "{{ resource_prefix }}-7" + wait: true + register: rax + +- name: Validate rax multi server group with exact_count 3 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 4 + - rax.success|length == 2 + - rax.instances|map(attribute='rax_name')|unique|length == 4 + + +- name: "Test delete integration 7" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-7-%02d" + count: 0 + exact_count: true + group: "{{ resource_prefix }}-7" + wait: true + register: rax + +- name: "Validate delete integration 7" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 4 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group without exact_count and disabled auto_increment 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-8" + count: 2 + group: "{{ resource_prefix }}-8" + auto_increment: false + wait: true + register: rax + +- name: Validate rax multi server group without exact_count and disabled auto_increment 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|length == 1 + +- name: "Test delete integration 8" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-8" + count: 2 + group: "{{ resource_prefix }}-8" + auto_increment: false + wait: true + state: absent + register: rax + +- name: "Validate delete integration 8" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 2 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group with exact_count and no printf 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-9" + count: 2 + exact_count: true + group: "{{ resource_prefix }}-9" + wait: true + register: rax + +- name: Validate rax multi server group with exact_count and no printf 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|list|sort == ['{{ resource_prefix }}-91', '{{ resource_prefix }}-92'] + +- name: "Test delete integration 9" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-9" + count: 0 + exact_count: true + group: "{{ resource_prefix }}-9" + wait: true + register: rax + +- name: "Validate delete integration 9" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 2 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group with exact_count and offset 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-10-%03d" + count: 2 + count_offset: 10 + exact_count: true + group: "{{ resource_prefix }}-10" + wait: true + register: rax + +- name: Validate rax multi server group with exact_count and offset 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|list|sort == ['{{ resource_prefix }}-10-010', '{{ resource_prefix }}-10-011'] + +- name: "Test delete integration 10" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-10-%03d" + count: 0 + count_offset: 10 + exact_count: true + group: "{{ resource_prefix }}-10" + wait: true + register: rax + +- name: "Validate delete integration 10" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 2 + - not rax.instances +# ============================================================ + + + +# ============================================================ +- name: Test rax multi server group with exact_count and offset 1 + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-10-%03d" + count: 2 + count_offset: 10 + exact_count: true + group: "{{ resource_prefix }}-10" + wait: true + register: rax + +- name: Validate rax multi server group with exact_count and offset 1 + assert: + that: + - rax|success + - rax|changed + - rax.action == 'create' + - rax.instances|length == 2 + - rax.instances == rax.success + - rax.instances|map(attribute='rax_name')|unique|list|sort == ['{{ resource_prefix }}-10-010', '{{ resource_prefix }}-10-011'] + +- name: "Test delete integration 10" + rax: + username: "{{ rackspace_username }}" + api_key: "{{ rackspace_api_key }}" + region: "{{ rackspace_region }}" + image: ubuntu-1204-lts-precise-pangolin + flavor: performance1-1 + name: "{{ resource_prefix }}-10-%03d" + count: 0 + count_offset: 10 + exact_count: true + group: "{{ resource_prefix }}-10" + wait: true + register: rax + +- name: "Validate delete integration 10" + assert: + that: + - rax|success + - rax|changed + - rax.action == 'delete' + - rax.success|length == 2 + - not rax.instances +# ============================================================ diff --git a/test/integration/roles/test_subversion/tasks/main.yml b/test/integration/roles/test_subversion/tasks/main.yml index c3d741a290..796f2727a5 100644 --- a/test/integration/roles/test_subversion/tasks/main.yml +++ b/test/integration/roles/test_subversion/tasks/main.yml @@ -44,7 +44,7 @@ # "Revision: 9", # "URL: https://github.com/jimi-c/test_role" # ], -# "befbore": null, +# "before": null, # "changed": true, # "invocation": { # "module_args": "repo=https://github.com/jimi-c/test_role dest=~/ansible_testing/svn", @@ -59,12 +59,17 @@ - "'after' in subverted" - "subverted.after.1 == 'URL: https://github.com/jimi-c/test_role'" - "not subverted.before" - - "subverted.changed" + - "subverted.changed" - name: repeated checkout subversion: repo={{ repo }} dest={{ checkout_dir }} register: subverted2 +- name: verify on a reclone things are marked unchanged + assert: + that: + - "not subverted2.changed" + - name: check for tags stat: path={{ checkout_dir }}/tags register: tags @@ -91,15 +96,28 @@ - debug: var=subverted3 - name: checkout with export - subversion: repo={{ repo }} dest={{ checkout_dir }} export=True + subversion: repo={{ repo }} dest={{ output_dir }}/svn-export export=True register: subverted4 -- name: verify on a reclone things are marked unchanged +- name: check for tags + stat: path={{ output_dir }}/svn-export/tags + register: export_tags + +- name: check for trunk + stat: path={{ output_dir }}/svn-export/trunk + register: export_trunk + +- name: check for branches + stat: path={{ output_dir }}/svn-export/branches + register: export_branches + +- name: assert presence of tags/trunk/branches in export assert: that: - - "not subverted4.changed" + - "export_tags.stat.isdir" + - "export_trunk.stat.isdir" + - "export_branches.stat.isdir" + - "subverted4.changed" # TBA: test for additional options or URL variants welcome - - diff --git a/test/integration/roles/test_unarchive/tasks/main.yml b/test/integration/roles/test_unarchive/tasks/main.yml index 7caa68e65c..3e315a7e94 100644 --- a/test/integration/roles/test_unarchive/tasks/main.yml +++ b/test/integration/roles/test_unarchive/tasks/main.yml @@ -15,6 +15,15 @@ # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# Make sure we start fresh + +- name: Ensure zip is present to create test archive (yum) + yum: name=zip state=latest + when: ansible_pkg_mgr == 'yum' + +- name: Ensure zip is present to create test archive (apt) + apt: name=zip state=latest + when: ansible_pkg_mgr == 'apt' - name: prep our file copy: src=foo.txt dest={{output_dir}}/foo-unarchive.txt @@ -26,7 +35,7 @@ shell: tar cvf test-unarchive.tar.gz foo-unarchive.txt chdir={{output_dir}} - name: prep a zip file - shell: tar cvf test-unarchive.zip foo-unarchive.txt chdir={{output_dir}} + shell: zip test-unarchive.zip foo-unarchive.txt chdir={{output_dir}} - name: create our tar unarchive destination file: path={{output_dir}}/test-unarchive-tar state=directory @@ -89,6 +98,20 @@ - "unarchive02c.changed == false" - "unarchive02c.skipped == true" +- name: unarchive a tar.gz file with creates over an existing file using complex_args + unarchive: + src: "{{output_dir}}/test-unarchive.tar.gz" + dest: "{{output_dir | expanduser}}/test-unarchive-tar-gz" + copy: no + creates: "{{output_dir}}/test-unarchive-tar-gz/foo-unarchive.txt" + register: unarchive02d + +- name: verify that the file was not marked as changed + assert: + that: + - "unarchive02d.changed == false" + - "unarchive02d.skipped == true" + - name: remove our tar.gz unarchive destination file: path={{output_dir}}/test-unarchive-tar-gz state=absent @@ -107,7 +130,7 @@ - name: verify that the file was unarchived file: path={{output_dir}}/test-unarchive-zip/foo-unarchive.txt state=file -- name: remove our tar unarchive destination +- name: remove our zip unarchive destination file: path={{output_dir}}/test-unarchive-zip state=absent - name: remove our test file for the archive @@ -136,3 +159,71 @@ - name: remove our unarchive destination file: path=/tmp/foo-unarchive.txt state=absent + +- name: create our unarchive destination + file: path={{output_dir}}/test-unarchive-tar-gz state=directory + +- name: unarchive and set mode + unarchive: + src: "{{ output_dir }}/test-unarchive.tar.gz" + dest: "{{ output_dir | expanduser }}/test-unarchive-tar-gz" + copy: no + mode: "u+rwX,g-rwx,o-rwx" + register: unarchive06 + +- name: Test that the file modes were changed + stat: + path: "{{ output_dir | expanduser }}/test-unarchive-tar-gz/foo-unarchive.txt" + register: unarchive06_stat + +- name: Test that the file modes were changed + assert: + that: + - "unarchive06.changed == true" + - "unarchive06_stat.stat.mode == '0600'" + +- name: unarchive and set mode + unarchive: + src: "{{ output_dir }}/test-unarchive.tar.gz" + dest: "{{ output_dir | expanduser }}/test-unarchive-tar-gz" + copy: no + mode: "u+rwX,g-rwx,o-rwx" + register: unarchive07 + +- name: Test that the files were not changed + assert: + that: + - "unarchive07.changed == false" + +- name: remove our tar.gz unarchive destination + file: path={{ output_dir }}/test-unarchive-tar-gz state=absent + +- name: create a directory with quotable chars + file: path="{{ output_dir }}/test-quotes~root" state=directory + +- name: unarchive into directory with quotable chars + unarchive: + src: "{{ output_dir }}/test-unarchive.tar.gz" + dest: "{{ output_dir | expanduser }}/test-quotes~root" + copy: no + register: unarchive08 + +- name: Test that unarchive succeeded + assert: + that: + - "unarchive08.changed == true" + +- name: unarchive into directory with quotable chars a second time + unarchive: + src: "{{ output_dir }}/test-unarchive.tar.gz" + dest: "{{ output_dir | expanduser }}/test-quotes~root" + copy: no + register: unarchive09 + +- name: Test that unarchive did nothing + assert: + that: + - "unarchive09.changed == false" + +- name: remove quotable chars test + file: path="{{ output_dir }}/test-quotes~root" state=absent diff --git a/test/integration/roles/test_win_copy/files/foo.txt b/test/integration/roles/test_win_copy/files/foo.txt new file mode 100644 index 0000000000..7c6ded14ec --- /dev/null +++ b/test/integration/roles/test_win_copy/files/foo.txt @@ -0,0 +1 @@ +foo.txt diff --git a/test/integration/roles/test_win_copy/files/subdir/bar.txt b/test/integration/roles/test_win_copy/files/subdir/bar.txt new file mode 100644 index 0000000000..76018072e0 --- /dev/null +++ b/test/integration/roles/test_win_copy/files/subdir/bar.txt @@ -0,0 +1 @@ +baz diff --git a/test/integration/roles/test_win_copy/files/subdir/subdir2/baz.txt b/test/integration/roles/test_win_copy/files/subdir/subdir2/baz.txt new file mode 100644 index 0000000000..76018072e0 --- /dev/null +++ b/test/integration/roles/test_win_copy/files/subdir/subdir2/baz.txt @@ -0,0 +1 @@ +baz diff --git a/test/integration/roles/test_win_copy/files/subdir/subdir2/subdir3/subdir4/qux.txt b/test/integration/roles/test_win_copy/files/subdir/subdir2/subdir3/subdir4/qux.txt new file mode 100644 index 0000000000..78df5b06bd --- /dev/null +++ b/test/integration/roles/test_win_copy/files/subdir/subdir2/subdir3/subdir4/qux.txt @@ -0,0 +1 @@ +qux
\ No newline at end of file diff --git a/test/integration/roles/test_win_copy/meta/main.yml b/test/integration/roles/test_win_copy/meta/main.yml new file mode 100644 index 0000000000..55200b3fc6 --- /dev/null +++ b/test/integration/roles/test_win_copy/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_win_tests + diff --git a/test/integration/roles/test_win_copy/tasks/main.yml b/test/integration/roles/test_win_copy/tasks/main.yml new file mode 100644 index 0000000000..d898219a85 --- /dev/null +++ b/test/integration/roles/test_win_copy/tasks/main.yml @@ -0,0 +1,261 @@ +# test code for the copy module and action plugin +# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com> + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +- name: record the output directory + set_fact: output_file={{win_output_dir}}/foo.txt + +- name: initiate a basic copy +#- name: initiate a basic copy, and also test the mode +# win_copy: src=foo.txt dest={{output_file}} mode=0444 + win_copy: src=foo.txt dest={{output_file}} + register: copy_result + +- debug: var=copy_result + +#- name: check the presence of the output file +- name: check the mode of the output file + win_file: name={{output_file}} state=file + register: file_result_check + +- debug: var=file_result_check + + +#- name: assert the mode is correct +# assert: +# that: +# - "file_result_check.mode == '0444'" + +- name: assert basic copy worked + assert: + that: + - "'changed' in copy_result" +# - "'dest' in copy_result" +# - "'group' in copy_result" +# - "'gid' in copy_result" + - "'checksum' in copy_result" +# - "'owner' in copy_result" +# - "'size' in copy_result" +# - "'src' in copy_result" +# - "'state' in copy_result" +# - "'uid' in copy_result" + +- name: verify that the file was marked as changed + assert: + that: + - "copy_result.changed == true" + +- name: verify that the file checksum is correct + assert: + that: + - "copy_result.checksum[0] == 'c79a6506c1c948be0d456ab5104d5e753ab2f3e6'" + +- name: check the stat results of the file + win_stat: path={{output_file}} + register: stat_results + +- debug: var=stat_results + +- name: assert the stat results are correct + assert: + that: + - "stat_results.stat.exists == true" +# - "stat_results.stat.isblk == false" +# - "stat_results.stat.isfifo == false" +# - "stat_results.stat.isreg == true" +# - "stat_results.stat.issock == false" + - "stat_results.stat.checksum[0] == 'c79a6506c1c948be0d456ab5104d5e753ab2f3e6'" + +- name: overwrite the file via same means + win_copy: src=foo.txt dest={{output_file}} + register: copy_result2 + +- name: assert that the file was not changed + assert: + that: + - "not copy_result2|changed" + +# content system not available in win_copy right now +#- name: overwrite the file using the content system +# win_copy: content="modified" dest={{output_file}} +# register: copy_result3 +# +#- name: assert that the file has changed +# assert: +# that: +# - "copy_result3|changed" +# - "'content' not in copy_result3" + +# test recursive copy + +- name: set the output subdirectory + set_fact: output_subdir={{win_output_dir}}/sub/ + +- name: make an output subdirectory + win_file: name={{output_subdir}} state=directory + +- name: test recursive copy to directory +# win_copy: src=subdir dest={{output_subdir}} directory_mode=0700 + win_copy: src=subdir dest={{output_subdir}} + register: recursive_copy_result + +- debug: var=recursive_copy_result + +- name: check that a file in a directory was transferred + win_stat: path={{win_output_dir}}/sub/subdir/bar.txt + register: stat_bar + +- name: check that a file in a deeper directory was transferred + win_stat: path={{win_output_dir}}/sub/subdir/subdir2/baz.txt + register: stat_bar2 + +- name: check that a file in a directory whose parent contains a directory alone was transferred + win_stat: path={{win_output_dir}}/sub/subdir/subdir2/subdir3/subdir4/qux.txt + register: stat_bar3 + +- name: assert recursive copy things + assert: + that: + - "stat_bar.stat.exists" + - "stat_bar2.stat.exists" + - "stat_bar3.stat.exists" + +- name: stat the recursively copied directories + win_stat: path={{win_output_dir}}/sub/{{item}} + register: dir_stats + with_items: + - "subdir" + - "subdir/subdir2" + - "subdir/subdir2/subdir3" + - "subdir/subdir2/subdir3/subdir4" + +# can't check file mode on windows so commenting this one out. +#- name: assert recursive copied directories mode +# assert: +# that: +# - "{{item.stat.mode}} == 0700" +# with_items: dir_stats.results + + +# errors on this aren't presently ignored so this test is commented out. But it would be nice to fix. +# + +# content param not available in win_copy +#- name: overwrite the file again using the content system, also passing along file params +# win_copy: content="modified" dest={{output_file}} +# register: copy_result4 + +#- name: assert invalid copy input location fails +# win_copy: src=invalid_file_location_does_not_exist dest={{win_output_dir}}/file.txt +# ignore_errors: True +# register: failed_copy + +# owner not available in win_copy, commenting out +#- name: copy already copied directory again +# win_copy: src=subdir dest={{output_subdir | expanduser}} owner={{ansible_ssh_user}} +# register: copy_result5 + +#- name: assert that the directory was not changed +# assert: +# that: +# - "not copy_result5|changed" + +# content not available in win_copy, commenting out. +# issue 8394 +#- name: create a file with content and a literal multiline block +# win_copy: | +# content='this is the first line +# this is the second line +# +# this line is after an empty line +# this line is the last line +# ' +# dest={{win_output_dir}}/multiline.txt +# register: copy_result6 + +#- debug: var=copy_result6 + +#- name: assert the multiline file was created correctly +# assert: +# that: +# - "copy_result6.changed" +# - "copy_result6.dest == '{{win_output_dir|expanduser}}/multiline.txt'" +# - "copy_result6.checksum == '1627d51e7e607c92cf1a502bf0c6cce3'" + +# test overwriting a file as an unprivileged user (pull request #8624) +# this can't be relative to {{win_output_dir}} as ~root usually has mode 700 + +#- name: create world writable directory + #win_file: dest=/tmp/worldwritable state=directory mode=0777 + +#- name: create world writable file +# win_copy: dest=/tmp/worldwritable/file.txt content="bar" mode=0666 + +#- name: overwrite the file as user nobody +# win_copy: dest=/tmp/worldwritable/file.txt content="baz" +# sudo: yes +# sudo_user: nobody +# register: copy_result7 + +#- name: assert the file was overwritten +# assert: +# that: +# - "copy_result7.changed" +# - "copy_result7.dest == '/tmp/worldwritable/file.txt'" +# - "copy_result7.checksum == '73feffa4b7f6bb68e44cf984c85f6e88'" + +#- name: clean up +# win_file: dest=/tmp/worldwritable state=absent + +# test overwritting a link using "follow=yes" so that the link +# is preserved and the link target is updated + +#- name: create a test file to symlink to +# win_copy: dest={{win_output_dir}}/follow_test content="this is the follow test file\n" +# +#- name: create a symlink to the test file +# win_file: path={{win_output_dir}}/follow_link src='./follow_test' state=link +# +#- name: update the test file using follow=True to preserve the link +# win_copy: dest={{win_output_dir}}/follow_link content="this is the new content\n" follow=yes +# register: replace_follow_result + +#- name: stat the link path +# win_stat: path={{win_output_dir}}/follow_link +# register: stat_link_result +# +#- name: assert that the link is still a link +# assert: +# that: +# - stat_link_result.stat.islnk +# +#- name: get the md5 of the link target +# shell: checksum {{win_output_dir}}/follow_test | cut -f1 -sd ' ' +# register: target_file_result + +#- name: assert that the link target was updated +# assert: +# that: +# - replace_follow_result.checksum == target_file_result.stdout + +- name: clean up sub + win_file: path={{win_output_dir}}/sub state=absent + +- name: clean up foo.txt + win_file: path={{win_output_dir}}/foo.txt state=absent + + diff --git a/test/integration/roles/test_win_feature/defaults/main.yml b/test/integration/roles/test_win_feature/defaults/main.yml new file mode 100644 index 0000000000..e1833cd8a8 --- /dev/null +++ b/test/integration/roles/test_win_feature/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +# Feature not normally installed by default. +test_win_feature_name: Telnet-Client diff --git a/test/integration/roles/test_win_feature/tasks/main.yml b/test/integration/roles/test_win_feature/tasks/main.yml new file mode 100644 index 0000000000..a49622c232 --- /dev/null +++ b/test/integration/roles/test_win_feature/tasks/main.yml @@ -0,0 +1,131 @@ +# test code for the win_feature module +# (c) 2014, Chris Church <chris@ninemoreminutes.com> + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + + +- name: start with feature absent + win_feature: + name: "{{ test_win_feature_name }}" + state: absent + +- name: install feature + win_feature: + name: "{{ test_win_feature_name }}" + state: present + restart: no + include_sub_features: yes + include_management_tools: yes + register: win_feature_install_result + +- name: check result of installing feature + assert: + that: + - "win_feature_install_result|changed" + - "win_feature_install_result.success" + - "win_feature_install_result.exitcode == 'Success'" + - "not win_feature_install_result.restart_needed" + - "win_feature_install_result.feature_result|length == 1" + - "win_feature_install_result.feature_result[0].id" + - "win_feature_install_result.feature_result[0].display_name" + - "win_feature_install_result.feature_result[0].message is defined" + - "win_feature_install_result.feature_result[0].restart_needed is defined" + - "win_feature_install_result.feature_result[0].skip_reason" + - "win_feature_install_result.feature_result[0].success is defined" + +- name: install feature again + win_feature: + name: "{{ test_win_feature_name }}" + state: present + restart: no + include_sub_features: yes + include_management_tools: yes + register: win_feature_install_again_result + +- name: check result of installing feature again + assert: + that: + - "not win_feature_install_again_result|changed" + - "win_feature_install_again_result.success" + - "win_feature_install_again_result.exitcode == 'NoChangeNeeded'" + - "not win_feature_install_again_result.restart_needed" + - "win_feature_install_again_result.feature_result == []" + +- name: remove feature + win_feature: + name: "{{ test_win_feature_name }}" + state: absent + register: win_feature_remove_result + +- name: check result of removing feature + assert: + that: + - "win_feature_remove_result|changed" + - "win_feature_remove_result.success" + - "win_feature_remove_result.exitcode == 'Success'" + - "not win_feature_remove_result.restart_needed" + - "win_feature_remove_result.feature_result|length == 1" + - "win_feature_remove_result.feature_result[0].id" + - "win_feature_remove_result.feature_result[0].display_name" + - "win_feature_remove_result.feature_result[0].message is defined" + - "win_feature_remove_result.feature_result[0].restart_needed is defined" + - "win_feature_remove_result.feature_result[0].skip_reason" + - "win_feature_remove_result.feature_result[0].success is defined" + +- name: remove feature again + win_feature: + name: "{{ test_win_feature_name }}" + state: absent + register: win_feature_remove_again_result + +- name: check result of removing feature again + assert: + that: + - "not win_feature_remove_again_result|changed" + - "win_feature_remove_again_result.success" + - "win_feature_remove_again_result.exitcode == 'NoChangeNeeded'" + - "not win_feature_remove_again_result.restart_needed" + - "win_feature_remove_again_result.feature_result == []" + +- name: try to install an invalid feature name + win_feature: + name: "Microsoft-Bob" + state: present + register: win_feature_install_invalid_result + ignore_errors: true + +- name: check result of installing invalid feature name + assert: + that: + - "win_feature_install_invalid_result|failed" + - "not win_feature_install_invalid_result|changed" + - "win_feature_install_invalid_result.msg" + - "win_feature_install_invalid_result.exitcode == 'InvalidArgs'" + +- name: try to remove an invalid feature name + win_feature: + name: "Microsoft-Bob" + state: absent + register: win_feature_remove_invalid_result + ignore_errors: true + +- name: check result of removing invalid feature name + assert: + that: + - "win_feature_remove_invalid_result|failed" + - "not win_feature_remove_invalid_result|changed" + - "win_feature_remove_invalid_result.msg" + - "win_feature_remove_invalid_result.exitcode == 'InvalidArgs'" diff --git a/test/integration/roles/test_win_file/files/foo.txt b/test/integration/roles/test_win_file/files/foo.txt new file mode 100644 index 0000000000..7c6ded14ec --- /dev/null +++ b/test/integration/roles/test_win_file/files/foo.txt @@ -0,0 +1 @@ +foo.txt diff --git a/test/integration/roles/test_win_file/files/foobar/directory/fileC b/test/integration/roles/test_win_file/files/foobar/directory/fileC new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/integration/roles/test_win_file/files/foobar/directory/fileC diff --git a/test/integration/roles/test_win_file/files/foobar/directory/fileD b/test/integration/roles/test_win_file/files/foobar/directory/fileD new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/integration/roles/test_win_file/files/foobar/directory/fileD diff --git a/test/integration/roles/test_win_file/files/foobar/fileA b/test/integration/roles/test_win_file/files/foobar/fileA new file mode 100644 index 0000000000..ab47708c98 --- /dev/null +++ b/test/integration/roles/test_win_file/files/foobar/fileA @@ -0,0 +1 @@ +fileA diff --git a/test/integration/roles/test_win_file/files/foobar/fileB b/test/integration/roles/test_win_file/files/foobar/fileB new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/integration/roles/test_win_file/files/foobar/fileB diff --git a/test/integration/roles/test_win_file/meta/main.yml b/test/integration/roles/test_win_file/meta/main.yml new file mode 100644 index 0000000000..55200b3fc6 --- /dev/null +++ b/test/integration/roles/test_win_file/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_win_tests + diff --git a/test/integration/roles/test_win_file/tasks/main.yml b/test/integration/roles/test_win_file/tasks/main.yml new file mode 100644 index 0000000000..35ecfb6387 --- /dev/null +++ b/test/integration/roles/test_win_file/tasks/main.yml @@ -0,0 +1,421 @@ +# Test code for the file module. +# (c) 2014, Richard Isaacson <richard.c.isaacson@gmail.com> + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +- set_fact: output_file={{win_output_dir}}\\foo.txt + +- name: prep with a basic win copy + win_copy: src=foo.txt dest={{output_file}} + +- name: verify that we are checking a file and it is present + win_file: path={{output_file}} state=file + register: file_result + +- name: verify that the file was marked as changed + assert: + that: + - "file_result.changed == false" +# - "file_result.state == 'file'" + +- name: verify that we are checking an absent file + win_file: path={{win_output_dir}}\bar.txt state=absent + register: file2_result + +- name: verify that the file was marked as changed + assert: + that: + - "file2_result.changed == false" +# - "file2_result.state == 'absent'" + +- name: verify we can touch a file + win_file: path={{win_output_dir}}\baz.txt state=touch + register: file3_result + +- name: verify that the file was marked as changed + assert: + that: + - "file3_result.changed == true" +# - "file3_result.state == 'file'" +# - "file3_result.mode == '0644'" + +#- name: change file mode +# win_file: path={{win_output_dir}}/baz.txt mode=0600 +# register: file4_result + +#- name: verify that the file was marked as changed +# assert: +# that: +# - "file4_result.changed == true" +# - "file4_result.mode == '0600'" +# +#- name: change ownership and group +# win_file: path={{win_output_dir}}/baz.txt owner=1234 group=1234 +# +#- name: setup a tmp-like directory for ownership test +# win_file: path=/tmp/worldwritable mode=1777 state=directory + +#- name: Ask to create a file without enough perms to change ownership +# win_file: path=/tmp/worldwritable/baz.txt state=touch owner=root +# sudo: yes +# sudo_user: nobody +# register: chown_result +# ignore_errors: True + +#- name: Ask whether the new file exists +# win_stat: path=/tmp/worldwritable/baz.txt +# register: file_exists_result + +#- name: Verify that the file doesn't exist on failure +# assert: +# that: +# - "chown_result.failed == True" +# - "file_exists_result.stat.exists == False" +# +- name: clean up + win_file: path=/tmp/worldwritable state=absent + +#- name: create soft link to file +# win_file: src={{output_file}} dest={{win_output_dir}}/soft.txt state=link +# register: file5_result + +#- name: verify that the file was marked as changed +# assert: +# that: +# - "file5_result.changed == true" +# +#- name: create hard link to file +# win_file: src={{output_file}} dest={{win_output_dir}}/hard.txt state=hard +# register: file6_result +# +#- name: verify that the file was marked as changed +# assert: +# that: +# - "file6_result.changed == true" +# +- name: create a directory + win_file: path={{win_output_dir}}\foobar state=directory + register: file7_result + +- debug: var=file7_result + +- name: verify that the file was marked as changed + assert: + that: + - "file7_result.changed == true" +# - "file7_result.state == 'directory'" + +# windows and selinux unlikely to ever mix, removing these tests: +#- name: determine if selinux is installed +# shell: which getenforce || exit 0 +# register: selinux_installed + +#- name: determine if selinux is enabled +# shell: getenforce +# register: selinux_enabled +# when: selinux_installed.stdout != "" +# ignore_errors: true + +#- name: decide to include or not include selinux tests +# include: selinux_tests.yml +# when: selinux_installed.stdout != "" and selinux_enabled.stdout != "Disabled" + +- name: remote directory foobar + win_file: path={{win_output_dir}}\foobar state=absent + +- name: remove file foo.txt + win_file: path={{win_output_dir}}\foo.txt state=absent + +- name: remove file bar.txt + win_file: path={{win_output_dir}}\foo.txt state=absent + +- name: remove file baz.txt + win_file: path={{win_output_dir}}\foo.txt state=absent + +- name: win copy directory structure over + win_copy: src=foobar dest={{win_output_dir}} + +- name: remove directory foobar + win_file: path={{win_output_dir}}\foobar state=absent + register: file14_result + +- debug: var=file14_result + +- name: verify that the directory was removed + assert: + that: + - 'file14_result.changed == true' +# - 'file14_result.state == "absent"' + +- name: create a test sub-directory + win_file: dest={{win_output_dir}}/sub1 state=directory + register: file15_result + +- name: verify that the new directory was created + assert: + that: + - 'file15_result.changed == true' +# - 'file15_result.state == "directory"' + +- name: create test files in the sub-directory + win_file: dest={{win_output_dir}}/sub1/{{item}} state=touch + with_items: + - file1 + - file2 + - file3 + register: file16_result + +- name: verify the files were created + assert: + that: + - 'item.changed == true' +# - 'item.state == "file"' + with_items: file16_result.results + +#- name: try to force the sub-directory to a link +# win_file: src={{win_output_dir}}/testing dest={{win_output_dir}}/sub1 state=link force=yes +# register: file17_result +# ignore_errors: true + +#- name: verify the directory was not replaced with a link +# assert: +# that: +# - 'file17_result.failed == true' +# - 'file17_result.state == "directory"' + +#- name: create soft link to directory using absolute path +# win_file: src=/ dest={{win_output_dir}}/root state=link +# register: file18_result +# +#- name: verify that the result was marked as changed +# assert: +# that: +# - "file18_result.changed == true" +# +- name: create another test sub-directory + win_file: dest={{win_output_dir}}/sub2 state=directory + register: file19_result + +- name: verify that the new directory was created + assert: + that: + - 'file19_result.changed == true' +# - 'file19_result.state == "directory"' + +#- name: create soft link to relative file +# win_file: src=../sub1/file1 dest={{win_output_dir}}/sub2/link1 state=link +# register: file20_result +# +#- name: verify that the result was marked as changed +# assert: +# that: +# - "file20_result.changed == true" + +#- name: create soft link to relative directory +# win_file: src=sub1 dest={{win_output_dir}}/sub1-link state=link +# register: file21_result +# +#- name: verify that the result was marked as changed +# assert: +# that: +# - "file21_result.changed == true" +# +#- name: test file creation with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=u=rwx,g=rwx,o=rwx +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0777' + +#- name: modify symbolic mode for all +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=a=r +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0444' + +#- name: modify symbolic mode for owner +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=u+w +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0644' + +#- name: modify symbolic mode for group +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=g+w +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0664' +# +#- name: modify symbolic mode for world +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=o+w +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0666' +# +#- name: modify symbolic mode for owner +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=u+x +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0766' +## +#- name: modify symbolic mode for group +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=g+x +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0776' +# +#- name: modify symbolic mode for world +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=o+x +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0777' + +#- name: remove symbolic mode for world +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=o-wx +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0774' +# +#- name: remove symbolic mode for group +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=g-wx +# register: result +# +#- name: assert file mode +### assert: +# that: +# - result.mode == '0744' + +#- name: remove symbolic mode for owner +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=u-wx +# register: result + +#- name: assert file mode +# assert: +# that: +# - result.mode == '0444' +# +#- name: set sticky bit with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=o+t +# register: result + +#- name: assert file mode +# assert: +# that: +# - result.mode == '01444' +# +#- name: remove sticky bit with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=o-t +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0444' + +#- name: add setgid with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=g+s +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '02444' +# +#- name: remove setgid with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=g-s +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0444' + +#- name: add setuid with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=u+s +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '04444' + +#- name: remove setuid with symbolic mode +# win_file: dest={{win_output_dir}}/test_symbolic state=touch mode=u-s +# register: result +# +#- name: assert file mode +# assert: +# that: +# - result.mode == '0444' + +# test the file module using follow=yes, so that the target of a +# symlink is modified, rather than the link itself + +#- name: create a test file +# win_copy: dest={{win_output_dir}}\test_follow content="this is a test file\n" mode=0666 + +#- name: create a symlink to the test file +# win_file: path={{win_output_dir}}\test_follow_link src="./test_follow" state=link +# +#- name: modify the permissions on the link using follow=yes +# win_file: path={{win_output_dir}}\test_follow_link mode=0644 follow=yes +# register: result + +#- name: assert that the chmod worked +# assert: +# that: +# - result.changed +# +#- name: stat the link target +# win_stat: path={{win_output_dir}}/test_follow +# register: result +# +#- name: assert that the link target was modified correctly +# assert: +# that: +## - result.stat.mode == '0644' + +- name: clean up sub1 + win_file: path={{win_output_dir}}/sub1 state=absent + +- name: clean up sub2 + win_file: path={{win_output_dir}}/sub2 state=absent + diff --git a/test/integration/roles/test_win_template/files/foo.txt b/test/integration/roles/test_win_template/files/foo.txt new file mode 100644 index 0000000000..3e96db9b3e --- /dev/null +++ b/test/integration/roles/test_win_template/files/foo.txt @@ -0,0 +1 @@ +templated_var_loaded diff --git a/test/integration/roles/test_win_template/meta/main.yml b/test/integration/roles/test_win_template/meta/main.yml new file mode 100644 index 0000000000..55200b3fc6 --- /dev/null +++ b/test/integration/roles/test_win_template/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_win_tests + diff --git a/test/integration/roles/test_win_template/tasks/main.yml b/test/integration/roles/test_win_template/tasks/main.yml new file mode 100644 index 0000000000..9c2ea920ff --- /dev/null +++ b/test/integration/roles/test_win_template/tasks/main.yml @@ -0,0 +1,103 @@ +# test code for the template module +# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com> + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +- name: fill in a basic template +# win_template: src=foo.j2 dest={{win_output_dir}}/foo.templated mode=0644 + win_template: src=foo.j2 dest={{win_output_dir}}/foo.templated + register: template_result + +- assert: + that: + - "'changed' in template_result" +# - "'dest' in template_result" +# - "'group' in template_result" +# - "'gid' in template_result" +# - "'checksum' in template_result" +# - "'owner' in template_result" +# - "'size' in template_result" +# - "'src' in template_result" +# - "'state' in template_result" +# - "'uid' in template_result" + +- name: verify that the file was marked as changed + assert: + that: + - "template_result.changed == true" + +# VERIFY CONTENTS + +- name: copy known good into place + win_copy: src=foo.txt dest={{win_output_dir}}\foo.txt + +- name: compare templated file to known good + raw: fc.exe {{win_output_dir}}\foo.templated {{win_output_dir}}\foo.txt + register: diff_result + +- debug: var=diff_result + +- name: verify templated file matches known good + assert: + that: +# - 'diff_result.stdout == ""' + - 'diff_result.stdout_lines[1] == "FC: no differences encountered"' + - "diff_result.rc == 0" + +# VERIFY MODE +# can't set file mode on windows so commenting this test out +#- name: set file mode +# win_file: path={{win_output_dir}}/foo.templated mode=0644 +# register: file_result + +#- name: ensure file mode did not change +# assert: +# that: +# - "file_result.changed != True" + +# commenting out all the following tests as expanduser and file modes not windows concepts. + +# VERIFY dest as a directory does not break file attributes +# Note: expanduser is needed to go down the particular codepath that was broken before +#- name: setup directory for test +# win_file: state=directory dest={{win_output_dir | expanduser}}/template-dir mode=0755 owner=nobody group=root + +#- name: set file mode when the destination is a directory +# win_template: src=foo.j2 dest={{win_output_dir | expanduser}}/template-dir/ mode=0600 owner=root group=root + +#- name: set file mode when the destination is a directory +# win_template: src=foo.j2 dest={{win_output_dir | expanduser}}/template-dir/ mode=0600 owner=root group=root +# register: file_result +# +#- name: check that the file has the correct attributes +# win_stat: path={{win_output_dir | expanduser}}/template-dir/foo.j2 +# register: file_attrs +# +#- assert: +# that: +# - "file_attrs.stat.uid == 0" +# - "file_attrs.stat.pw_name == 'root'" +# - "file_attrs.stat.mode == '0600'" +# +#- name: check that the containing directory did not change attributes +# win_stat: path={{win_output_dir | expanduser}}/template-dir/ +# register: dir_attrs +# +#- assert: +# that: +# - "dir_attrs.stat.uid != 0" +# - "dir_attrs.stat.pw_name == 'nobody'" +# - "dir_attrs.stat.mode == '0755'" diff --git a/test/integration/roles/test_win_template/templates/foo.j2 b/test/integration/roles/test_win_template/templates/foo.j2 new file mode 100644 index 0000000000..55aab8f1ea --- /dev/null +++ b/test/integration/roles/test_win_template/templates/foo.j2 @@ -0,0 +1 @@ +{{ templated_var }} diff --git a/test/integration/roles/test_win_template/vars/main.yml b/test/integration/roles/test_win_template/vars/main.yml new file mode 100644 index 0000000000..1e8f64ccf4 --- /dev/null +++ b/test/integration/roles/test_win_template/vars/main.yml @@ -0,0 +1 @@ +templated_var: templated_var_loaded diff --git a/test/integration/test_winrm.yml b/test/integration/test_winrm.yml index c05a130831..e2a282e061 100644 --- a/test/integration/test_winrm.yml +++ b/test/integration/test_winrm.yml @@ -29,3 +29,7 @@ - { role: test_win_get_url, tags: test_win_get_url } - { role: test_win_msi, tags: test_win_msi } - { role: test_win_service, tags: test_win_service } + - { role: test_win_feature, tags: test_win_feature } + - { role: test_win_file, tags: test_win_file } + - { role: test_win_copy, tags: test_win_copy } + - { role: test_win_template, tags: test_win_template } diff --git a/test/integration/unicode.yml b/test/integration/unicode.yml index 60fb14214b..4e7fe635f4 100644 --- a/test/integration/unicode.yml +++ b/test/integration/unicode.yml @@ -35,6 +35,10 @@ host_id: '{{item}}' with_sequence: start=1 end={{num_hosts}} format=%d + - name: 'A task with unicode extra vars' + debug: var=extra_var + + - name: 'A play for hosts in group: ĪīĬÄ' hosts: 'ĪīĬÄ' gather_facts: true diff --git a/test/integration/vars_file.yml b/test/integration/vars_file.yml index bd162327d2..c43bf81866 100644 --- a/test/integration/vars_file.yml +++ b/test/integration/vars_file.yml @@ -2,4 +2,11 @@ # in general define test data in the individual role: # roles/role_name/vars/main.yml +foo: "Hello" +things1: + - 1 + - 2 +things2: + - "{{ foo }}" + - "{{ foob }}" vars_file_var: 321 diff --git a/test/units/TestFilters.py b/test/units/TestFilters.py index 7d921a7e37..d15147b098 100644 --- a/test/units/TestFilters.py +++ b/test/units/TestFilters.py @@ -131,6 +131,11 @@ class TestFilters(unittest.TestCase): 'a\\1') assert a == 'ansible' + def test_to_uuid(self): + a = ansible.runner.filter_plugins.core.to_uuid('example.com') + + assert a == 'ae780c3a-a3ab-53c2-bfb4-098da300b3fe' + #def test_filters(self): # this test is pretty low level using a playbook, hence I am disabling it for now -- MPD. diff --git a/v2/ansible/__init__.py b/v2/ansible/__init__.py index ae8ccff595..26869775ea 100644 --- a/v2/ansible/__init__.py +++ b/v2/ansible/__init__.py @@ -18,3 +18,5 @@ # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type + +__version__ = '1.v2' diff --git a/v2/ansible/constants.py b/v2/ansible/constants.py index e74720b8a6..1c2bc092b2 100644 --- a/v2/ansible/constants.py +++ b/v2/ansible/constants.py @@ -104,7 +104,8 @@ YAML_FILENAME_EXTENSIONS = [ "", ".yml", ".yaml", ".json" ] DEFAULTS='defaults' # configurable things -DEFAULT_HOST_LIST = shell_expand_path(get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts')) +DEFAULT_DEBUG = get_config(p, DEFAULTS, 'debug', 'ANSIBLE_DEBUG', False, boolean=True) +DEFAULT_HOST_LIST = shell_expand_path(get_config(p, DEFAULTS, 'inventory', 'ANSIBLE_INVENTORY', get_config(p, DEFAULTS,'hostfile','ANSIBLE_HOSTS', '/etc/ansible/hosts'))) DEFAULT_MODULE_PATH = get_config(p, DEFAULTS, 'library', 'ANSIBLE_LIBRARY', None) DEFAULT_ROLES_PATH = shell_expand_path(get_config(p, DEFAULTS, 'roles_path', 'ANSIBLE_ROLES_PATH', '/etc/ansible/roles')) DEFAULT_REMOTE_TMP = get_config(p, DEFAULTS, 'remote_tmp', 'ANSIBLE_REMOTE_TEMP', '$HOME/.ansible/tmp') diff --git a/v2/ansible/errors/__init__.py b/v2/ansible/errors/__init__.py index 2813507df2..7effe41df7 100644 --- a/v2/ansible/errors/__init__.py +++ b/v2/ansible/errors/__init__.py @@ -21,7 +21,7 @@ __metaclass__ = type import os -from ansible.parsing.yaml.strings import * +from ansible.errors.yaml_strings import * class AnsibleError(Exception): ''' @@ -45,12 +45,12 @@ class AnsibleError(Exception): self._obj = obj self._show_content = show_content - if isinstance(self._obj, AnsibleBaseYAMLObject): + if obj and isinstance(obj, AnsibleBaseYAMLObject): extended_error = self._get_extended_error() if extended_error: - self.message = '%s\n\n%s' % (message, extended_error) + self.message = 'ERROR! %s\n\n%s' % (message, extended_error) else: - self.message = message + self.message = 'ERROR! %s' % message def __str__(self): return self.message @@ -98,8 +98,9 @@ class AnsibleError(Exception): (target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1) if target_line: stripped_line = target_line.replace(" ","") - arrow_line = (" " * (col_number-1)) + "^" - error_message += "%s\n%s\n%s\n" % (prev_line.rstrip(), target_line.rstrip(), arrow_line) + arrow_line = (" " * (col_number-1)) + "^ here" + #header_line = ("=" * 73) + error_message += "\nThe offending line appears to be:\n\n%s\n%s\n%s\n" % (prev_line.rstrip(), target_line.rstrip(), arrow_line) # common error/remediation checking here: # check for unquoted vars starting lines @@ -158,3 +159,11 @@ class AnsibleModuleError(AnsibleRuntimeError): class AnsibleConnectionFailure(AnsibleRuntimeError): ''' the transport / connection_plugin had a fatal error ''' pass + +class AnsibleFilterError(AnsibleRuntimeError): + ''' a templating failure ''' + pass + +class AnsibleUndefinedVariable(AnsibleRuntimeError): + ''' a templating failure ''' + pass diff --git a/v2/ansible/parsing/yaml/strings.py b/v2/ansible/errors/yaml_strings.py index dcd6ffd79f..dcd6ffd79f 100644 --- a/v2/ansible/parsing/yaml/strings.py +++ b/v2/ansible/errors/yaml_strings.py diff --git a/v2/ansible/executor/connection_info.py b/v2/ansible/executor/connection_info.py new file mode 100644 index 0000000000..5c48ff0089 --- /dev/null +++ b/v2/ansible/executor/connection_info.py @@ -0,0 +1,217 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import pipes +import random + +from ansible import constants as C +from ansible.template import Templar +from ansible.utils.boolean import boolean + + +__all__ = ['ConnectionInformation'] + + +class ConnectionInformation: + + ''' + This class is used to consolidate the connection information for + hosts in a play and child tasks, where the task may override some + connection/authentication information. + ''' + + def __init__(self, play=None, options=None): + # FIXME: implement the new methodology here for supporting + # various different auth escalation methods (becomes, etc.) + + self.connection = C.DEFAULT_TRANSPORT + self.remote_user = 'root' + self.password = '' + self.port = 22 + self.su = False + self.su_user = '' + self.su_pass = '' + self.sudo = False + self.sudo_user = '' + self.sudo_pass = '' + self.verbosity = 0 + self.only_tags = set() + self.skip_tags = set() + + self.no_log = False + self.check_mode = False + + if play: + self.set_play(play) + + if options: + self.set_options(options) + + def set_play(self, play): + ''' + Configures this connection information instance with data from + the play class. + ''' + + if play.connection: + self.connection = play.connection + + self.remote_user = play.remote_user + self.password = '' + self.port = int(play.port) if play.port else 22 + self.su = play.su + self.su_user = play.su_user + self.su_pass = play.su_pass + self.sudo = play.sudo + self.sudo_user = play.sudo_user + self.sudo_pass = play.sudo_pass + + # non connection related + self.no_log = play.no_log + self.environment = play.environment + + def set_options(self, options): + ''' + Configures this connection information instance with data from + options specified by the user on the command line. These have a + higher precedence than those set on the play or host. + ''' + + # FIXME: set other values from options here? + + self.verbosity = options.verbosity + if options.connection: + self.connection = options.connection + + if options.check: + self.check_mode = boolean(options.check) + + # get the tag info from options, converting a comma-separated list + # of values into a proper list if need be. We check to see if the + # options have the attribute, as it is not always added via the CLI + if hasattr(options, 'tags'): + if isinstance(options.tags, list): + self.only_tags.update(options.tags) + elif isinstance(options.tags, basestring): + self.only_tags.update(options.tags.split(',')) + + if len(self.only_tags) == 0: + self.only_tags = set(['all']) + + if hasattr(options, 'skip_tags'): + if isinstance(options.skip_tags, list): + self.skip_tags.update(options.skip_tags) + elif isinstance(options.skip_tags, basestring): + self.skip_tags.update(options.skip_tags.split(',')) + + def copy(self, ci): + ''' + Copies the connection info from another connection info object, used + when merging in data from task overrides. + ''' + + #self.connection = ci.connection + #self.remote_user = ci.remote_user + #self.password = ci.password + #self.port = ci.port + #self.su = ci.su + #self.su_user = ci.su_user + #self.su_pass = ci.su_pass + #self.sudo = ci.sudo + #self.sudo_user = ci.sudo_user + #self.sudo_pass = ci.sudo_pass + #self.verbosity = ci.verbosity + + # other + #self.no_log = ci.no_log + #self.environment = ci.environment + + # requested tags + #self.only_tags = ci.only_tags.copy() + #self.skip_tags = ci.skip_tags.copy() + + for field in self._get_fields(): + value = getattr(ci, field, None) + if isinstance(value, dict): + setattr(self, field, value.copy()) + elif isinstance(value, set): + setattr(self, field, value.copy()) + elif isinstance(value, list): + setattr(self, field, value[:]) + else: + setattr(self, field, value) + + def set_task_override(self, task): + ''' + Sets attributes from the task if they are set, which will override + those from the play. + ''' + + new_info = ConnectionInformation() + new_info.copy(self) + + for attr in ('connection', 'remote_user', 'su', 'su_user', 'su_pass', 'sudo', 'sudo_user', 'sudo_pass', 'environment', 'no_log'): + if hasattr(task, attr): + attr_val = getattr(task, attr) + if attr_val: + setattr(new_info, attr, attr_val) + + return new_info + + def make_sudo_cmd(self, sudo_exe, executable, cmd): + """ + Helper function for wrapping commands with sudo. + + Rather than detect if sudo wants a password this time, -k makes + sudo always ask for a password if one is required. Passing a quoted + compound command to sudo (or sudo -s) directly doesn't work, so we + shellquote it with pipes.quote() and pass the quoted string to the + user's shell. We loop reading output until we see the randomly- + generated sudo prompt set with the -p option. + """ + + randbits = ''.join(chr(random.randint(ord('a'), ord('z'))) for x in xrange(32)) + prompt = '[sudo via ansible, key=%s] password: ' % randbits + success_key = 'SUDO-SUCCESS-%s' % randbits + + sudocmd = '%s -k && %s %s -S -p "%s" -u %s %s -c %s' % ( + sudo_exe, sudo_exe, C.DEFAULT_SUDO_FLAGS, prompt, + self.sudo_user, executable or '$SHELL', + pipes.quote('echo %s; %s' % (success_key, cmd)) + ) + + # FIXME: old code, can probably be removed as it's been commented out for a while + #return ('/bin/sh -c ' + pipes.quote(sudocmd), prompt, success_key) + return (sudocmd, prompt, success_key) + + def _get_fields(self): + return [i for i in self.__dict__.keys() if i[:1] != '_'] + + def post_validate(self, variables, loader): + ''' + Finalizes templated values which may be set on this objects fields. + ''' + + templar = Templar(loader=loader, variables=variables) + for field in self._get_fields(): + value = templar.template(getattr(self, field)) + setattr(self, field, value) + diff --git a/v2/ansible/executor/manager.py b/v2/ansible/executor/manager.py new file mode 100644 index 0000000000..33a76e143b --- /dev/null +++ b/v2/ansible/executor/manager.py @@ -0,0 +1,66 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from multiprocessing.managers import SyncManager, BaseProxy +from ansible.playbook.handler import Handler +from ansible.playbook.task import Task +from ansible.playbook.play import Play +from ansible.errors import AnsibleError + +__all__ = ['AnsibleManager'] + + +class VariableManagerWrapper: + ''' + This class simply acts as a wrapper around the VariableManager class, + since manager proxies expect a new object to be returned rather than + any existing one. Using this wrapper, a shared proxy can be created + and an existing VariableManager class assigned to it, which can then + be accessed through the exposed proxy methods. + ''' + + def __init__(self): + self._vm = None + + def get_vars(self, loader, play=None, host=None, task=None): + return self._vm.get_vars(loader=loader, play=play, host=host, task=task) + + def set_variable_manager(self, vm): + self._vm = vm + + def set_host_variable(self, host, varname, value): + self._vm.set_host_variable(host, varname, value) + + def set_host_facts(self, host, facts): + self._vm.set_host_facts(host, facts) + +class AnsibleManager(SyncManager): + ''' + This is our custom manager class, which exists only so we may register + the new proxy below + ''' + pass + +AnsibleManager.register( + typeid='VariableManagerWrapper', + callable=VariableManagerWrapper, +) + diff --git a/v2/ansible/executor/module_common.py b/v2/ansible/executor/module_common.py new file mode 100644 index 0000000000..ade1a3b064 --- /dev/null +++ b/v2/ansible/executor/module_common.py @@ -0,0 +1,184 @@ +# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# from python and deps +from cStringIO import StringIO +import inspect +import json +import os +import shlex + +# from Ansible +from ansible import __version__ +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.parsing.utils.jsonify import jsonify + +REPLACER = "#<<INCLUDE_ANSIBLE_MODULE_COMMON>>" +REPLACER_ARGS = "\"<<INCLUDE_ANSIBLE_MODULE_ARGS>>\"" +REPLACER_COMPLEX = "\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\"" +REPLACER_WINDOWS = "# POWERSHELL_COMMON" +REPLACER_VERSION = "\"<<ANSIBLE_VERSION>>\"" + +class ModuleReplacer(object): + + """ + The Replacer is used to insert chunks of code into modules before + transfer. Rather than doing classical python imports, this allows for more + efficient transfer in a no-bootstrapping scenario by not moving extra files + over the wire, and also takes care of embedding arguments in the transferred + modules. + + This version is done in such a way that local imports can still be + used in the module code, so IDEs don't have to be aware of what is going on. + + Example: + + from ansible.module_utils.basic import * + + ... will result in the insertion basic.py into the module + + from the module_utils/ directory in the source tree. + + All modules are required to import at least basic, though there will also + be other snippets. + + # POWERSHELL_COMMON + + Also results in the inclusion of the common code in powershell.ps1 + + """ + + # ****************************************************************************** + + def __init__(self, strip_comments=False): + # FIXME: these members need to be prefixed with '_' and the rest of the file fixed + this_file = inspect.getfile(inspect.currentframe()) + # we've moved the module_common relative to the snippets, so fix the path + self.snippet_path = os.path.join(os.path.dirname(this_file), '..', 'module_utils') + self.strip_comments = strip_comments + + # ****************************************************************************** + + + def slurp(self, path): + if not os.path.exists(path): + raise AnsibleError("imported module support code does not exist at %s" % path) + fd = open(path) + data = fd.read() + fd.close() + return data + + def _find_snippet_imports(self, module_data, module_path): + """ + Given the source of the module, convert it to a Jinja2 template to insert + module code and return whether it's a new or old style module. + """ + + module_style = 'old' + if REPLACER in module_data: + module_style = 'new' + elif 'from ansible.module_utils.' in module_data: + module_style = 'new' + elif 'WANT_JSON' in module_data: + module_style = 'non_native_want_json' + + output = StringIO() + lines = module_data.split('\n') + snippet_names = [] + + for line in lines: + + if REPLACER in line: + output.write(self.slurp(os.path.join(self.snippet_path, "basic.py"))) + snippet_names.append('basic') + if REPLACER_WINDOWS in line: + ps_data = self.slurp(os.path.join(self.snippet_path, "powershell.ps1")) + output.write(ps_data) + snippet_names.append('powershell') + elif line.startswith('from ansible.module_utils.'): + tokens=line.split(".") + import_error = False + if len(tokens) != 3: + import_error = True + if " import *" not in line: + import_error = True + if import_error: + raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.basic import *'" % module_path) + snippet_name = tokens[2].split()[0] + snippet_names.append(snippet_name) + output.write(self.slurp(os.path.join(self.snippet_path, snippet_name + ".py"))) + else: + if self.strip_comments and line.startswith("#") or line == '': + pass + output.write(line) + output.write("\n") + + if not module_path.endswith(".ps1"): + # Unixy modules + if len(snippet_names) > 0 and not 'basic' in snippet_names: + raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path) + else: + # Windows modules + if len(snippet_names) > 0 and not 'powershell' in snippet_names: + raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path) + + return (output.getvalue(), module_style) + + # ****************************************************************************** + + def modify_module(self, module_path, module_args): + + with open(module_path) as f: + + # read in the module source + module_data = f.read() + + (module_data, module_style) = self._find_snippet_imports(module_data, module_path) + + #module_args_json = jsonify(module_args) + module_args_json = json.dumps(module_args) + encoded_args = repr(module_args_json.encode('utf-8')) + + # these strings should be part of the 'basic' snippet which is required to be included + module_data = module_data.replace(REPLACER_VERSION, repr(__version__)) + module_data = module_data.replace(REPLACER_COMPLEX, encoded_args) + + # FIXME: we're not passing around an inject dictionary anymore, so + # this needs to be fixed with whatever method we use for vars + # like this moving forward + #if module_style == 'new': + # facility = C.DEFAULT_SYSLOG_FACILITY + # if 'ansible_syslog_facility' in inject: + # facility = inject['ansible_syslog_facility'] + # module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility) + + lines = module_data.split("\n") + shebang = None + if lines[0].startswith("#!"): + shebang = lines[0].strip() + args = shlex.split(str(shebang[2:])) + interpreter = args[0] + interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter) + + # FIXME: more inject stuff here... + #if interpreter_config in inject: + # lines[0] = shebang = "#!%s %s" % (inject[interpreter_config], " ".join(args[1:])) + # module_data = "\n".join(lines) + + return (module_data, module_style, shebang) + diff --git a/v2/ansible/executor/play_iterator.py b/v2/ansible/executor/play_iterator.py new file mode 100644 index 0000000000..f1d8914f84 --- /dev/null +++ b/v2/ansible/executor/play_iterator.py @@ -0,0 +1,269 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import * +from ansible.playbook.task import Task + +from ansible.utils.boolean import boolean + +__all__ = ['PlayIterator'] + + +# the primary running states for the play iteration +ITERATING_SETUP = 0 +ITERATING_TASKS = 1 +ITERATING_RESCUE = 2 +ITERATING_ALWAYS = 3 +ITERATING_COMPLETE = 4 + +# the failure states for the play iteration +FAILED_NONE = 0 +FAILED_SETUP = 1 +FAILED_TASKS = 2 +FAILED_RESCUE = 3 +FAILED_ALWAYS = 4 + +class PlayState: + + ''' + A helper class, which keeps track of the task iteration + state for a given playbook. This is used in the PlaybookIterator + class on a per-host basis. + ''' + + # FIXME: this class is the representation of a finite state machine, + # so we really should have a well defined state representation + # documented somewhere... + + def __init__(self, parent_iterator, host): + ''' + Create the initial state, which tracks the running state as well + as the failure state, which are used when executing block branches + (rescue/always) + ''' + + self._parent_iterator = parent_iterator + self._run_state = ITERATING_SETUP + self._failed_state = FAILED_NONE + self._task_list = parent_iterator._play.compile() + self._gather_facts = parent_iterator._play.gather_facts + self._host = host + + self._cur_block = None + self._cur_role = None + self._cur_task_pos = 0 + self._cur_rescue_pos = 0 + self._cur_always_pos = 0 + self._cur_handler_pos = 0 + + def next(self, peek=False): + ''' + Determines and returns the next available task from the playbook, + advancing through the list of plays as it goes. If peek is set to True, + the internal state is not stored. + ''' + + task = None + + # save this locally so that we can peek at the next task + # without updating the internal state of the iterator + run_state = self._run_state + failed_state = self._failed_state + cur_block = self._cur_block + cur_role = self._cur_role + cur_task_pos = self._cur_task_pos + cur_rescue_pos = self._cur_rescue_pos + cur_always_pos = self._cur_always_pos + cur_handler_pos = self._cur_handler_pos + + + while True: + if run_state == ITERATING_SETUP: + if failed_state == FAILED_SETUP: + run_state = ITERATING_COMPLETE + else: + run_state = ITERATING_TASKS + + if self._gather_facts == 'smart' and not self._host.gathered_facts or boolean(self._gather_facts): + self._host.set_gathered_facts(True) + task = Task() + # FIXME: this is not the best way to get this... + task.set_loader(self._parent_iterator._play._loader) + task.action = 'setup' + break + elif run_state == ITERATING_TASKS: + # if there is any failure state besides FAILED_NONE, we should + # change to some other running state + if failed_state != FAILED_NONE or cur_task_pos > len(self._task_list) - 1: + # if there is a block (and there always should be), start running + # the rescue portion if it exists (and if we haven't failed that + # already), or the always portion (if it exists and we didn't fail + # there too). Otherwise, we're done iterating. + if cur_block: + if failed_state != FAILED_RESCUE and cur_block.rescue: + run_state = ITERATING_RESCUE + cur_rescue_pos = 0 + elif failed_state != FAILED_ALWAYS and cur_block.always: + run_state = ITERATING_ALWAYS + cur_always_pos = 0 + else: + run_state = ITERATING_COMPLETE + else: + run_state = ITERATING_COMPLETE + else: + task = self._task_list[cur_task_pos] + if cur_block is not None and cur_block != task._block: + run_state = ITERATING_ALWAYS + continue + else: + cur_block = task._block + cur_task_pos += 1 + + # Break out of the while loop now that we have our task + break + + elif run_state == ITERATING_RESCUE: + # If we're iterating through the rescue tasks, make sure we haven't + # failed yet. If so, move on to the always block or if not get the + # next rescue task (if one exists) + if failed_state == FAILED_RESCUE or cur_block.rescue is None or cur_rescue_pos > len(cur_block.rescue) - 1: + run_state = ITERATING_ALWAYS + else: + task = cur_block.rescue[cur_rescue_pos] + cur_rescue_pos += 1 + break + + elif run_state == ITERATING_ALWAYS: + # If we're iterating through the always tasks, make sure we haven't + # failed yet. If so, we're done iterating otherwise get the next always + # task (if one exists) + if failed_state == FAILED_ALWAYS or cur_block.always is None or cur_always_pos > len(cur_block.always) - 1: + cur_block = None + if failed_state == FAILED_ALWAYS or cur_task_pos > len(self._task_list) - 1: + run_state = ITERATING_COMPLETE + else: + run_state = ITERATING_TASKS + else: + task = cur_block.always[cur_always_pos] + cur_always_pos += 1 + break + + elif run_state == ITERATING_COMPLETE: + # done iterating, return None to signify that + return None + + if task._role: + # if we had a current role, mark that role as completed + if cur_role and task._role != cur_role and not peek: + cur_role._completed = True + + cur_role = task._role + + # if the current role has not had its task run flag set, mark + # clear the completed flag so we can correctly determine if the + # role was run + if not cur_role._had_task_run and not peek: + cur_role._completed = False + + # If we're not just peeking at the next task, save the internal state + if not peek: + self._run_state = run_state + self._failed_state = failed_state + self._cur_block = cur_block + self._cur_role = cur_role + self._cur_task_pos = cur_task_pos + self._cur_rescue_pos = cur_rescue_pos + self._cur_always_pos = cur_always_pos + self._cur_handler_pos = cur_handler_pos + + return task + + def mark_failed(self): + ''' + Escalates the failed state relative to the running state. + ''' + if self._run_state == ITERATING_SETUP: + self._failed_state = FAILED_SETUP + elif self._run_state == ITERATING_TASKS: + self._failed_state = FAILED_TASKS + elif self._run_state == ITERATING_RESCUE: + self._failed_state = FAILED_RESCUE + elif self._run_state == ITERATING_ALWAYS: + self._failed_state = FAILED_ALWAYS + + +class PlayIterator: + + ''' + The main iterator class, which keeps the state of the playbook + on a per-host basis using the above PlaybookState class. + ''' + + def __init__(self, inventory, play): + self._play = play + self._inventory = inventory + self._host_entries = dict() + self._first_host = None + + # Build the per-host dictionary of playbook states, using a copy + # of the play object so we can post_validate it to ensure any templated + # fields are filled in without modifying the original object, since + # post_validate() saves the templated values. + + # FIXME: this is a hacky way of doing this, the iterator should + # instead get the loader and variable manager directly + # as args to __init__ + all_vars = inventory._variable_manager.get_vars(loader=inventory._loader, play=play) + new_play = play.copy() + new_play.post_validate(all_vars, fail_on_undefined=False) + + for host in inventory.get_hosts(new_play.hosts): + if self._first_host is None: + self._first_host = host + self._host_entries[host.get_name()] = PlayState(parent_iterator=self, host=host) + + # FIXME: remove, probably not required anymore + #def get_next_task(self, peek=False): + # ''' returns the next task for host[0] ''' + # + # first_entry = self._host_entries[self._first_host.get_name()] + # if not peek: + # for entry in self._host_entries: + # if entry != self._first_host.get_name(): + # target_entry = self._host_entries[entry] + # if target_entry._cur_task_pos == first_entry._cur_task_pos: + # target_entry.next() + # return first_entry.next(peek=peek) + + def get_next_task_for_host(self, host, peek=False): + ''' fetch the next task for the given host ''' + if host.get_name() not in self._host_entries: + raise AnsibleError("invalid host (%s) specified for playbook iteration" % host) + + return self._host_entries[host.get_name()].next(peek=peek) + + def mark_host_failed(self, host): + ''' mark the given host as failed ''' + if host.get_name() not in self._host_entries: + raise AnsibleError("invalid host (%s) specified for playbook iteration" % host) + + self._host_entries[host.get_name()].mark_failed() + diff --git a/v2/ansible/executor/playbook_executor.py b/v2/ansible/executor/playbook_executor.py index 7031e51142..88ec05b9e8 100644 --- a/v2/ansible/executor/playbook_executor.py +++ b/v2/ansible/executor/playbook_executor.py @@ -19,17 +19,113 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import signal + +from ansible import constants as C +from ansible.errors import * +from ansible.executor.task_queue_manager import TaskQueueManager +from ansible.playbook import Playbook + +from ansible.utils.debug import debug + class PlaybookExecutor: - def __init__(self, list_of_plays=[]): - # self.tqm = TaskQueueManager(forks) - assert False + ''' + This is the primary class for executing playbooks, and thus the + basis for bin/ansible-playbook operation. + ''' + + def __init__(self, playbooks, inventory, variable_manager, loader, options): + self._playbooks = playbooks + self._inventory = inventory + self._variable_manager = variable_manager + self._loader = loader + self._options = options + + self._tqm = TaskQueueManager(inventory=inventory, callback='default', variable_manager=variable_manager, loader=loader, options=options) + + def run(self): + + ''' + Run the given playbook, based on the settings in the play which + may limit the runs to serialized groups, etc. + ''' + + signal.signal(signal.SIGINT, self._cleanup) + + result = 0 + try: + for playbook_path in self._playbooks: + pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) + + # FIXME: playbook entries are just plays, so we should rename them + for play in pb.get_entries(): + self._inventory.remove_restriction() + + # Create a temporary copy of the play here, so we can run post_validate + # on it without the templating changes affecting the original object. + all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) + new_play = play.copy() + new_play.post_validate(all_vars, fail_on_undefined=False) + + for batch in self._get_serialized_batches(new_play): + if len(batch) == 0: + self._tqm._callback.playbook_on_play_start(new_play.name) + self._tqm._callback.playbook_on_no_hosts_matched() + result = 0 + break + # restrict the inventory to the hosts in the serialized batch + self._inventory.restrict_to_hosts(batch) + # and run it... + result = self._tqm.run(play=play) + if result != 0: + break + + if result != 0: + # FIXME: do something here, to signify the playbook execution failed + self._cleanup() + return result + except: + self._cleanup() + raise + + self._cleanup() + return result + + def _cleanup(self, signum=None, framenum=None): + return self._tqm.cleanup() + + def _get_serialized_batches(self, play): + ''' + Returns a list of hosts, subdivided into batches based on + the serial size specified in the play. + ''' + + # make sure we have a unique list of hosts + all_hosts = self._inventory.get_hosts(play.hosts) + + # check to see if the serial number was specified as a percentage, + # and convert it to an integer value based on the number of hosts + if isinstance(play.serial, basestring) and play.serial.endswith('%'): + serial_pct = int(play.serial.replace("%","")) + serial = int((serial_pct/100.0) * len(all_hosts)) + else: + serial = int(play.serial) + + # if the serial count was not specified or is invalid, default to + # a list of all hosts, otherwise split the list of hosts into chunks + # which are based on the serial size + if serial <= 0: + return [all_hosts] + else: + serialized_batches = [] + + while len(all_hosts) > 0: + play_hosts = [] + for x in range(serial): + if len(all_hosts) > 0: + play_hosts.append(all_hosts.pop(0)) - def run(self): - # for play in list_of_plays: - # for block in play.blocks: - # # block must know it’s playbook class and context - # tqm.enqueue(block) - # tqm.go()... - assert False + serialized_batches.append(play_hosts) + return serialized_batches diff --git a/v2/ansible/executor/playbook_iterator.py b/v2/ansible/executor/playbook_iterator.py deleted file mode 100644 index 88bec5a331..0000000000 --- a/v2/ansible/executor/playbook_iterator.py +++ /dev/null @@ -1,125 +0,0 @@ -# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see <http://www.gnu.org/licenses/>. - -# Make coding more python3-ish -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -class PlaybookState: - - ''' - A helper class, which keeps track of the task iteration - state for a given playbook. This is used in the PlaybookIterator - class on a per-host basis. - ''' - def __init__(self, parent_iterator): - self._parent_iterator = parent_iterator - self._cur_play = 0 - self._task_list = None - self._cur_task_pos = 0 - self._done = False - - def next(self, peek=False): - ''' - Determines and returns the next available task from the playbook, - advancing through the list of plays as it goes. - ''' - - task = None - - # we save these locally so that we can peek at the next task - # without updating the internal state of the iterator - cur_play = self._cur_play - task_list = self._task_list - cur_task_pos = self._cur_task_pos - - while True: - # when we hit the end of the playbook entries list, we set a flag - # and return None to indicate we're there - # FIXME: accessing the entries and parent iterator playbook members - # should be done through accessor functions - if self._done or cur_play > len(self._parent_iterator._playbook._entries) - 1: - self._done = True - return None - - # initialize the task list by calling the .compile() method - # on the play, which will call compile() for all child objects - if task_list is None: - task_list = self._parent_iterator._playbook._entries[cur_play].compile() - - # if we've hit the end of this plays task list, move on to the next - # and reset the position values for the next iteration - if cur_task_pos > len(task_list) - 1: - cur_play += 1 - task_list = None - cur_task_pos = 0 - continue - else: - # FIXME: do tag/conditional evaluation here and advance - # the task position if it should be skipped without - # returning a task - task = task_list[cur_task_pos] - cur_task_pos += 1 - - # Skip the task if it is the member of a role which has already - # been run, unless the role allows multiple executions - if task._role: - # FIXME: this should all be done via member functions - # instead of direct access to internal variables - if task._role.has_run() and not task._role._metadata._allow_duplicates: - continue - - # Break out of the while loop now that we have our task - break - - # If we're not just peeking at the next task, save the internal state - if not peek: - self._cur_play = cur_play - self._task_list = task_list - self._cur_task_pos = cur_task_pos - - return task - -class PlaybookIterator: - - ''' - The main iterator class, which keeps the state of the playbook - on a per-host basis using the above PlaybookState class. - ''' - - def __init__(self, inventory, log_manager, playbook): - self._playbook = playbook - self._log_manager = log_manager - self._host_entries = dict() - self._first_host = None - - # build the per-host dictionary of playbook states - for host in inventory.get_hosts(): - if self._first_host is None: - self._first_host = host - self._host_entries[host.get_name()] = PlaybookState(parent_iterator=self) - - def get_next_task(self, peek=False): - ''' returns the next task for host[0] ''' - return self._host_entries[self._first_host.get_name()].next(peek=peek) - - def get_next_task_for_host(self, host, peek=False): - ''' fetch the next task for the given host ''' - if host.get_name() not in self._host_entries: - raise AnsibleError("invalid host specified for playbook iteration") - - return self._host_entries[host.get_name()].next(peek=peek) diff --git a/v2/ansible/plugins/filter/__init__.py b/v2/ansible/executor/process/__init__.py index 785fc45992..785fc45992 100644 --- a/v2/ansible/plugins/filter/__init__.py +++ b/v2/ansible/executor/process/__init__.py diff --git a/v2/ansible/executor/process/result.py b/v2/ansible/executor/process/result.py new file mode 100644 index 0000000000..b9e54df9dc --- /dev/null +++ b/v2/ansible/executor/process/result.py @@ -0,0 +1,170 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import Queue +import multiprocessing +import os +import signal +import sys +import time +import traceback + +HAS_ATFORK=True +try: + from Crypto.Random import atfork +except ImportError: + HAS_ATFORK=False + +from ansible.playbook.handler import Handler +from ansible.playbook.task import Task + +from ansible.utils.debug import debug + +__all__ = ['ResultProcess'] + + +class ResultProcess(multiprocessing.Process): + ''' + The result worker thread, which reads results from the results + queue and fires off callbacks/etc. as necessary. + ''' + + def __init__(self, final_q, workers): + + # takes a task queue manager as the sole param: + self._final_q = final_q + self._workers = workers + self._cur_worker = 0 + self._terminated = False + + super(ResultProcess, self).__init__() + + def _send_result(self, result): + debug("sending result: %s" % (result,)) + self._final_q.put(result, block=False) + debug("done sending result") + + def _read_worker_result(self): + result = None + starting_point = self._cur_worker + while True: + (worker_prc, main_q, rslt_q) = self._workers[self._cur_worker] + self._cur_worker += 1 + if self._cur_worker >= len(self._workers): + self._cur_worker = 0 + + try: + if not rslt_q.empty(): + debug("worker %d has data to read" % self._cur_worker) + result = rslt_q.get(block=False) + debug("got a result from worker %d: %s" % (self._cur_worker, result)) + break + except Queue.Empty: + pass + + if self._cur_worker == starting_point: + break + + return result + + def terminate(self): + self._terminated = True + super(ResultProcess, self).terminate() + + def run(self): + ''' + The main thread execution, which reads from the results queue + indefinitely and sends callbacks/etc. when results are received. + ''' + + if HAS_ATFORK: + atfork() + + while True: + try: + result = self._read_worker_result() + if result is None: + time.sleep(0.1) + continue + + host_name = result._host.get_name() + + # send callbacks, execute other options based on the result status + # FIXME: this should all be cleaned up and probably moved to a sub-function. + # the fact that this sometimes sends a TaskResult and other times + # sends a raw dictionary back may be confusing, but the result vs. + # results implementation for tasks with loops should be cleaned up + # better than this + if result.is_unreachable(): + self._send_result(('host_unreachable', result)) + elif result.is_failed(): + self._send_result(('host_task_failed', result)) + elif result.is_skipped(): + self._send_result(('host_task_skipped', result)) + else: + # if this task is notifying a handler, do it now + if result._task.notify: + # The shared dictionary for notified handlers is a proxy, which + # does not detect when sub-objects within the proxy are modified. + # So, per the docs, we reassign the list so the proxy picks up and + # notifies all other threads + for notify in result._task.notify: + self._send_result(('notify_handler', result._host, notify)) + + if result._task.loop: + # this task had a loop, and has more than one result, so + # loop over all of them instead of a single result + result_items = result._result['results'] + else: + result_items = [ result._result ] + + for result_item in result_items: + if 'add_host' in result_item: + # this task added a new host (add_host module) + self._send_result(('add_host', result_item)) + elif 'add_group' in result_item: + # this task added a new group (group_by module) + self._send_result(('add_group', result._host, result_item)) + elif 'ansible_facts' in result_item: + # if this task is registering facts, do that now + if result._task.action in ('set_fact', 'include_vars'): + for (key, value) in result_item['ansible_facts'].iteritems(): + self._send_result(('set_host_var', result._host, key, value)) + else: + self._send_result(('set_host_facts', result._host, result_item['ansible_facts'])) + + # finally, send the ok for this task + self._send_result(('host_task_ok', result)) + + # if this task is registering a result, do it now + if result._task.register: + self._send_result(('set_host_var', result._host, result._task.register, result._result)) + + except Queue.Empty: + pass + except (KeyboardInterrupt, IOError, EOFError): + break + except: + # FIXME: we should probably send a proper callback here instead of + # simply dumping a stack trace on the screen + traceback.print_exc() + break + diff --git a/v2/ansible/executor/process/worker.py b/v2/ansible/executor/process/worker.py new file mode 100644 index 0000000000..bf5ee8c93f --- /dev/null +++ b/v2/ansible/executor/process/worker.py @@ -0,0 +1,158 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import Queue +import multiprocessing +import os +import signal +import sys +import time +import traceback + +HAS_ATFORK=True +try: + from Crypto.Random import atfork +except ImportError: + HAS_ATFORK=False + +from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.executor.task_executor import TaskExecutor +from ansible.executor.task_result import TaskResult +from ansible.playbook.handler import Handler +from ansible.playbook.task import Task + +from ansible.utils.debug import debug + +__all__ = ['ExecutorProcess'] + + +class WorkerProcess(multiprocessing.Process): + ''' + The worker thread class, which uses TaskExecutor to run tasks + read from a job queue and pushes results into a results queue + for reading later. + ''' + + def __init__(self, tqm, main_q, rslt_q, loader, new_stdin): + + # takes a task queue manager as the sole param: + self._main_q = main_q + self._rslt_q = rslt_q + self._loader = loader + + # dupe stdin, if we have one + try: + fileno = sys.stdin.fileno() + except ValueError: + fileno = None + + self._new_stdin = new_stdin + if not new_stdin and fileno is not None: + try: + self._new_stdin = os.fdopen(os.dup(fileno)) + except OSError, e: + # couldn't dupe stdin, most likely because it's + # not a valid file descriptor, so we just rely on + # using the one that was passed in + pass + + if self._new_stdin: + sys.stdin = self._new_stdin + + super(WorkerProcess, self).__init__() + + def run(self): + ''' + Called when the process is started, and loops indefinitely + until an error is encountered (typically an IOerror from the + queue pipe being disconnected). During the loop, we attempt + to pull tasks off the job queue and run them, pushing the result + onto the results queue. We also remove the host from the blocked + hosts list, to signify that they are ready for their next task. + ''' + + if HAS_ATFORK: + atfork() + + while True: + task = None + try: + if not self._main_q.empty(): + debug("there's work to be done!") + (host, task, basedir, job_vars, connection_info, module_loader) = self._main_q.get(block=False) + debug("got a task/handler to work on: %s" % task) + + # because the task queue manager starts workers (forks) before the + # playbook is loaded, set the basedir of the loader inherted by + # this fork now so that we can find files correctly + self._loader.set_basedir(basedir) + + # Serializing/deserializing tasks does not preserve the loader attribute, + # since it is passed to the worker during the forking of the process and + # would be wasteful to serialize. So we set it here on the task now, and + # the task handles updating parent/child objects as needed. + task.set_loader(self._loader) + + # apply the given task's information to the connection info, + # which may override some fields already set by the play or + # the options specified on the command line + new_connection_info = connection_info.set_task_override(task) + + # execute the task and build a TaskResult from the result + debug("running TaskExecutor() for %s/%s" % (host, task)) + executor_result = TaskExecutor(host, task, job_vars, new_connection_info, self._loader, module_loader).run() + debug("done running TaskExecutor() for %s/%s" % (host, task)) + task_result = TaskResult(host, task, executor_result) + + # put the result on the result queue + debug("sending task result") + self._rslt_q.put(task_result, block=False) + debug("done sending task result") + + else: + time.sleep(0.1) + + except Queue.Empty: + pass + except (IOError, EOFError, KeyboardInterrupt): + break + except AnsibleConnectionFailure: + try: + if task: + task_result = TaskResult(host, task, dict(unreachable=True)) + self._rslt_q.put(task_result, block=False) + except: + # FIXME: most likely an abort, catch those kinds of errors specifically + break + except Exception, e: + debug("WORKER EXCEPTION: %s" % e) + debug("WORKER EXCEPTION: %s" % traceback.format_exc()) + try: + if task: + task_result = TaskResult(host, task, dict(failed=True, exception=traceback.format_exc(), stdout='')) + self._rslt_q.put(task_result, block=False) + except: + # FIXME: most likely an abort, catch those kinds of errors specifically + break + + debug("WORKER PROCESS EXITING") + + diff --git a/v2/ansible/executor/task_executor.py b/v2/ansible/executor/task_executor.py index 878c15c489..5bd9d51842 100644 --- a/v2/ansible/executor/task_executor.py +++ b/v2/ansible/executor/task_executor.py @@ -19,14 +19,334 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.executor.connection_info import ConnectionInformation +from ansible.playbook.conditional import Conditional +from ansible.playbook.task import Task +from ansible.plugins import lookup_loader, connection_loader, action_loader +from ansible.utils.listify import listify_lookup_plugin_terms + +from ansible.utils.debug import debug + +__all__ = ['TaskExecutor'] + +import json +import time + class TaskExecutor: - def __init__(self, task, host): - pass + ''' + This is the main worker class for the executor pipeline, which + handles loading an action plugin to actually dispatch the task to + a given host. This class roughly corresponds to the old Runner() + class. + ''' + + def __init__(self, host, task, job_vars, connection_info, loader, module_loader): + self._host = host + self._task = task + self._job_vars = job_vars + self._connection_info = connection_info + self._loader = loader + self._module_loader = module_loader + + def run(self): + ''' + The main executor entrypoint, where we determine if the specified + task requires looping and either runs the task with + ''' + + debug("in run()") + + try: + # lookup plugins need to know if this task is executing from + # a role, so that it can properly find files/templates/etc. + roledir = None + if self._task._role: + roledir = self._task._role._role_path + self._job_vars['roledir'] = roledir + + items = self._get_loop_items() + if items is not None: + if len(items) > 0: + item_results = self._run_loop(items) + res = dict(results=item_results) + else: + res = dict(changed=False, skipped=True, skipped_reason='No items in the list', results=[]) + else: + debug("calling self._execute()") + res = self._execute() + debug("_execute() done") + + # make sure changed is set in the result, if it's not present + if 'changed' not in res: + res['changed'] = False + + debug("dumping result to json") + result = json.dumps(res) + debug("done dumping result, returning") + return result + except AnsibleError, e: + return dict(failed=True, msg=str(e)) + + def _get_loop_items(self): + ''' + Loads a lookup plugin to handle the with_* portion of a task (if specified), + and returns the items result. + ''' + + items = None + if self._task.loop and self._task.loop in lookup_loader: + loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, variables=self._job_vars, loader=self._loader) + items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=loop_terms, variables=self._job_vars) + + return items + + def _run_loop(self, items): + ''' + Runs the task with the loop items specified and collates the result + into an array named 'results' which is inserted into the final result + along with the item for which the loop ran. + ''' + + results = [] + + # make copies of the job vars and task so we can add the item to + # the variables and re-validate the task with the item variable + task_vars = self._job_vars.copy() + + items = self._squash_items(items, task_vars) + for item in items: + task_vars['item'] = item + + try: + tmp_task = self._task.copy() + except AnsibleParserError, e: + results.append(dict(failed=True, msg=str(e))) + continue + + # now we swap the internal task with the copy, execute, + # and swap them back so we can do the next iteration cleanly + (self._task, tmp_task) = (tmp_task, self._task) + res = self._execute(variables=task_vars) + (self._task, tmp_task) = (tmp_task, self._task) + + # FIXME: we should be sending back a callback result for each item in the loop here + + # now update the result with the item info, and append the result + # to the list of results + res['item'] = item + results.append(res) + + return results + + def _squash_items(self, items, variables): + ''' + Squash items down to a comma-separated list for certain modules which support it + (typically package management modules). + ''' + + if len(items) > 0 and self._task.action in ('apt', 'yum', 'pkgng', 'zypper'): + final_items = [] + for item in items: + variables['item'] = item + if self._task.evaluate_conditional(variables): + final_items.append(item) + return [",".join(final_items)] + else: + return items + + def _execute(self, variables=None): + ''' + The primary workhorse of the executor system, this runs the task + on the specified host (which may be the delegated_to host) and handles + the retry/until and block rescue/always execution + ''' + + if variables is None: + variables = self._job_vars + + # fields set from the play/task may be based on variables, so we have to + # do the same kind of post validation step on it here before we use it + self._connection_info.post_validate(variables=variables, loader=self._loader) + + # get the connection and the handler for this execution + self._connection = self._get_connection() + self._handler = self._get_action_handler(connection=self._connection) + + # Evaluate the conditional (if any) for this task, which we do before running + # the final task post-validation. We do this before the post validation due to + # the fact that the conditional may specify that the task be skipped due to a + # variable not being present which would otherwise cause validation to fail + if not self._task.evaluate_conditional(variables): + debug("when evaulation failed, skipping this task") + return dict(changed=False, skipped=True, skip_reason='Conditional check failed') + + # Now we do final validation on the task, which sets all fields to their final values + self._task.post_validate(variables) + + # And filter out any fields which were set to default(omit), and got the omit token value + omit_token = variables.get('omit') + if omit_token is not None: + self._task.args = dict(filter(lambda x: x[1] != omit_token, self._task.args.iteritems())) + + # Read some values from the task, so that we can modify them if need be + retries = self._task.retries + if retries <= 0: + retries = 1 + + delay = self._task.delay + if delay < 0: + delay = 1 + + # make a copy of the job vars here, in case we need to update them + # with the registered variable value later on when testing conditions + vars_copy = variables.copy() + + debug("starting attempt loop") + result = None + for attempt in range(retries): + if attempt > 0: + # FIXME: this should use the callback/message passing mechanism + print("FAILED - RETRYING: %s (%d retries left)" % (self._task, retries-attempt)) + result['attempts'] = attempt + 1 + + debug("running the handler") + result = self._handler.run(task_vars=variables) + debug("handler run complete") + + if self._task.async > 0: + # the async_wrapper module returns dumped JSON via its stdout + # response, so we parse it here and replace the result + try: + result = json.loads(result.get('stdout')) + except ValueError, e: + return dict(failed=True, msg="The async task did not return valid JSON: %s" % str(e)) + + if self._task.poll > 0: + result = self._poll_async_result(result=result) + + # update the local copy of vars with the registered value, if specified + if self._task.register: + vars_copy[self._task.register] = result + + # create a conditional object to evaluate task conditions + cond = Conditional(loader=self._loader) + + # FIXME: make sure until is mutually exclusive with changed_when/failed_when + if self._task.until: + cond.when = self._task.until + if cond.evaluate_conditional(vars_copy): + break + elif (self._task.changed_when or self._task.failed_when) and 'skipped' not in result: + if self._task.changed_when: + cond.when = [ self._task.changed_when ] + result['changed'] = cond.evaluate_conditional(vars_copy) + if self._task.failed_when: + cond.when = [ self._task.failed_when ] + failed_when_result = cond.evaluate_conditional(vars_copy) + result['failed_when_result'] = result['failed'] = failed_when_result + if failed_when_result: + break + elif 'failed' not in result and result.get('rc', 0) == 0: + # if the result is not failed, stop trying + break + + if attempt < retries - 1: + time.sleep(delay) + + debug("attempt loop complete, returning result") + return result + + def _poll_async_result(self, result): + ''' + Polls for the specified JID to be complete + ''' + + async_jid = result.get('ansible_job_id') + if async_jid is None: + return dict(failed=True, msg="No job id was returned by the async task") + + # Create a new psuedo-task to run the async_status module, and run + # that (with a sleep for "poll" seconds between each retry) until the + # async time limit is exceeded. + + async_task = Task().load(dict(action='async_status jid=%s' % async_jid)) + + # Because this is an async task, the action handler is async. However, + # we need the 'normal' action handler for the status check, so get it + # now via the action_loader + normal_handler = action_loader.get( + 'normal', + task=async_task, + connection=self._connection, + connection_info=self._connection_info, + loader=self._loader, + module_loader=self._module_loader, + ) + + time_left = self._task.async + while time_left > 0: + time.sleep(self._task.poll) + + async_result = normal_handler.run() + if int(async_result.get('finished', 0)) == 1 or 'failed' in async_result or 'skipped' in async_result: + break + + time_left -= self._task.poll + + if int(async_result.get('finished', 0)) != 1: + return dict(failed=True, msg="async task did not complete within the requested time") + else: + return async_result + + def _get_connection(self): + ''' + Reads the connection property for the host, and returns the + correct connection object from the list of connection plugins + ''' + + # FIXME: delegate_to calculation should be done here + # FIXME: calculation of connection params/auth stuff should be done here + + # FIXME: add all port/connection type munging here (accelerated mode, + # fixing up options for ssh, etc.)? and 'smart' conversion + conn_type = self._connection_info.connection + if conn_type == 'smart': + conn_type = 'ssh' + + connection = connection_loader.get(conn_type, self._host, self._connection_info) + if not connection: + raise AnsibleError("the connection plugin '%s' was not found" % conn_type) + + connection.connect() + + return connection + + def _get_action_handler(self, connection): + ''' + Returns the correct action plugin to handle the requestion task action + ''' - def run(self): - # returns TaskResult - pass + if self._task.action in action_loader: + if self._task.async != 0: + raise AnsibleError("async mode is not supported with the %s module" % module_name) + handler_name = self._task.action + elif self._task.async == 0: + handler_name = 'normal' + else: + handler_name = 'async' - + handler = action_loader.get( + handler_name, + task=self._task, + connection=connection, + connection_info=self._connection_info, + loader=self._loader, + module_loader=self._module_loader, + ) + if not handler: + raise AnsibleError("the handler '%s' was not found" % handler_name) + return handler diff --git a/v2/ansible/executor/task_queue_manager.py b/v2/ansible/executor/task_queue_manager.py index a79235bfd0..72ff04d53d 100644 --- a/v2/ansible/executor/task_queue_manager.py +++ b/v2/ansible/executor/task_queue_manager.py @@ -19,18 +19,191 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -class TaskQueueManagerHostPlaybookIterator: +import multiprocessing +import os +import socket +import sys - def __init__(self, host, playbook): - pass +from ansible.errors import AnsibleError +from ansible.executor.connection_info import ConnectionInformation +#from ansible.executor.manager import AnsibleManager +from ansible.executor.play_iterator import PlayIterator +from ansible.executor.process.worker import WorkerProcess +from ansible.executor.process.result import ResultProcess +from ansible.plugins import callback_loader, strategy_loader - def get_next_task(self): - assert False +from ansible.utils.debug import debug - def is_blocked(self): - # depending on strategy, either - # ‘linear’ -- all prev tasks must be completed for all hosts - # ‘free’ -- this host doesn’t have any more work to do - assert False +__all__ = ['TaskQueueManager'] +class TaskQueueManager: + + ''' + This class handles the multiprocessing requirements of Ansible by + creating a pool of worker forks, a result handler fork, and a + manager object with shared datastructures/queues for coordinating + work between all processes. + + The queue manager is responsible for loading the play strategy plugin, + which dispatches the Play's tasks to hosts. + ''' + + def __init__(self, inventory, callback, variable_manager, loader, options): + + self._inventory = inventory + self._variable_manager = variable_manager + self._loader = loader + self._options = options + + # a special flag to help us exit cleanly + self._terminated = False + + # create and start the multiprocessing manager + #self._manager = AnsibleManager() + #self._manager.start() + + # this dictionary is used to keep track of notified handlers + self._notified_handlers = dict() + + # dictionaries to keep track of failed/unreachable hosts + self._failed_hosts = dict() + self._unreachable_hosts = dict() + + self._final_q = multiprocessing.Queue() + + # FIXME: hard-coded the default callback plugin here, which + # should be configurable. + self._callback = callback_loader.get(callback) + + # create the pool of worker threads, based on the number of forks specified + try: + fileno = sys.stdin.fileno() + except ValueError: + fileno = None + + self._workers = [] + for i in range(self._options.forks): + # duplicate stdin, if possible + new_stdin = None + if fileno is not None: + try: + new_stdin = os.fdopen(os.dup(fileno)) + except OSError, e: + # couldn't dupe stdin, most likely because it's + # not a valid file descriptor, so we just rely on + # using the one that was passed in + pass + + main_q = multiprocessing.Queue() + rslt_q = multiprocessing.Queue() + + prc = WorkerProcess(self, main_q, rslt_q, loader, new_stdin) + prc.start() + + self._workers.append((prc, main_q, rslt_q)) + + self._result_prc = ResultProcess(self._final_q, self._workers) + self._result_prc.start() + + def _initialize_notified_handlers(self, handlers): + ''' + Clears and initializes the shared notified handlers dict with entries + for each handler in the play, which is an empty array that will contain + inventory hostnames for those hosts triggering the handler. + ''' + + # Zero the dictionary first by removing any entries there. + # Proxied dicts don't support iteritems, so we have to use keys() + for key in self._notified_handlers.keys(): + del self._notified_handlers[key] + + # FIXME: there is a block compile helper for this... + handler_list = [] + for handler_block in handlers: + handler_list.extend(handler_block.compile()) + + # then initalize it with the handler names from the handler list + for handler in handler_list: + self._notified_handlers[handler.get_name()] = [] + + def run(self, play): + ''' + Iterates over the roles/tasks in a play, using the given (or default) + strategy for queueing tasks. The default is the linear strategy, which + operates like classic Ansible by keeping all hosts in lock-step with + a given task (meaning no hosts move on to the next task until all hosts + are done with the current task). + ''' + + connection_info = ConnectionInformation(play, self._options) + self._callback.set_connection_info(connection_info) + + # run final validation on the play now, to make sure fields are templated + # FIXME: is this even required? Everything is validated and merged at the + # task level, so else in the play needs to be templated + #all_vars = self._vmw.get_vars(loader=self._dlw, play=play) + #all_vars = self._vmw.get_vars(loader=self._loader, play=play) + #play.post_validate(all_vars=all_vars) + + self._callback.playbook_on_play_start(play.name) + + # initialize the shared dictionary containing the notified handlers + self._initialize_notified_handlers(play.handlers) + + # load the specified strategy (or the default linear one) + strategy = strategy_loader.get(play.strategy, self) + if strategy is None: + raise AnsibleError("Invalid play strategy specified: %s" % play.strategy, obj=play._ds) + + # build the iterator + iterator = PlayIterator(inventory=self._inventory, play=play) + + # and run the play using the strategy + return strategy.run(iterator, connection_info) + + def cleanup(self): + debug("RUNNING CLEANUP") + + self.terminate() + + self._final_q.close() + self._result_prc.terminate() + + for (worker_prc, main_q, rslt_q) in self._workers: + rslt_q.close() + main_q.close() + worker_prc.terminate() + + def get_inventory(self): + return self._inventory + + def get_callback(self): + return self._callback + + def get_variable_manager(self): + return self._variable_manager + + def get_loader(self): + return self._loader + + def get_server_pipe(self): + return self._server_pipe + + def get_client_pipe(self): + return self._client_pipe + + def get_pending_results(self): + return self._pending_results + + def get_allow_processing(self): + return self._allow_processing + + def get_notified_handlers(self): + return self._notified_handlers + + def get_workers(self): + return self._workers[:] + + def terminate(self): + self._terminated = True diff --git a/v2/ansible/executor/task_result.py b/v2/ansible/executor/task_result.py index 785fc45992..2b760bac00 100644 --- a/v2/ansible/executor/task_result.py +++ b/v2/ansible/executor/task_result.py @@ -19,3 +19,43 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible.parsing import DataLoader + +class TaskResult: + ''' + This class is responsible for interpretting the resulting data + from an executed task, and provides helper methods for determining + the result of a given task. + ''' + + def __init__(self, host, task, return_data): + self._host = host + self._task = task + if isinstance(return_data, dict): + self._result = return_data.copy() + else: + self._result = DataLoader().load(return_data) + + def is_changed(self): + return self._check_key('changed') + + def is_skipped(self): + return self._check_key('skipped') + + def is_failed(self): + if 'failed_when_result' in self._result: + return self._check_key('failed_when_result') + else: + return self._check_key('failed') or self._result.get('rc', 0) != 0 + + def is_unreachable(self): + return self._check_key('unreachable') + + def _check_key(self, key): + if 'results' in self._result: + flag = False + for res in self._result.get('results', []): + if isinstance(res, dict): + flag |= res.get(key, False) + else: + return self._result.get(key, False) diff --git a/v2/ansible/inventory/__init__.py b/v2/ansible/inventory/__init__.py index 5ad688eaf0..c8e3cddeba 100644 --- a/v2/ansible/inventory/__init__.py +++ b/v2/ansible/inventory/__init__.py @@ -16,73 +16,654 @@ # along with Ansible. If not, see <http://www.gnu.org/licenses/>. ############################################# +import fnmatch +import os +import sys +import re +import stat +import subprocess -# Make coding more python3-ish -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from ansible import constants as C +from ansible.errors import * + +from ansible.inventory.ini import InventoryParser +from ansible.inventory.script import InventoryScript +from ansible.inventory.dir import InventoryDirectory +from ansible.inventory.group import Group +from ansible.inventory.host import Host +from ansible.plugins import vars_loader +from ansible.utils.path import is_executable +from ansible.utils.vars import combine_vars + +class Inventory(object): + """ + Host inventory for ansible. + """ + + #__slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', + # 'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list', + # '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir'] + + def __init__(self, loader, variable_manager, host_list=C.DEFAULT_HOST_LIST): + + # the host file file, or script path, or list of hosts + # if a list, inventory data will NOT be loaded + self.host_list = host_list + self._loader = loader + self._variable_manager = variable_manager + + # caching to avoid repeated calculations, particularly with + # external inventory scripts. + + self._vars_per_host = {} + self._vars_per_group = {} + self._hosts_cache = {} + self._groups_list = {} + self._pattern_cache = {} + + # to be set by calling set_playbook_basedir by playbook code + self._playbook_basedir = None + + # the inventory object holds a list of groups + self.groups = [] + + # a list of host(names) to contain current inquiries to + self._restriction = None + self._also_restriction = None + self._subset = None + + if isinstance(host_list, basestring): + if "," in host_list: + host_list = host_list.split(",") + host_list = [ h for h in host_list if h and h.strip() ] + + if host_list is None: + self.parser = None + elif isinstance(host_list, list): + self.parser = None + all = Group('all') + self.groups = [ all ] + ipv6_re = re.compile('\[([a-f:A-F0-9]*[%[0-z]+]?)\](?::(\d+))?') + for x in host_list: + m = ipv6_re.match(x) + if m: + all.add_host(Host(m.groups()[0], m.groups()[1])) + else: + if ":" in x: + tokens = x.rsplit(":", 1) + # if there is ':' in the address, then this is an ipv6 + if ':' in tokens[0]: + all.add_host(Host(x)) + else: + all.add_host(Host(tokens[0], tokens[1])) + else: + all.add_host(Host(x)) + elif os.path.exists(host_list): + if os.path.isdir(host_list): + # Ensure basedir is inside the directory + self.host_list = os.path.join(self.host_list, "") + self.parser = InventoryDirectory(loader=self._loader, filename=host_list) + self.groups = self.parser.groups.values() + else: + # check to see if the specified file starts with a + # shebang (#!/), so if an error is raised by the parser + # class we can show a more apropos error + shebang_present = False + try: + inv_file = open(host_list) + first_line = inv_file.readlines()[0] + inv_file.close() + if first_line.startswith('#!'): + shebang_present = True + except: + pass + + if is_executable(host_list): + try: + self.parser = InventoryScript(loader=self._loader, filename=host_list) + self.groups = self.parser.groups.values() + except: + if not shebang_present: + raise errors.AnsibleError("The file %s is marked as executable, but failed to execute correctly. " % host_list + \ + "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % host_list) + else: + raise + else: + try: + self.parser = InventoryParser(filename=host_list) + self.groups = self.parser.groups.values() + except: + if shebang_present: + raise errors.AnsibleError("The file %s looks like it should be an executable inventory script, but is not marked executable. " % host_list + \ + "Perhaps you want to correct this with `chmod +x %s`?" % host_list) + else: + raise + + vars_loader.add_directory(self.basedir(), with_subdir=True) + else: + raise errors.AnsibleError("Unable to find an inventory file, specify one with -i ?") + + self._vars_plugins = [ x for x in vars_loader.all(self) ] + + # FIXME: shouldn't be required, since the group/host vars file + # management will be done in VariableManager + # get group vars from group_vars/ files and vars plugins + for group in self.groups: + # FIXME: combine_vars + group.vars = combine_vars(group.vars, self.get_group_variables(group.name)) + + # get host vars from host_vars/ files and vars plugins + for host in self.get_hosts(): + # FIXME: combine_vars + host.vars = combine_vars(host.vars, self.get_host_variables(host.name)) + + + def _match(self, str, pattern_str): + try: + if pattern_str.startswith('~'): + return re.search(pattern_str[1:], str) + else: + return fnmatch.fnmatch(str, pattern_str) + except Exception, e: + raise errors.AnsibleError('invalid host pattern: %s' % pattern_str) + + def _match_list(self, items, item_attr, pattern_str): + results = [] + try: + if not pattern_str.startswith('~'): + pattern = re.compile(fnmatch.translate(pattern_str)) + else: + pattern = re.compile(pattern_str[1:]) + except Exception, e: + raise errors.AnsibleError('invalid host pattern: %s' % pattern_str) + + for item in items: + if pattern.match(getattr(item, item_attr)): + results.append(item) + return results -class Inventory: - def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None): - pass def get_hosts(self, pattern="all"): - pass + """ + find all host names matching a pattern string, taking into account any inventory restrictions or + applied subsets. + """ + + # process patterns + if isinstance(pattern, list): + pattern = ';'.join(pattern) + patterns = pattern.replace(";",":").split(":") + hosts = self._get_hosts(patterns) + + # exclude hosts not in a subset, if defined + if self._subset: + subset = self._get_hosts(self._subset) + hosts = [ h for h in hosts if h in subset ] + + # exclude hosts mentioned in any restriction (ex: failed hosts) + if self._restriction is not None: + hosts = [ h for h in hosts if h in self._restriction ] + if self._also_restriction is not None: + hosts = [ h for h in hosts if h in self._also_restriction ] + + return hosts + + def _get_hosts(self, patterns): + """ + finds hosts that match a list of patterns. Handles negative + matches as well as intersection matches. + """ + + # Host specifiers should be sorted to ensure consistent behavior + pattern_regular = [] + pattern_intersection = [] + pattern_exclude = [] + for p in patterns: + if p.startswith("!"): + pattern_exclude.append(p) + elif p.startswith("&"): + pattern_intersection.append(p) + elif p: + pattern_regular.append(p) + + # if no regular pattern was given, hence only exclude and/or intersection + # make that magically work + if pattern_regular == []: + pattern_regular = ['all'] + + # when applying the host selectors, run those without the "&" or "!" + # first, then the &s, then the !s. + patterns = pattern_regular + pattern_intersection + pattern_exclude + + hosts = [] + + for p in patterns: + # avoid resolving a pattern that is a plain host + if p in self._hosts_cache: + hosts.append(self.get_host(p)) + else: + that = self.__get_hosts(p) + if p.startswith("!"): + hosts = [ h for h in hosts if h not in that ] + elif p.startswith("&"): + hosts = [ h for h in hosts if h in that ] + else: + to_append = [ h for h in that if h.name not in [ y.name for y in hosts ] ] + hosts.extend(to_append) + return hosts + + def __get_hosts(self, pattern): + """ + finds hosts that positively match a particular pattern. Does not + take into account negative matches. + """ + + if pattern in self._pattern_cache: + return self._pattern_cache[pattern] + + (name, enumeration_details) = self._enumeration_info(pattern) + hpat = self._hosts_in_unenumerated_pattern(name) + result = self._apply_ranges(pattern, hpat) + self._pattern_cache[pattern] = result + return result + + def _enumeration_info(self, pattern): + """ + returns (pattern, limits) taking a regular pattern and finding out + which parts of it correspond to start/stop offsets. limits is + a tuple of (start, stop) or None + """ + + # Do not parse regexes for enumeration info + if pattern.startswith('~'): + return (pattern, None) + + # The regex used to match on the range, which can be [x] or [x-y]. + pattern_re = re.compile("^(.*)\[([-]?[0-9]+)(?:(?:-)([0-9]+))?\](.*)$") + m = pattern_re.match(pattern) + if m: + (target, first, last, rest) = m.groups() + first = int(first) + if last: + if first < 0: + raise errors.AnsibleError("invalid range: negative indices cannot be used as the first item in a range") + last = int(last) + else: + last = first + return (target, (first, last)) + else: + return (pattern, None) + + def _apply_ranges(self, pat, hosts): + """ + given a pattern like foo, that matches hosts, return all of hosts + given a pattern like foo[0:5], where foo matches hosts, return the first 6 hosts + """ + + # If there are no hosts to select from, just return the + # empty set. This prevents trying to do selections on an empty set. + # issue#6258 + if not hosts: + return hosts + + (loose_pattern, limits) = self._enumeration_info(pat) + if not limits: + return hosts + + (left, right) = limits + + if left == '': + left = 0 + if right == '': + right = 0 + left=int(left) + right=int(right) + try: + if left != right: + return hosts[left:right] + else: + return [ hosts[left] ] + except IndexError: + raise errors.AnsibleError("no hosts matching the pattern '%s' were found" % pat) + + def _create_implicit_localhost(self, pattern): + new_host = Host(pattern) + new_host.set_variable("ansible_python_interpreter", sys.executable) + new_host.set_variable("ansible_connection", "local") + new_host.ipv4_address = '127.0.0.1' + + ungrouped = self.get_group("ungrouped") + if ungrouped is None: + self.add_group(Group('ungrouped')) + ungrouped = self.get_group('ungrouped') + self.get_group('all').add_child_group(ungrouped) + ungrouped.add_host(new_host) + return new_host + + def _hosts_in_unenumerated_pattern(self, pattern): + """ Get all host names matching the pattern """ + + results = [] + hosts = [] + hostnames = set() + + # ignore any negative checks here, this is handled elsewhere + pattern = pattern.replace("!","").replace("&", "") + + def __append_host_to_results(host): + if host not in results and host.name not in hostnames: + hostnames.add(host.name) + results.append(host) + + groups = self.get_groups() + for group in groups: + if pattern == 'all': + for host in group.get_hosts(): + __append_host_to_results(host) + else: + if self._match(group.name, pattern): + for host in group.get_hosts(): + __append_host_to_results(host) + else: + matching_hosts = self._match_list(group.get_hosts(), 'name', pattern) + for host in matching_hosts: + __append_host_to_results(host) + + if pattern in ["localhost", "127.0.0.1"] and len(results) == 0: + new_host = self._create_implicit_localhost(pattern) + results.append(new_host) + return results + def clear_pattern_cache(self): - # Possibly not needed? - pass + ''' called exclusively by the add_host plugin to allow patterns to be recalculated ''' + self._pattern_cache = {} + def groups_for_host(self, host): - pass + if host in self._hosts_cache: + return self._hosts_cache[host].get_groups() + else: + return [] + def groups_list(self): - pass + if not self._groups_list: + groups = {} + for g in self.groups: + groups[g.name] = [h.name for h in g.get_hosts()] + ancestors = g.get_ancestors() + for a in ancestors: + if a.name not in groups: + groups[a.name] = [h.name for h in a.get_hosts()] + self._groups_list = groups + return self._groups_list + def get_groups(self): - pass + return self.groups + def get_host(self, hostname): - pass + if hostname not in self._hosts_cache: + self._hosts_cache[hostname] = self._get_host(hostname) + return self._hosts_cache[hostname] + + def _get_host(self, hostname): + if hostname in ['localhost','127.0.0.1']: + for host in self.get_group('all').get_hosts(): + if host.name in ['localhost', '127.0.0.1']: + return host + return self._create_implicit_localhost(hostname) + else: + for group in self.groups: + for host in group.get_hosts(): + if hostname == host.name: + return host + return None + def get_group(self, groupname): - pass + for group in self.groups: + if group.name == groupname: + return group + return None + def get_group_variables(self, groupname, update_cached=False, vault_password=None): - pass - def get_variables(self, hostname, update_cached=False, vault_password=None): - pass + if groupname not in self._vars_per_group or update_cached: + self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password) + return self._vars_per_group[groupname] + + def _get_group_variables(self, groupname, vault_password=None): + + group = self.get_group(groupname) + if group is None: + raise Exception("group not found: %s" % groupname) + + vars = {} + + # plugin.get_group_vars retrieves just vars for specific group + vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')] + for updated in vars_results: + if updated is not None: + # FIXME: combine_vars + vars = combine_vars(vars, updated) + + # Read group_vars/ files + # FIXME: combine_vars + vars = combine_vars(vars, self.get_group_vars(group)) + + return vars + + def get_vars(self, hostname, update_cached=False, vault_password=None): + + host = self.get_host(hostname) + if not host: + raise Exception("host not found: %s" % hostname) + return host.get_vars() + def get_host_variables(self, hostname, update_cached=False, vault_password=None): - pass + + if hostname not in self._vars_per_host or update_cached: + self._vars_per_host[hostname] = self._get_host_variables(hostname, vault_password=vault_password) + return self._vars_per_host[hostname] + + def _get_host_variables(self, hostname, vault_password=None): + + host = self.get_host(hostname) + if host is None: + raise errors.AnsibleError("host not found: %s" % hostname) + + vars = {} + + # plugin.run retrieves all vars (also from groups) for host + vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')] + for updated in vars_results: + if updated is not None: + # FIXME: combine_vars + vars = combine_vars(vars, updated) + + # plugin.get_host_vars retrieves just vars for specific host + vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')] + for updated in vars_results: + if updated is not None: + # FIXME: combine_vars + vars = combine_vars(vars, updated) + + # still need to check InventoryParser per host vars + # which actually means InventoryScript per host, + # which is not performant + if self.parser is not None: + # FIXME: combine_vars + vars = combine_vars(vars, self.parser.get_host_variables(host)) + + # Read host_vars/ files + # FIXME: combine_vars + vars = combine_vars(vars, self.get_host_vars(host)) + + return vars + def add_group(self, group): - pass + if group.name not in self.groups_list(): + self.groups.append(group) + self._groups_list = None # invalidate internal cache + else: + raise errors.AnsibleError("group already in inventory: %s" % group.name) + def list_hosts(self, pattern="all"): - pass + + """ return a list of hostnames for a pattern """ + + result = [ h for h in self.get_hosts(pattern) ] + if len(result) == 0 and pattern in ["localhost", "127.0.0.1"]: + result = [pattern] + return result + def list_groups(self): - pass - def get_restriction(self): - pass - def restrict_to(self, restriction): - pass + return sorted([ g.name for g in self.groups ], key=lambda x: x) + + def restrict_to_hosts(self, restriction): + """ + Restrict list operations to the hosts given in restriction. This is used + to exclude failed hosts in main playbook code, don't use this for other + reasons. + """ + if not isinstance(restriction, list): + restriction = [ restriction ] + self._restriction = restriction + def also_restrict_to(self, restriction): - pass - def subset(self, subset_pattern): """ + Works like restict_to but offers an additional restriction. Playbooks use this + to implement serial behavior. + """ + if not isinstance(restriction, list): + restriction = [ restriction ] + self._also_restriction = restriction + + def subset(self, subset_pattern): + """ Limits inventory results to a subset of inventory that matches a given pattern, such as to select a given geographic of numeric slice amongst - a previous 'hosts' selection that only select roles, or vice versa... + a previous 'hosts' selection that only select roles, or vice versa. Corresponds to --limit parameter to ansible-playbook - """ - pass - def lift_restriction(self): - # HACK -- - pass + """ + if subset_pattern is None: + self._subset = None + else: + subset_pattern = subset_pattern.replace(',',':') + subset_pattern = subset_pattern.replace(";",":").split(":") + results = [] + # allow Unix style @filename data + for x in subset_pattern: + if x.startswith("@"): + fd = open(x[1:]) + results.extend(fd.read().split("\n")) + fd.close() + else: + results.append(x) + self._subset = results + + def remove_restriction(self): + """ Do not restrict list operations """ + self._restriction = None + def lift_also_restriction(self): - # HACK -- dead host skipping - pass + """ Clears the also restriction """ + self._also_restriction = None + def is_file(self): - pass + """ did inventory come from a file? """ + if not isinstance(self.host_list, basestring): + return False + return os.path.exists(self.host_list) + def basedir(self): - pass + """ if inventory came from a file, what's the directory? """ + if not self.is_file(): + return None + dname = os.path.dirname(self.host_list) + if dname is None or dname == '' or dname == '.': + cwd = os.getcwd() + return os.path.abspath(cwd) + return os.path.abspath(dname) + def src(self): - pass + """ if inventory came from a file, what's the directory and file name? """ + if not self.is_file(): + return None + return self.host_list + def playbook_basedir(self): - pass + """ returns the directory of the current playbook """ + return self._playbook_basedir + def set_playbook_basedir(self, dir): - pass + """ + sets the base directory of the playbook so inventory can use it as a + basedir for host_ and group_vars, and other things. + """ + # Only update things if dir is a different playbook basedir + if dir != self._playbook_basedir: + self._playbook_basedir = dir + # get group vars from group_vars/ files + for group in self.groups: + # FIXME: combine_vars + group.vars = combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True)) + # get host vars from host_vars/ files + for host in self.get_hosts(): + # FIXME: combine_vars + host.vars = combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True)) + # invalidate cache + self._vars_per_host = {} + self._vars_per_group = {} + def get_host_vars(self, host, new_pb_basedir=False): - pass + """ Read host_vars/ files """ + return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=new_pb_basedir) + def get_group_vars(self, group, new_pb_basedir=False): - pass + """ Read group_vars/ files """ + return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=new_pb_basedir) + + def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False): + """ + Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel + to the inventory base directory or in the same directory as the playbook. Variables in the playbook + dir will win over the inventory dir if files are in both. + """ + + results = {} + scan_pass = 0 + _basedir = self.basedir() + + # look in both the inventory base directory and the playbook base directory + # unless we do an update for a new playbook base dir + if not new_pb_basedir: + basedirs = [_basedir, self._playbook_basedir] + else: + basedirs = [self._playbook_basedir] + + for basedir in basedirs: + + # this can happen from particular API usages, particularly if not run + # from /usr/bin/ansible-playbook + if basedir is None: + continue + + scan_pass = scan_pass + 1 + + # it's not an eror if the directory does not exist, keep moving + if not os.path.exists(basedir): + continue + + # save work of second scan if the directories are the same + if _basedir == self._playbook_basedir and scan_pass != 1: + continue + + # FIXME: these should go to VariableManager + if group and host is None: + # load vars in dir/group_vars/name_of_group + base_path = os.path.join(basedir, "group_vars/%s" % group.name) + self._variable_manager.add_group_vars_file(base_path, self._loader) + elif host and group is None: + # same for hostvars in dir/host_vars/name_of_host + base_path = os.path.join(basedir, "host_vars/%s" % host.name) + self._variable_manager.add_host_vars_file(base_path, self._loader) + + # all done, results is a dictionary of variables for this particular host. + return results diff --git a/v2/ansible/inventory/dir.py b/v2/ansible/inventory/dir.py new file mode 100644 index 0000000000..52f7af8b53 --- /dev/null +++ b/v2/ansible/inventory/dir.py @@ -0,0 +1,234 @@ +# (c) 2013, Daniel Hokka Zakrisson <daniel@hozac.com> +# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +import os + +from ansible import constants as C +from ansible.errors import * + +from ansible.inventory.host import Host +from ansible.inventory.group import Group +from ansible.inventory.ini import InventoryParser +from ansible.inventory.script import InventoryScript +from ansible.utils.path import is_executable +from ansible.utils.vars import combine_vars + +class InventoryDirectory(object): + ''' Host inventory parser for ansible using a directory of inventories. ''' + + def __init__(self, loader, filename=C.DEFAULT_HOST_LIST): + self.names = os.listdir(filename) + self.names.sort() + self.directory = filename + self.parsers = [] + self.hosts = {} + self.groups = {} + + self._loader = loader + + for i in self.names: + + # Skip files that end with certain extensions or characters + if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo")): + continue + # Skip hidden files + if i.startswith('.') and not i.startswith('./'): + continue + # These are things inside of an inventory basedir + if i in ("host_vars", "group_vars", "vars_plugins"): + continue + fullpath = os.path.join(self.directory, i) + if os.path.isdir(fullpath): + parser = InventoryDirectory(loader=loader, filename=fullpath) + elif is_executable(fullpath): + parser = InventoryScript(loader=loader, filename=fullpath) + else: + parser = InventoryParser(filename=fullpath) + self.parsers.append(parser) + + # retrieve all groups and hosts form the parser and add them to + # self, don't look at group lists yet, to avoid + # recursion trouble, but just make sure all objects exist in self + newgroups = parser.groups.values() + for group in newgroups: + for host in group.hosts: + self._add_host(host) + for group in newgroups: + self._add_group(group) + + # now check the objects lists so they contain only objects from + # self; membership data in groups is already fine (except all & + # ungrouped, see later), but might still reference objects not in self + for group in self.groups.values(): + # iterate on a copy of the lists, as those lists get changed in + # the loop + # list with group's child group objects: + for child in group.child_groups[:]: + if child != self.groups[child.name]: + group.child_groups.remove(child) + group.child_groups.append(self.groups[child.name]) + # list with group's parent group objects: + for parent in group.parent_groups[:]: + if parent != self.groups[parent.name]: + group.parent_groups.remove(parent) + group.parent_groups.append(self.groups[parent.name]) + # list with group's host objects: + for host in group.hosts[:]: + if host != self.hosts[host.name]: + group.hosts.remove(host) + group.hosts.append(self.hosts[host.name]) + # also check here that the group that contains host, is + # also contained in the host's group list + if group not in self.hosts[host.name].groups: + self.hosts[host.name].groups.append(group) + + # extra checks on special groups all and ungrouped + # remove hosts from 'ungrouped' if they became member of other groups + if 'ungrouped' in self.groups: + ungrouped = self.groups['ungrouped'] + # loop on a copy of ungrouped hosts, as we want to change that list + for host in ungrouped.hosts[:]: + if len(host.groups) > 1: + host.groups.remove(ungrouped) + ungrouped.hosts.remove(host) + + # remove hosts from 'all' if they became member of other groups + # all should only contain direct children, not grandchildren + # direct children should have dept == 1 + if 'all' in self.groups: + allgroup = self.groups['all' ] + # loop on a copy of all's child groups, as we want to change that list + for group in allgroup.child_groups[:]: + # groups might once have beeen added to all, and later be added + # to another group: we need to remove the link wit all then + if len(group.parent_groups) > 1 and allgroup in group.parent_groups: + # real children of all have just 1 parent, all + # this one has more, so not a direct child of all anymore + group.parent_groups.remove(allgroup) + allgroup.child_groups.remove(group) + elif allgroup not in group.parent_groups: + # this group was once added to all, but doesn't list it as + # a parent any more; the info in the group is the correct + # info + allgroup.child_groups.remove(group) + + + def _add_group(self, group): + """ Merge an existing group or add a new one; + Track parent and child groups, and hosts of the new one """ + + if group.name not in self.groups: + # it's brand new, add him! + self.groups[group.name] = group + if self.groups[group.name] != group: + # different object, merge + self._merge_groups(self.groups[group.name], group) + + def _add_host(self, host): + if host.name not in self.hosts: + # Papa's got a brand new host + self.hosts[host.name] = host + if self.hosts[host.name] != host: + # different object, merge + self._merge_hosts(self.hosts[host.name], host) + + def _merge_groups(self, group, newgroup): + """ Merge all of instance newgroup into group, + update parent/child relationships + group lists may still contain group objects that exist in self with + same name, but was instanciated as a different object in some other + inventory parser; these are handled later """ + + # name + if group.name != newgroup.name: + raise errors.AnsibleError("Cannot merge group %s with %s" % (group.name, newgroup.name)) + + # depth + group.depth = max([group.depth, newgroup.depth]) + + # hosts list (host objects are by now already added to self.hosts) + for host in newgroup.hosts: + grouphosts = dict([(h.name, h) for h in group.hosts]) + if host.name in grouphosts: + # same host name but different object, merge + self._merge_hosts(grouphosts[host.name], host) + else: + # new membership, add host to group from self + # group from self will also be added again to host.groups, but + # as different object + group.add_host(self.hosts[host.name]) + # now remove this the old object for group in host.groups + for hostgroup in [g for g in host.groups]: + if hostgroup.name == group.name and hostgroup != self.groups[group.name]: + self.hosts[host.name].groups.remove(hostgroup) + + + # group child membership relation + for newchild in newgroup.child_groups: + # dict with existing child groups: + childgroups = dict([(g.name, g) for g in group.child_groups]) + # check if child of new group is already known as a child + if newchild.name not in childgroups: + self.groups[group.name].add_child_group(newchild) + + # group parent membership relation + for newparent in newgroup.parent_groups: + # dict with existing parent groups: + parentgroups = dict([(g.name, g) for g in group.parent_groups]) + # check if parent of new group is already known as a parent + if newparent.name not in parentgroups: + if newparent.name not in self.groups: + # group does not exist yet in self, import him + self.groups[newparent.name] = newparent + # group now exists but not yet as a parent here + self.groups[newparent.name].add_child_group(group) + + # variables + group.vars = combine_vars(group.vars, newgroup.vars) + + def _merge_hosts(self,host, newhost): + """ Merge all of instance newhost into host """ + + # name + if host.name != newhost.name: + raise errors.AnsibleError("Cannot merge host %s with %s" % (host.name, newhost.name)) + + # group membership relation + for newgroup in newhost.groups: + # dict with existing groups: + hostgroups = dict([(g.name, g) for g in host.groups]) + # check if new group is already known as a group + if newgroup.name not in hostgroups: + if newgroup.name not in self.groups: + # group does not exist yet in self, import him + self.groups[newgroup.name] = newgroup + # group now exists but doesn't have host yet + self.groups[newgroup.name].add_host(host) + + # variables + host.vars = combine_vars(host.vars, newhost.vars) + + def get_host_variables(self, host): + """ Gets additional host variables from all inventories """ + vars = {} + for i in self.parsers: + vars.update(i.get_host_variables(host)) + return vars + diff --git a/v2/ansible/inventory/expand_hosts.py b/v2/ansible/inventory/expand_hosts.py new file mode 100644 index 0000000000..f129740935 --- /dev/null +++ b/v2/ansible/inventory/expand_hosts.py @@ -0,0 +1,116 @@ +# (c) 2012, Zettar Inc. +# Written by Chin Fang <fangchin@zettar.com> +# +# This file is part of Ansible +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. +# + +''' +This module is for enhancing ansible's inventory parsing capability such +that it can deal with hostnames specified using a simple pattern in the +form of [beg:end], example: [1:5], [a:c], [D:G]. If beg is not specified, +it defaults to 0. + +If beg is given and is left-zero-padded, e.g. '001', it is taken as a +formatting hint when the range is expanded. e.g. [001:010] is to be +expanded into 001, 002 ...009, 010. + +Note that when beg is specified with left zero padding, then the length of +end must be the same as that of beg, else an exception is raised. +''' +import string + +from ansible import errors + +def detect_range(line = None): + ''' + A helper function that checks a given host line to see if it contains + a range pattern described in the docstring above. + + Returnes True if the given line contains a pattern, else False. + ''' + if 0 <= line.find("[") < line.find(":") < line.find("]"): + return True + else: + return False + +def expand_hostname_range(line = None): + ''' + A helper function that expands a given line that contains a pattern + specified in top docstring, and returns a list that consists of the + expanded version. + + The '[' and ']' characters are used to maintain the pseudo-code + appearance. They are replaced in this function with '|' to ease + string splitting. + + References: http://ansible.github.com/patterns.html#hosts-and-groups + ''' + all_hosts = [] + if line: + # A hostname such as db[1:6]-node is considered to consists + # three parts: + # head: 'db' + # nrange: [1:6]; range() is a built-in. Can't use the name + # tail: '-node' + + # Add support for multiple ranges in a host so: + # db[01:10:3]node-[01:10] + # - to do this we split off at the first [...] set, getting the list + # of hosts and then repeat until none left. + # - also add an optional third parameter which contains the step. (Default: 1) + # so range can be [01:10:2] -> 01 03 05 07 09 + # FIXME: make this work for alphabetic sequences too. + + (head, nrange, tail) = line.replace('[','|',1).replace(']','|',1).split('|') + bounds = nrange.split(":") + if len(bounds) != 2 and len(bounds) != 3: + raise errors.AnsibleError("host range incorrectly specified") + beg = bounds[0] + end = bounds[1] + if len(bounds) == 2: + step = 1 + else: + step = bounds[2] + if not beg: + beg = "0" + if not end: + raise errors.AnsibleError("host range end value missing") + if beg[0] == '0' and len(beg) > 1: + rlen = len(beg) # range length formatting hint + if rlen != len(end): + raise errors.AnsibleError("host range format incorrectly specified!") + fill = lambda _: str(_).zfill(rlen) # range sequence + else: + fill = str + + try: + i_beg = string.ascii_letters.index(beg) + i_end = string.ascii_letters.index(end) + if i_beg > i_end: + raise errors.AnsibleError("host range format incorrectly specified!") + seq = string.ascii_letters[i_beg:i_end+1] + except ValueError: # not an alpha range + seq = range(int(beg), int(end)+1, int(step)) + + for rseq in seq: + hname = ''.join((head, fill(rseq), tail)) + + if detect_range(hname): + all_hosts.extend( expand_hostname_range( hname ) ) + else: + all_hosts.append(hname) + + return all_hosts diff --git a/v2/ansible/inventory/group.py b/v2/ansible/inventory/group.py new file mode 100644 index 0000000000..87d6f64dfc --- /dev/null +++ b/v2/ansible/inventory/group.py @@ -0,0 +1,159 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.utils.debug import debug + +class Group: + ''' a group of ansible hosts ''' + + #__slots__ = [ 'name', 'hosts', 'vars', 'child_groups', 'parent_groups', 'depth', '_hosts_cache' ] + + def __init__(self, name=None): + + self.depth = 0 + self.name = name + self.hosts = [] + self.vars = {} + self.child_groups = [] + self.parent_groups = [] + self._hosts_cache = None + + #self.clear_hosts_cache() + #if self.name is None: + # raise Exception("group name is required") + + def __repr__(self): + return self.get_name() + + def __getstate__(self): + return self.serialize() + + def __setstate__(self, data): + return self.deserialize(data) + + def serialize(self): + parent_groups = [] + for parent in self.parent_groups: + parent_groups.append(parent.serialize()) + + result = dict( + name=self.name, + vars=self.vars.copy(), + parent_groups=parent_groups, + depth=self.depth, + ) + + debug("serializing group, result is: %s" % result) + return result + + def deserialize(self, data): + debug("deserializing group, data is: %s" % data) + self.__init__() + self.name = data.get('name') + self.vars = data.get('vars', dict()) + + parent_groups = data.get('parent_groups', []) + for parent_data in parent_groups: + g = Group() + g.deserialize(parent_data) + self.parent_groups.append(g) + + def get_name(self): + return self.name + + def add_child_group(self, group): + + if self == group: + raise Exception("can't add group to itself") + + # don't add if it's already there + if not group in self.child_groups: + self.child_groups.append(group) + + # update the depth of the child + group.depth = max([self.depth+1, group.depth]) + + # update the depth of the grandchildren + group._check_children_depth() + + # now add self to child's parent_groups list, but only if there + # isn't already a group with the same name + if not self.name in [g.name for g in group.parent_groups]: + group.parent_groups.append(self) + + self.clear_hosts_cache() + + def _check_children_depth(self): + + for group in self.child_groups: + group.depth = max([self.depth+1, group.depth]) + group._check_children_depth() + + def add_host(self, host): + + self.hosts.append(host) + host.add_group(self) + self.clear_hosts_cache() + + def set_variable(self, key, value): + + self.vars[key] = value + + def clear_hosts_cache(self): + + self._hosts_cache = None + for g in self.parent_groups: + g.clear_hosts_cache() + + def get_hosts(self): + + if self._hosts_cache is None: + self._hosts_cache = self._get_hosts() + + return self._hosts_cache + + def _get_hosts(self): + + hosts = [] + seen = {} + for kid in self.child_groups: + kid_hosts = kid.get_hosts() + for kk in kid_hosts: + if kk not in seen: + seen[kk] = 1 + hosts.append(kk) + for mine in self.hosts: + if mine not in seen: + seen[mine] = 1 + hosts.append(mine) + return hosts + + def get_vars(self): + return self.vars.copy() + + def _get_ancestors(self): + + results = {} + for g in self.parent_groups: + results[g.name] = g + results.update(g._get_ancestors()) + return results + + def get_ancestors(self): + + return self._get_ancestors().values() + diff --git a/v2/ansible/inventory/host.py b/v2/ansible/inventory/host.py new file mode 100644 index 0000000000..414ec34b96 --- /dev/null +++ b/v2/ansible/inventory/host.py @@ -0,0 +1,127 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible import constants as C +from ansible.inventory.group import Group +from ansible.utils.vars import combine_vars + +__all__ = ['Host'] + +class Host: + ''' a single ansible host ''' + + #__slots__ = [ 'name', 'vars', 'groups' ] + + def __getstate__(self): + return self.serialize() + + def __setstate__(self, data): + return self.deserialize(data) + + def serialize(self): + groups = [] + for group in self.groups: + groups.append(group.serialize()) + + return dict( + name=self.name, + vars=self.vars.copy(), + ipv4_address=self.ipv4_address, + ipv6_address=self.ipv6_address, + port=self.port, + gathered_facts=self._gathered_facts, + groups=groups, + ) + + def deserialize(self, data): + self.__init__() + + self.name = data.get('name') + self.vars = data.get('vars', dict()) + self.ipv4_address = data.get('ipv4_address', '') + self.ipv6_address = data.get('ipv6_address', '') + self.port = data.get('port') + + groups = data.get('groups', []) + for group_data in groups: + g = Group() + g.deserialize(group_data) + self.groups.append(g) + + def __init__(self, name=None, port=None): + + self.name = name + self.vars = {} + self.groups = [] + + self.ipv4_address = name + self.ipv6_address = name + + if port and port != C.DEFAULT_REMOTE_PORT: + self.port = int(port) + else: + self.port = C.DEFAULT_REMOTE_PORT + + self._gathered_facts = False + + def __repr__(self): + return self.get_name() + + def get_name(self): + return self.name + + @property + def gathered_facts(self): + return self._gathered_facts + + def set_gathered_facts(self, gathered): + self._gathered_facts = gathered + + def add_group(self, group): + + self.groups.append(group) + + def set_variable(self, key, value): + + self.vars[key]=value + + def get_groups(self): + + groups = {} + for g in self.groups: + groups[g.name] = g + ancestors = g.get_ancestors() + for a in ancestors: + groups[a.name] = a + return groups.values() + + def get_vars(self): + + results = {} + groups = self.get_groups() + for group in sorted(groups, key=lambda g: g.depth): + results = combine_vars(results, group.get_vars()) + results = combine_vars(results, self.vars) + results['inventory_hostname'] = self.name + results['inventory_hostname_short'] = self.name.split('.')[0] + results['group_names'] = sorted([ g.name for g in groups if g.name != 'all']) + return results + diff --git a/v2/ansible/inventory/ini.py b/v2/ansible/inventory/ini.py new file mode 100644 index 0000000000..075701c056 --- /dev/null +++ b/v2/ansible/inventory/ini.py @@ -0,0 +1,215 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +import ast +import shlex +import re + +from ansible import constants as C +from ansible.errors import * +from ansible.inventory.host import Host +from ansible.inventory.group import Group +from ansible.inventory.expand_hosts import detect_range +from ansible.inventory.expand_hosts import expand_hostname_range + +class InventoryParser(object): + """ + Host inventory for ansible. + """ + + def __init__(self, filename=C.DEFAULT_HOST_LIST): + self.filename = filename + with open(filename) as fh: + self.lines = fh.readlines() + self.groups = {} + self.hosts = {} + self._parse() + + def _parse(self): + + self._parse_base_groups() + self._parse_group_children() + self._add_allgroup_children() + self._parse_group_variables() + return self.groups + + @staticmethod + def _parse_value(v): + if "#" not in v: + try: + return ast.literal_eval(v) + # Using explicit exceptions. + # Likely a string that literal_eval does not like. We wil then just set it. + except ValueError: + # For some reason this was thought to be malformed. + pass + except SyntaxError: + # Is this a hash with an equals at the end? + pass + return v + + # [webservers] + # alpha + # beta:2345 + # gamma sudo=True user=root + # delta asdf=jkl favcolor=red + + def _add_allgroup_children(self): + + for group in self.groups.values(): + if group.depth == 0 and group.name != 'all': + self.groups['all'].add_child_group(group) + + + def _parse_base_groups(self): + # FIXME: refactor + + ungrouped = Group(name='ungrouped') + all = Group(name='all') + all.add_child_group(ungrouped) + + self.groups = dict(all=all, ungrouped=ungrouped) + active_group_name = 'ungrouped' + + for line in self.lines: + line = self._before_comment(line).strip() + if line.startswith("[") and line.endswith("]"): + active_group_name = line.replace("[","").replace("]","") + if ":vars" in line or ":children" in line: + active_group_name = active_group_name.rsplit(":", 1)[0] + if active_group_name not in self.groups: + new_group = self.groups[active_group_name] = Group(name=active_group_name) + active_group_name = None + elif active_group_name not in self.groups: + new_group = self.groups[active_group_name] = Group(name=active_group_name) + elif line.startswith(";") or line == '': + pass + elif active_group_name: + tokens = shlex.split(line) + if len(tokens) == 0: + continue + hostname = tokens[0] + port = C.DEFAULT_REMOTE_PORT + # Three cases to check: + # 0. A hostname that contains a range pesudo-code and a port + # 1. A hostname that contains just a port + if hostname.count(":") > 1: + # Possible an IPv6 address, or maybe a host line with multiple ranges + # IPv6 with Port XXX:XXX::XXX.port + # FQDN foo.example.com + if hostname.count(".") == 1: + (hostname, port) = hostname.rsplit(".", 1) + elif ("[" in hostname and + "]" in hostname and + ":" in hostname and + (hostname.rindex("]") < hostname.rindex(":")) or + ("]" not in hostname and ":" in hostname)): + (hostname, port) = hostname.rsplit(":", 1) + + hostnames = [] + if detect_range(hostname): + hostnames = expand_hostname_range(hostname) + else: + hostnames = [hostname] + + for hn in hostnames: + host = None + if hn in self.hosts: + host = self.hosts[hn] + else: + host = Host(name=hn, port=port) + self.hosts[hn] = host + if len(tokens) > 1: + for t in tokens[1:]: + if t.startswith('#'): + break + try: + (k,v) = t.split("=", 1) + except ValueError, e: + raise AnsibleError("Invalid ini entry in %s: %s - %s" % (self.filename, t, str(e))) + if k == 'ansible_ssh_host': + host.ipv4_address = self._parse_value(v) + else: + host.set_variable(k, self._parse_value(v)) + self.groups[active_group_name].add_host(host) + + # [southeast:children] + # atlanta + # raleigh + + def _parse_group_children(self): + group = None + + for line in self.lines: + line = line.strip() + if line is None or line == '': + continue + if line.startswith("[") and ":children]" in line: + line = line.replace("[","").replace(":children]","") + group = self.groups.get(line, None) + if group is None: + group = self.groups[line] = Group(name=line) + elif line.startswith("#") or line.startswith(";"): + pass + elif line.startswith("["): + group = None + elif group: + kid_group = self.groups.get(line, None) + if kid_group is None: + raise AnsibleError("child group is not defined: (%s)" % line) + else: + group.add_child_group(kid_group) + + + # [webservers:vars] + # http_port=1234 + # maxRequestsPerChild=200 + + def _parse_group_variables(self): + group = None + for line in self.lines: + line = line.strip() + if line.startswith("[") and ":vars]" in line: + line = line.replace("[","").replace(":vars]","") + group = self.groups.get(line, None) + if group is None: + raise AnsibleError("can't add vars to undefined group: %s" % line) + elif line.startswith("#") or line.startswith(";"): + pass + elif line.startswith("["): + group = None + elif line == '': + pass + elif group: + if "=" not in line: + raise AnsibleError("variables assigned to group must be in key=value form") + else: + (k, v) = [e.strip() for e in line.split("=", 1)] + group.set_variable(k, self._parse_value(v)) + + def get_host_variables(self, host): + return {} + + def _before_comment(self, msg): + ''' what's the part of a string before a comment? ''' + msg = msg.replace("\#","**NOT_A_COMMENT**") + msg = msg.split("#")[0] + msg = msg.replace("**NOT_A_COMMENT**","#") + return msg + diff --git a/v2/ansible/inventory/script.py b/v2/ansible/inventory/script.py new file mode 100644 index 0000000000..13b53a24f5 --- /dev/null +++ b/v2/ansible/inventory/script.py @@ -0,0 +1,156 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +import os +import subprocess +import sys + +from ansible import constants as C +from ansible.errors import * +from ansible.inventory.host import Host +from ansible.inventory.group import Group +from ansible.module_utils.basic import json_dict_bytes_to_unicode + + +class InventoryScript: + ''' Host inventory parser for ansible using external inventory scripts. ''' + + def __init__(self, loader, filename=C.DEFAULT_HOST_LIST): + + self._loader = loader + + # Support inventory scripts that are not prefixed with some + # path information but happen to be in the current working + # directory when '.' is not in PATH. + self.filename = os.path.abspath(filename) + cmd = [ self.filename, "--list" ] + try: + sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError, e: + raise AnsibleError("problem running %s (%s)" % (' '.join(cmd), e)) + (stdout, stderr) = sp.communicate() + + if sp.returncode != 0: + raise AnsibleError("Inventory script (%s) had an execution error: %s " % (filename,stderr)) + + self.data = stdout + # see comment about _meta below + self.host_vars_from_top = None + self.groups = self._parse(stderr) + + + def _parse(self, err): + + all_hosts = {} + + # not passing from_remote because data from CMDB is trusted + self.raw = self._loader.load(self.data) + self.raw = json_dict_bytes_to_unicode(self.raw) + + all = Group('all') + groups = dict(all=all) + group = None + + + if 'failed' in self.raw: + sys.stderr.write(err + "\n") + raise AnsibleError("failed to parse executable inventory script results: %s" % self.raw) + + for (group_name, data) in self.raw.items(): + + # in Ansible 1.3 and later, a "_meta" subelement may contain + # a variable "hostvars" which contains a hash for each host + # if this "hostvars" exists at all then do not call --host for each + # host. This is for efficiency and scripts should still return data + # if called with --host for backwards compat with 1.2 and earlier. + + if group_name == '_meta': + if 'hostvars' in data: + self.host_vars_from_top = data['hostvars'] + continue + + if group_name != all.name: + group = groups[group_name] = Group(group_name) + else: + group = all + host = None + + if not isinstance(data, dict): + data = {'hosts': data} + # is not those subkeys, then simplified syntax, host with vars + elif not any(k in data for k in ('hosts','vars')): + data = {'hosts': [group_name], 'vars': data} + + if 'hosts' in data: + if not isinstance(data['hosts'], list): + raise AnsibleError("You defined a group \"%s\" with bad " + "data for the host list:\n %s" % (group_name, data)) + + for hostname in data['hosts']: + if not hostname in all_hosts: + all_hosts[hostname] = Host(hostname) + host = all_hosts[hostname] + group.add_host(host) + + if 'vars' in data: + if not isinstance(data['vars'], dict): + raise AnsibleError("You defined a group \"%s\" with bad " + "data for variables:\n %s" % (group_name, data)) + + for k, v in data['vars'].iteritems(): + if group.name == all.name: + all.set_variable(k, v) + else: + group.set_variable(k, v) + + # Separate loop to ensure all groups are defined + for (group_name, data) in self.raw.items(): + if group_name == '_meta': + continue + if isinstance(data, dict) and 'children' in data: + for child_name in data['children']: + if child_name in groups: + groups[group_name].add_child_group(groups[child_name]) + + for group in groups.values(): + if group.depth == 0 and group.name != 'all': + all.add_child_group(group) + + return groups + + def get_host_variables(self, host): + """ Runs <script> --host <hostname> to determine additional host variables """ + if self.host_vars_from_top is not None: + got = self.host_vars_from_top.get(host.name, {}) + return got + + + cmd = [self.filename, "--host", host.name] + try: + sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError, e: + raise AnsibleError("problem running %s (%s)" % (' '.join(cmd), e)) + (out, err) = sp.communicate() + if out.strip() == '': + return dict() + try: + return json_dict_bytes_to_unicode(self._loader.load(out)) + except ValueError: + raise AnsibleError("could not parse post variable response: %s, %s" % (cmd, out)) + diff --git a/v2/ansible/inventory/vars_plugins/__init__.py b/v2/ansible/inventory/vars_plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/v2/ansible/inventory/vars_plugins/__init__.py diff --git a/v2/ansible/inventory/vars_plugins/noop.py b/v2/ansible/inventory/vars_plugins/noop.py new file mode 100644 index 0000000000..5d4b4b6658 --- /dev/null +++ b/v2/ansible/inventory/vars_plugins/noop.py @@ -0,0 +1,48 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +class VarsModule(object): + + """ + Loads variables for groups and/or hosts + """ + + def __init__(self, inventory): + + """ constructor """ + + self.inventory = inventory + self.inventory_basedir = inventory.basedir() + + + def run(self, host, vault_password=None): + """ For backwards compatibility, when only vars per host were retrieved + This method should return both host specific vars as well as vars + calculated from groups it is a member of """ + return {} + + + def get_host_vars(self, host, vault_password=None): + """ Get host specific variables. """ + return {} + + + def get_group_vars(self, group, vault_password=None): + """ Get group specific variables. """ + return {} + diff --git a/v2/ansible/module_utils/__init__.py b/v2/ansible/module_utils/__init__.py new file mode 100644 index 0000000000..266d06a613 --- /dev/null +++ b/v2/ansible/module_utils/__init__.py @@ -0,0 +1,17 @@ +# 2013, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + diff --git a/v2/ansible/module_utils/a10.py b/v2/ansible/module_utils/a10.py new file mode 100644 index 0000000000..cfc217ee61 --- /dev/null +++ b/v2/ansible/module_utils/a10.py @@ -0,0 +1,103 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +AXAPI_PORT_PROTOCOLS = { + 'tcp': 2, + 'udp': 3, +} + +AXAPI_VPORT_PROTOCOLS = { + 'tcp': 2, + 'udp': 3, + 'fast-http': 9, + 'http': 11, + 'https': 12, +} + +def a10_argument_spec(): + return dict( + host=dict(type='str', required=True), + username=dict(type='str', aliases=['user', 'admin'], required=True), + password=dict(type='str', aliases=['pass', 'pwd'], required=True, no_log=True), + write_config=dict(type='bool', default=False) + ) + +def axapi_failure(result): + if 'response' in result and result['response'].get('status') == 'fail': + return True + return False + +def axapi_call(module, url, post=None): + ''' + Returns a datastructure based on the result of the API call + ''' + rsp, info = fetch_url(module, url, data=post) + if not rsp or info['status'] >= 400: + module.fail_json(msg="failed to connect (status code %s), error was %s" % (info['status'], info.get('msg', 'no error given'))) + try: + raw_data = rsp.read() + data = json.loads(raw_data) + except ValueError: + # at least one API call (system.action.write_config) returns + # XML even when JSON is requested, so do some minimal handling + # here to prevent failing even when the call succeeded + if 'status="ok"' in raw_data.lower(): + data = {"response": {"status": "OK"}} + else: + data = {"response": {"status": "fail", "err": {"msg": raw_data}}} + except: + module.fail_json(msg="could not read the result from the host") + finally: + rsp.close() + return data + +def axapi_authenticate(module, base_url, username, password): + url = '%s&method=authenticate&username=%s&password=%s' % (base_url, username, password) + result = axapi_call(module, url) + if axapi_failure(result): + return module.fail_json(msg=result['response']['err']['msg']) + sessid = result['session_id'] + return base_url + '&session_id=' + sessid + +def axapi_enabled_disabled(flag): + ''' + The axapi uses 0/1 integer values for flags, rather than strings + or booleans, so convert the given flag to a 0 or 1. For now, params + are specified as strings only so thats what we check. + ''' + if flag == 'enabled': + return 1 + else: + return 0 + +def axapi_get_port_protocol(protocol): + return AXAPI_PORT_PROTOCOLS.get(protocol.lower(), None) + +def axapi_get_vport_protocol(protocol): + return AXAPI_VPORT_PROTOCOLS.get(protocol.lower(), None) + diff --git a/v2/ansible/module_utils/basic.py b/v2/ansible/module_utils/basic.py new file mode 100644 index 0000000000..cd4d602453 --- /dev/null +++ b/v2/ansible/module_utils/basic.py @@ -0,0 +1,1573 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +# == BEGIN DYNAMICALLY INSERTED CODE == + +ANSIBLE_VERSION = "<<ANSIBLE_VERSION>>" + +MODULE_ARGS = "" +MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>" + +BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1] +BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0] +BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE + +# ansible modules can be written in any language. To simplify +# development of Python modules, the functions available here +# can be inserted in any module source automatically by including +# #<<INCLUDE_ANSIBLE_MODULE_COMMON>> on a blank line by itself inside +# of an ansible module. The source of this common code lives +# in lib/ansible/module_common.py + +import locale +import os +import re +import pipes +import shlex +import subprocess +import sys +import syslog +import types +import time +import select +import shutil +import stat +import tempfile +import traceback +import grp +import pwd +import platform +import errno +import tempfile + +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + sys.stderr.write('Error: ansible requires a json module, none found!') + sys.exit(1) + except SyntaxError: + sys.stderr.write('SyntaxError: probably due to json and python being for different versions') + sys.exit(1) + +HAVE_SELINUX=False +try: + import selinux + HAVE_SELINUX=True +except ImportError: + pass + +HAVE_HASHLIB=False +try: + from hashlib import sha1 as _sha1 + HAVE_HASHLIB=True +except ImportError: + from sha import sha as _sha1 + +try: + from hashlib import md5 as _md5 +except ImportError: + try: + from md5 import md5 as _md5 + except ImportError: + # MD5 unavailable. Possibly FIPS mode + _md5 = None + +try: + from hashlib import sha256 as _sha256 +except ImportError: + pass + +try: + from systemd import journal + has_journal = True +except ImportError: + import syslog + has_journal = False + +try: + from ast import literal_eval as _literal_eval +except ImportError: + # a replacement for literal_eval that works with python 2.4. from: + # https://mail.python.org/pipermail/python-list/2009-September/551880.html + # which is essentially a cut/past from an earlier (2.6) version of python's + # ast.py + from compiler import parse + from compiler.ast import * + def _literal_eval(node_or_string): + """ + Safely evaluate an expression node or a string containing a Python + expression. The string or node provided may only consist of the following + Python literal structures: strings, numbers, tuples, lists, dicts, booleans, + and None. + """ + _safe_names = {'None': None, 'True': True, 'False': False} + if isinstance(node_or_string, basestring): + node_or_string = parse(node_or_string, mode='eval') + if isinstance(node_or_string, Expression): + node_or_string = node_or_string.node + def _convert(node): + if isinstance(node, Const) and isinstance(node.value, (basestring, int, float, long, complex)): + return node.value + elif isinstance(node, Tuple): + return tuple(map(_convert, node.nodes)) + elif isinstance(node, List): + return list(map(_convert, node.nodes)) + elif isinstance(node, Dict): + return dict((_convert(k), _convert(v)) for k, v in node.items) + elif isinstance(node, Name): + if node.name in _safe_names: + return _safe_names[node.name] + elif isinstance(node, UnarySub): + return -_convert(node.expr) + raise ValueError('malformed string') + return _convert(node_or_string) + +FILE_COMMON_ARGUMENTS=dict( + src = dict(), + mode = dict(), + owner = dict(), + group = dict(), + seuser = dict(), + serole = dict(), + selevel = dict(), + setype = dict(), + follow = dict(type='bool', default=False), + # not taken by the file module, but other modules call file so it must ignore them. + content = dict(no_log=True), + backup = dict(), + force = dict(), + remote_src = dict(), # used by assemble + regexp = dict(), # used by assemble + delimiter = dict(), # used by assemble + directory_mode = dict(), # used by copy +) + + +def get_platform(): + ''' what's the platform? example: Linux is a platform. ''' + return platform.system() + +def get_distribution(): + ''' return the distribution name ''' + if platform.system() == 'Linux': + try: + distribution = platform.linux_distribution()[0].capitalize() + if not distribution and os.path.isfile('/etc/system-release'): + distribution = platform.linux_distribution(supported_dists=['system'])[0].capitalize() + if 'Amazon' in distribution: + distribution = 'Amazon' + else: + distribution = 'OtherLinux' + except: + # FIXME: MethodMissing, I assume? + distribution = platform.dist()[0].capitalize() + else: + distribution = None + return distribution + +def get_distribution_version(): + ''' return the distribution version ''' + if platform.system() == 'Linux': + try: + distribution_version = platform.linux_distribution()[1] + if not distribution_version and os.path.isfile('/etc/system-release'): + distribution_version = platform.linux_distribution(supported_dists=['system'])[1] + except: + # FIXME: MethodMissing, I assume? + distribution_version = platform.dist()[1] + else: + distribution_version = None + return distribution_version + +def load_platform_subclass(cls, *args, **kwargs): + ''' + used by modules like User to have different implementations based on detected platform. See User + module for an example. + ''' + + this_platform = get_platform() + distribution = get_distribution() + subclass = None + + # get the most specific superclass for this platform + if distribution is not None: + for sc in cls.__subclasses__(): + if sc.distribution is not None and sc.distribution == distribution and sc.platform == this_platform: + subclass = sc + if subclass is None: + for sc in cls.__subclasses__(): + if sc.platform == this_platform and sc.distribution is None: + subclass = sc + if subclass is None: + subclass = cls + + return super(cls, subclass).__new__(subclass) + + +def json_dict_unicode_to_bytes(d): + ''' Recursively convert dict keys and values to byte str + + Specialized for json return because this only handles, lists, tuples, + and dict container types (the containers that the json module returns) + ''' + + if isinstance(d, unicode): + return d.encode('utf-8') + elif isinstance(d, dict): + return dict(map(json_dict_unicode_to_bytes, d.iteritems())) + elif isinstance(d, list): + return list(map(json_dict_unicode_to_bytes, d)) + elif isinstance(d, tuple): + return tuple(map(json_dict_unicode_to_bytes, d)) + else: + return d + +def json_dict_bytes_to_unicode(d): + ''' Recursively convert dict keys and values to byte str + + Specialized for json return because this only handles, lists, tuples, + and dict container types (the containers that the json module returns) + ''' + + if isinstance(d, str): + return unicode(d, 'utf-8') + elif isinstance(d, dict): + return dict(map(json_dict_bytes_to_unicode, d.iteritems())) + elif isinstance(d, list): + return list(map(json_dict_bytes_to_unicode, d)) + elif isinstance(d, tuple): + return tuple(map(json_dict_bytes_to_unicode, d)) + else: + return d + + +class AnsibleModule(object): + + def __init__(self, argument_spec, bypass_checks=False, no_log=False, + check_invalid_arguments=True, mutually_exclusive=None, required_together=None, + required_one_of=None, add_file_common_args=False, supports_check_mode=False): + + ''' + common code for quickly building an ansible module in Python + (although you can write modules in anything that can return JSON) + see library/* for examples + ''' + + self.argument_spec = argument_spec + self.supports_check_mode = supports_check_mode + self.check_mode = False + self.no_log = no_log + self.cleanup_files = [] + + self.aliases = {} + + if add_file_common_args: + for k, v in FILE_COMMON_ARGUMENTS.iteritems(): + if k not in self.argument_spec: + self.argument_spec[k] = v + + # check the locale as set by the current environment, and + # reset to LANG=C if it's an invalid/unavailable locale + self._check_locale() + + self.params = self._load_params() + + self._legal_inputs = ['_ansible_check_mode', '_ansible_no_log'] + + self.aliases = self._handle_aliases() + + if check_invalid_arguments: + self._check_invalid_arguments() + self._check_for_check_mode() + self._check_for_no_log() + + # check exclusive early + if not bypass_checks: + self._check_mutually_exclusive(mutually_exclusive) + + self._set_defaults(pre=True) + + if not bypass_checks: + self._check_required_arguments() + self._check_argument_values() + self._check_argument_types() + self._check_required_together(required_together) + self._check_required_one_of(required_one_of) + + self._set_defaults(pre=False) + if not self.no_log: + self._log_invocation() + + # finally, make sure we're in a sane working dir + self._set_cwd() + + def load_file_common_arguments(self, params): + ''' + many modules deal with files, this encapsulates common + options that the file module accepts such that it is directly + available to all modules and they can share code. + ''' + + path = params.get('path', params.get('dest', None)) + if path is None: + return {} + else: + path = os.path.expanduser(path) + + # if the path is a symlink, and we're following links, get + # the target of the link instead for testing + if params.get('follow', False) and os.path.islink(path): + path = os.path.realpath(path) + + mode = params.get('mode', None) + owner = params.get('owner', None) + group = params.get('group', None) + + # selinux related options + seuser = params.get('seuser', None) + serole = params.get('serole', None) + setype = params.get('setype', None) + selevel = params.get('selevel', None) + secontext = [seuser, serole, setype] + + if self.selinux_mls_enabled(): + secontext.append(selevel) + + default_secontext = self.selinux_default_context(path) + for i in range(len(default_secontext)): + if i is not None and secontext[i] == '_default': + secontext[i] = default_secontext[i] + + return dict( + path=path, mode=mode, owner=owner, group=group, + seuser=seuser, serole=serole, setype=setype, + selevel=selevel, secontext=secontext, + ) + + + # Detect whether using selinux that is MLS-aware. + # While this means you can set the level/range with + # selinux.lsetfilecon(), it may or may not mean that you + # will get the selevel as part of the context returned + # by selinux.lgetfilecon(). + + def selinux_mls_enabled(self): + if not HAVE_SELINUX: + return False + if selinux.is_selinux_mls_enabled() == 1: + return True + else: + return False + + def selinux_enabled(self): + if not HAVE_SELINUX: + seenabled = self.get_bin_path('selinuxenabled') + if seenabled is not None: + (rc,out,err) = self.run_command(seenabled) + if rc == 0: + self.fail_json(msg="Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!") + return False + if selinux.is_selinux_enabled() == 1: + return True + else: + return False + + # Determine whether we need a placeholder for selevel/mls + def selinux_initial_context(self): + context = [None, None, None] + if self.selinux_mls_enabled(): + context.append(None) + return context + + def _to_filesystem_str(self, path): + '''Returns filesystem path as a str, if it wasn't already. + + Used in selinux interactions because it cannot accept unicode + instances, and specifying complex args in a playbook leaves + you with unicode instances. This method currently assumes + that your filesystem encoding is UTF-8. + + ''' + if isinstance(path, unicode): + path = path.encode("utf-8") + return path + + # If selinux fails to find a default, return an array of None + def selinux_default_context(self, path, mode=0): + context = self.selinux_initial_context() + if not HAVE_SELINUX or not self.selinux_enabled(): + return context + try: + ret = selinux.matchpathcon(self._to_filesystem_str(path), mode) + except OSError: + return context + if ret[0] == -1: + return context + # Limit split to 4 because the selevel, the last in the list, + # may contain ':' characters + context = ret[1].split(':', 3) + return context + + def selinux_context(self, path): + context = self.selinux_initial_context() + if not HAVE_SELINUX or not self.selinux_enabled(): + return context + try: + ret = selinux.lgetfilecon_raw(self._to_filesystem_str(path)) + except OSError, e: + if e.errno == errno.ENOENT: + self.fail_json(path=path, msg='path %s does not exist' % path) + else: + self.fail_json(path=path, msg='failed to retrieve selinux context') + if ret[0] == -1: + return context + # Limit split to 4 because the selevel, the last in the list, + # may contain ':' characters + context = ret[1].split(':', 3) + return context + + def user_and_group(self, filename): + filename = os.path.expanduser(filename) + st = os.lstat(filename) + uid = st.st_uid + gid = st.st_gid + return (uid, gid) + + def find_mount_point(self, path): + path = os.path.abspath(os.path.expanduser(os.path.expandvars(path))) + while not os.path.ismount(path): + path = os.path.dirname(path) + return path + + def is_nfs_path(self, path): + """ + Returns a tuple containing (True, selinux_context) if the given path + is on a NFS mount point, otherwise the return will be (False, None). + """ + try: + f = open('/proc/mounts', 'r') + mount_data = f.readlines() + f.close() + except: + return (False, None) + path_mount_point = self.find_mount_point(path) + for line in mount_data: + (device, mount_point, fstype, options, rest) = line.split(' ', 4) + if path_mount_point == mount_point and 'nfs' in fstype: + nfs_context = self.selinux_context(path_mount_point) + return (True, nfs_context) + return (False, None) + + def set_default_selinux_context(self, path, changed): + if not HAVE_SELINUX or not self.selinux_enabled(): + return changed + context = self.selinux_default_context(path) + return self.set_context_if_different(path, context, False) + + def set_context_if_different(self, path, context, changed): + + if not HAVE_SELINUX or not self.selinux_enabled(): + return changed + cur_context = self.selinux_context(path) + new_context = list(cur_context) + # Iterate over the current context instead of the + # argument context, which may have selevel. + + (is_nfs, nfs_context) = self.is_nfs_path(path) + if is_nfs: + new_context = nfs_context + else: + for i in range(len(cur_context)): + if len(context) > i: + if context[i] is not None and context[i] != cur_context[i]: + new_context[i] = context[i] + if context[i] is None: + new_context[i] = cur_context[i] + + if cur_context != new_context: + try: + if self.check_mode: + return True + rc = selinux.lsetfilecon(self._to_filesystem_str(path), + str(':'.join(new_context))) + except OSError: + self.fail_json(path=path, msg='invalid selinux context', new_context=new_context, cur_context=cur_context, input_was=context) + if rc != 0: + self.fail_json(path=path, msg='set selinux context failed') + changed = True + return changed + + def set_owner_if_different(self, path, owner, changed): + path = os.path.expanduser(path) + if owner is None: + return changed + orig_uid, orig_gid = self.user_and_group(path) + try: + uid = int(owner) + except ValueError: + try: + uid = pwd.getpwnam(owner).pw_uid + except KeyError: + self.fail_json(path=path, msg='chown failed: failed to look up user %s' % owner) + if orig_uid != uid: + if self.check_mode: + return True + try: + os.lchown(path, uid, -1) + except OSError: + self.fail_json(path=path, msg='chown failed') + changed = True + return changed + + def set_group_if_different(self, path, group, changed): + path = os.path.expanduser(path) + if group is None: + return changed + orig_uid, orig_gid = self.user_and_group(path) + try: + gid = int(group) + except ValueError: + try: + gid = grp.getgrnam(group).gr_gid + except KeyError: + self.fail_json(path=path, msg='chgrp failed: failed to look up group %s' % group) + if orig_gid != gid: + if self.check_mode: + return True + try: + os.lchown(path, -1, gid) + except OSError: + self.fail_json(path=path, msg='chgrp failed') + changed = True + return changed + + def set_mode_if_different(self, path, mode, changed): + path = os.path.expanduser(path) + path_stat = os.lstat(path) + + if mode is None: + return changed + + if not isinstance(mode, int): + try: + mode = int(mode, 8) + except Exception: + try: + mode = self._symbolic_mode_to_octal(path_stat, mode) + except Exception, e: + self.fail_json(path=path, + msg="mode must be in octal or symbolic form", + details=str(e)) + + prev_mode = stat.S_IMODE(path_stat.st_mode) + + if prev_mode != mode: + if self.check_mode: + return True + # FIXME: comparison against string above will cause this to be executed + # every time + try: + if 'lchmod' in dir(os): + os.lchmod(path, mode) + else: + os.chmod(path, mode) + except OSError, e: + if os.path.islink(path) and e.errno == errno.EPERM: # Can't set mode on symbolic links + pass + elif e.errno == errno.ENOENT: # Can't set mode on broken symbolic links + pass + else: + raise e + except Exception, e: + self.fail_json(path=path, msg='chmod failed', details=str(e)) + + path_stat = os.lstat(path) + new_mode = stat.S_IMODE(path_stat.st_mode) + + if new_mode != prev_mode: + changed = True + return changed + + def _symbolic_mode_to_octal(self, path_stat, symbolic_mode): + new_mode = stat.S_IMODE(path_stat.st_mode) + + mode_re = re.compile(r'^(?P<users>[ugoa]+)(?P<operator>[-+=])(?P<perms>[rwxXst]*|[ugo])$') + for mode in symbolic_mode.split(','): + match = mode_re.match(mode) + if match: + users = match.group('users') + operator = match.group('operator') + perms = match.group('perms') + + if users == 'a': users = 'ugo' + + for user in users: + mode_to_apply = self._get_octal_mode_from_symbolic_perms(path_stat, user, perms) + new_mode = self._apply_operation_to_mode(user, operator, mode_to_apply, new_mode) + else: + raise ValueError("bad symbolic permission for mode: %s" % mode) + return new_mode + + def _apply_operation_to_mode(self, user, operator, mode_to_apply, current_mode): + if operator == '=': + if user == 'u': mask = stat.S_IRWXU | stat.S_ISUID + elif user == 'g': mask = stat.S_IRWXG | stat.S_ISGID + elif user == 'o': mask = stat.S_IRWXO | stat.S_ISVTX + + # mask out u, g, or o permissions from current_mode and apply new permissions + inverse_mask = mask ^ 07777 + new_mode = (current_mode & inverse_mask) | mode_to_apply + elif operator == '+': + new_mode = current_mode | mode_to_apply + elif operator == '-': + new_mode = current_mode - (current_mode & mode_to_apply) + return new_mode + + def _get_octal_mode_from_symbolic_perms(self, path_stat, user, perms): + prev_mode = stat.S_IMODE(path_stat.st_mode) + + is_directory = stat.S_ISDIR(path_stat.st_mode) + has_x_permissions = (prev_mode & 00111) > 0 + apply_X_permission = is_directory or has_x_permissions + + # Permission bits constants documented at: + # http://docs.python.org/2/library/stat.html#stat.S_ISUID + if apply_X_permission: + X_perms = { + 'u': {'X': stat.S_IXUSR}, + 'g': {'X': stat.S_IXGRP}, + 'o': {'X': stat.S_IXOTH} + } + else: + X_perms = { + 'u': {'X': 0}, + 'g': {'X': 0}, + 'o': {'X': 0} + } + + user_perms_to_modes = { + 'u': { + 'r': stat.S_IRUSR, + 'w': stat.S_IWUSR, + 'x': stat.S_IXUSR, + 's': stat.S_ISUID, + 't': 0, + 'u': prev_mode & stat.S_IRWXU, + 'g': (prev_mode & stat.S_IRWXG) << 3, + 'o': (prev_mode & stat.S_IRWXO) << 6 }, + 'g': { + 'r': stat.S_IRGRP, + 'w': stat.S_IWGRP, + 'x': stat.S_IXGRP, + 's': stat.S_ISGID, + 't': 0, + 'u': (prev_mode & stat.S_IRWXU) >> 3, + 'g': prev_mode & stat.S_IRWXG, + 'o': (prev_mode & stat.S_IRWXO) << 3 }, + 'o': { + 'r': stat.S_IROTH, + 'w': stat.S_IWOTH, + 'x': stat.S_IXOTH, + 's': 0, + 't': stat.S_ISVTX, + 'u': (prev_mode & stat.S_IRWXU) >> 6, + 'g': (prev_mode & stat.S_IRWXG) >> 3, + 'o': prev_mode & stat.S_IRWXO } + } + + # Insert X_perms into user_perms_to_modes + for key, value in X_perms.items(): + user_perms_to_modes[key].update(value) + + or_reduce = lambda mode, perm: mode | user_perms_to_modes[user][perm] + return reduce(or_reduce, perms, 0) + + def set_fs_attributes_if_different(self, file_args, changed): + # set modes owners and context as needed + changed = self.set_context_if_different( + file_args['path'], file_args['secontext'], changed + ) + changed = self.set_owner_if_different( + file_args['path'], file_args['owner'], changed + ) + changed = self.set_group_if_different( + file_args['path'], file_args['group'], changed + ) + changed = self.set_mode_if_different( + file_args['path'], file_args['mode'], changed + ) + return changed + + def set_directory_attributes_if_different(self, file_args, changed): + return self.set_fs_attributes_if_different(file_args, changed) + + def set_file_attributes_if_different(self, file_args, changed): + return self.set_fs_attributes_if_different(file_args, changed) + + def add_path_info(self, kwargs): + ''' + for results that are files, supplement the info about the file + in the return path with stats about the file path. + ''' + + path = kwargs.get('path', kwargs.get('dest', None)) + if path is None: + return kwargs + if os.path.exists(path): + (uid, gid) = self.user_and_group(path) + kwargs['uid'] = uid + kwargs['gid'] = gid + try: + user = pwd.getpwuid(uid)[0] + except KeyError: + user = str(uid) + try: + group = grp.getgrgid(gid)[0] + except KeyError: + group = str(gid) + kwargs['owner'] = user + kwargs['group'] = group + st = os.lstat(path) + kwargs['mode'] = oct(stat.S_IMODE(st[stat.ST_MODE])) + # secontext not yet supported + if os.path.islink(path): + kwargs['state'] = 'link' + elif os.path.isdir(path): + kwargs['state'] = 'directory' + elif os.stat(path).st_nlink > 1: + kwargs['state'] = 'hard' + else: + kwargs['state'] = 'file' + if HAVE_SELINUX and self.selinux_enabled(): + kwargs['secontext'] = ':'.join(self.selinux_context(path)) + kwargs['size'] = st[stat.ST_SIZE] + else: + kwargs['state'] = 'absent' + return kwargs + + def _check_locale(self): + ''' + Uses the locale module to test the currently set locale + (per the LANG and LC_CTYPE environment settings) + ''' + try: + # setting the locale to '' uses the default locale + # as it would be returned by locale.getdefaultlocale() + locale.setlocale(locale.LC_ALL, '') + except locale.Error, e: + # fallback to the 'C' locale, which may cause unicode + # issues but is preferable to simply failing because + # of an unknown locale + locale.setlocale(locale.LC_ALL, 'C') + os.environ['LANG'] = 'C' + os.environ['LC_CTYPE'] = 'C' + except Exception, e: + self.fail_json(msg="An unknown error was encountered while attempting to validate the locale: %s" % e) + + def _handle_aliases(self): + aliases_results = {} #alias:canon + for (k,v) in self.argument_spec.iteritems(): + self._legal_inputs.append(k) + aliases = v.get('aliases', None) + default = v.get('default', None) + required = v.get('required', False) + if default is not None and required: + # not alias specific but this is a good place to check this + self.fail_json(msg="internal error: required and default are mutually exclusive for %s" % k) + if aliases is None: + continue + if type(aliases) != list: + self.fail_json(msg='internal error: aliases must be a list') + for alias in aliases: + self._legal_inputs.append(alias) + aliases_results[alias] = k + if alias in self.params: + self.params[k] = self.params[alias] + + return aliases_results + + def _check_for_check_mode(self): + for (k,v) in self.params.iteritems(): + if k == '_ansible_check_mode': + if not self.supports_check_mode: + self.exit_json(skipped=True, msg="remote module does not support check mode") + if self.supports_check_mode: + self.check_mode = True + + def _check_for_no_log(self): + for (k,v) in self.params.iteritems(): + if k == '_ansible_no_log': + self.no_log = self.boolean(v) + + def _check_invalid_arguments(self): + for (k,v) in self.params.iteritems(): + # these should be in legal inputs already + #if k in ('_ansible_check_mode', '_ansible_no_log'): + # continue + if k not in self._legal_inputs: + self.fail_json(msg="unsupported parameter for module: %s" % k) + + def _count_terms(self, check): + count = 0 + for term in check: + if term in self.params: + count += 1 + return count + + def _check_mutually_exclusive(self, spec): + if spec is None: + return + for check in spec: + count = self._count_terms(check) + if count > 1: + self.fail_json(msg="parameters are mutually exclusive: %s" % check) + + def _check_required_one_of(self, spec): + if spec is None: + return + for check in spec: + count = self._count_terms(check) + if count == 0: + self.fail_json(msg="one of the following is required: %s" % ','.join(check)) + + def _check_required_together(self, spec): + if spec is None: + return + for check in spec: + counts = [ self._count_terms([field]) for field in check ] + non_zero = [ c for c in counts if c > 0 ] + if len(non_zero) > 0: + if 0 in counts: + self.fail_json(msg="parameters are required together: %s" % check) + + def _check_required_arguments(self): + ''' ensure all required arguments are present ''' + missing = [] + for (k,v) in self.argument_spec.iteritems(): + required = v.get('required', False) + if required and k not in self.params: + missing.append(k) + if len(missing) > 0: + self.fail_json(msg="missing required arguments: %s" % ",".join(missing)) + + def _check_argument_values(self): + ''' ensure all arguments have the requested values, and there are no stray arguments ''' + for (k,v) in self.argument_spec.iteritems(): + choices = v.get('choices',None) + if choices is None: + continue + if type(choices) == list: + if k in self.params: + if self.params[k] not in choices: + choices_str=",".join([str(c) for c in choices]) + msg="value of %s must be one of: %s, got: %s" % (k, choices_str, self.params[k]) + self.fail_json(msg=msg) + else: + self.fail_json(msg="internal error: do not know how to interpret argument_spec") + + def safe_eval(self, str, locals=None, include_exceptions=False): + + # do not allow method calls to modules + if not isinstance(str, basestring): + # already templated to a datastructure, perhaps? + if include_exceptions: + return (str, None) + return str + if re.search(r'\w\.\w+\(', str): + if include_exceptions: + return (str, None) + return str + # do not allow imports + if re.search(r'import \w+', str): + if include_exceptions: + return (str, None) + return str + try: + result = None + if not locals: + result = _literal_eval(str) + else: + result = _literal_eval(str, None, locals) + if include_exceptions: + return (result, None) + else: + return result + except Exception, e: + if include_exceptions: + return (str, e) + return str + + def _check_argument_types(self): + ''' ensure all arguments have the requested type ''' + for (k, v) in self.argument_spec.iteritems(): + wanted = v.get('type', None) + if wanted is None: + continue + if k not in self.params: + continue + + value = self.params[k] + is_invalid = False + + if wanted == 'str': + if not isinstance(value, basestring): + self.params[k] = str(value) + elif wanted == 'list': + if not isinstance(value, list): + if isinstance(value, basestring): + self.params[k] = value.split(",") + elif isinstance(value, int) or isinstance(value, float): + self.params[k] = [ str(value) ] + else: + is_invalid = True + elif wanted == 'dict': + if not isinstance(value, dict): + if isinstance(value, basestring): + if value.startswith("{"): + try: + self.params[k] = json.loads(value) + except: + (result, exc) = self.safe_eval(value, dict(), include_exceptions=True) + if exc is not None: + self.fail_json(msg="unable to evaluate dictionary for %s" % k) + self.params[k] = result + elif '=' in value: + self.params[k] = dict([x.strip().split("=", 1) for x in value.split(",")]) + else: + self.fail_json(msg="dictionary requested, could not parse JSON or key=value") + else: + is_invalid = True + elif wanted == 'bool': + if not isinstance(value, bool): + if isinstance(value, basestring): + self.params[k] = self.boolean(value) + else: + is_invalid = True + elif wanted == 'int': + if not isinstance(value, int): + if isinstance(value, basestring): + self.params[k] = int(value) + else: + is_invalid = True + elif wanted == 'float': + if not isinstance(value, float): + if isinstance(value, basestring): + self.params[k] = float(value) + else: + is_invalid = True + else: + self.fail_json(msg="implementation error: unknown type %s requested for %s" % (wanted, k)) + + if is_invalid: + self.fail_json(msg="argument %s is of invalid type: %s, required: %s" % (k, type(value), wanted)) + + def _set_defaults(self, pre=True): + for (k,v) in self.argument_spec.iteritems(): + default = v.get('default', None) + if pre == True: + # this prevents setting defaults on required items + if default is not None and k not in self.params: + self.params[k] = default + else: + # make sure things without a default still get set None + if k not in self.params: + self.params[k] = default + + def _load_params(self): + ''' read the input and return a dictionary and the arguments string ''' + params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS)) + if params is None: + params = dict() + return params + + def _heuristic_log_sanitize(self, data): + ''' Remove strings that look like passwords from log messages ''' + # Currently filters: + # user:pass@foo/whatever and http://username:pass@wherever/foo + # This code has false positives and consumes parts of logs that are + # not passwds + + # begin: start of a passwd containing string + # end: end of a passwd containing string + # sep: char between user and passwd + # prev_begin: where in the overall string to start a search for + # a passwd + # sep_search_end: where in the string to end a search for the sep + output = [] + begin = len(data) + prev_begin = begin + sep = 1 + while sep: + # Find the potential end of a passwd + try: + end = data.rindex('@', 0, begin) + except ValueError: + # No passwd in the rest of the data + output.insert(0, data[0:begin]) + break + + # Search for the beginning of a passwd + sep = None + sep_search_end = end + while not sep: + # URL-style username+password + try: + begin = data.rindex('://', 0, sep_search_end) + except ValueError: + # No url style in the data, check for ssh style in the + # rest of the string + begin = 0 + # Search for separator + try: + sep = data.index(':', begin + 3, end) + except ValueError: + # No separator; choices: + if begin == 0: + # Searched the whole string so there's no password + # here. Return the remaining data + output.insert(0, data[0:begin]) + break + # Search for a different beginning of the password field. + sep_search_end = begin + continue + if sep: + # Password was found; remove it. + output.insert(0, data[end:prev_begin]) + output.insert(0, '********') + output.insert(0, data[begin:sep + 1]) + prev_begin = begin + + return ''.join(output) + + def _log_invocation(self): + ''' log that ansible ran the module ''' + # TODO: generalize a separate log function and make log_invocation use it + # Sanitize possible password argument when logging. + log_args = dict() + passwd_keys = ['password', 'login_password'] + + for param in self.params: + canon = self.aliases.get(param, param) + arg_opts = self.argument_spec.get(canon, {}) + no_log = arg_opts.get('no_log', False) + + if self.boolean(no_log): + log_args[param] = 'NOT_LOGGING_PARAMETER' + elif param in passwd_keys: + log_args[param] = 'NOT_LOGGING_PASSWORD' + else: + param_val = self.params[param] + if not isinstance(param_val, basestring): + param_val = str(param_val) + elif isinstance(param_val, unicode): + param_val = param_val.encode('utf-8') + log_args[param] = self._heuristic_log_sanitize(param_val) + + module = 'ansible-%s' % os.path.basename(__file__) + msg = [] + for arg in log_args: + arg_val = log_args[arg] + if not isinstance(arg_val, basestring): + arg_val = str(arg_val) + elif isinstance(arg_val, unicode): + arg_val = arg_val.encode('utf-8') + msg.append('%s=%s ' % (arg, arg_val)) + if msg: + msg = 'Invoked with %s' % ''.join(msg) + else: + msg = 'Invoked' + + # 6655 - allow for accented characters + if isinstance(msg, unicode): + # We should never get here as msg should be type str, not unicode + msg = msg.encode('utf-8') + + if (has_journal): + journal_args = [("MODULE", os.path.basename(__file__))] + for arg in log_args: + journal_args.append((arg.upper(), str(log_args[arg]))) + try: + journal.send("%s %s" % (module, msg), **dict(journal_args)) + except IOError, e: + # fall back to syslog since logging to journal failed + syslog.openlog(str(module), 0, syslog.LOG_USER) + syslog.syslog(syslog.LOG_NOTICE, msg) #1 + else: + syslog.openlog(str(module), 0, syslog.LOG_USER) + syslog.syslog(syslog.LOG_NOTICE, msg) #2 + + def _set_cwd(self): + try: + cwd = os.getcwd() + if not os.access(cwd, os.F_OK|os.R_OK): + raise + return cwd + except: + # we don't have access to the cwd, probably because of sudo. + # Try and move to a neutral location to prevent errors + for cwd in [os.path.expandvars('$HOME'), tempfile.gettempdir()]: + try: + if os.access(cwd, os.F_OK|os.R_OK): + os.chdir(cwd) + return cwd + except: + pass + # we won't error here, as it may *not* be a problem, + # and we don't want to break modules unnecessarily + return None + + def get_bin_path(self, arg, required=False, opt_dirs=[]): + ''' + find system executable in PATH. + Optional arguments: + - required: if executable is not found and required is true, fail_json + - opt_dirs: optional list of directories to search in addition to PATH + if found return full path; otherwise return None + ''' + sbin_paths = ['/sbin', '/usr/sbin', '/usr/local/sbin'] + paths = [] + for d in opt_dirs: + if d is not None and os.path.exists(d): + paths.append(d) + paths += os.environ.get('PATH', '').split(os.pathsep) + bin_path = None + # mangle PATH to include /sbin dirs + for p in sbin_paths: + if p not in paths and os.path.exists(p): + paths.append(p) + for d in paths: + path = os.path.join(d, arg) + if os.path.exists(path) and self.is_executable(path): + bin_path = path + break + if required and bin_path is None: + self.fail_json(msg='Failed to find required executable %s' % arg) + return bin_path + + def boolean(self, arg): + ''' return a bool for the arg ''' + if arg is None or type(arg) == bool: + return arg + if type(arg) in types.StringTypes: + arg = arg.lower() + if arg in BOOLEANS_TRUE: + return True + elif arg in BOOLEANS_FALSE: + return False + else: + self.fail_json(msg='Boolean %s not in either boolean list' % arg) + + def jsonify(self, data): + for encoding in ("utf-8", "latin-1", "unicode_escape"): + try: + return json.dumps(data, encoding=encoding) + # Old systems using simplejson module does not support encoding keyword. + except TypeError, e: + return json.dumps(data) + except UnicodeDecodeError, e: + continue + self.fail_json(msg='Invalid unicode encoding encountered') + + def from_json(self, data): + return json.loads(data) + + def add_cleanup_file(self, path): + if path not in self.cleanup_files: + self.cleanup_files.append(path) + + def do_cleanup_files(self): + for path in self.cleanup_files: + self.cleanup(path) + + def exit_json(self, **kwargs): + ''' return from the module, without error ''' + self.add_path_info(kwargs) + if not 'changed' in kwargs: + kwargs['changed'] = False + self.do_cleanup_files() + print self.jsonify(kwargs) + sys.exit(0) + + def fail_json(self, **kwargs): + ''' return from the module, with an error message ''' + self.add_path_info(kwargs) + assert 'msg' in kwargs, "implementation error -- msg to explain the error is required" + kwargs['failed'] = True + self.do_cleanup_files() + print self.jsonify(kwargs) + sys.exit(1) + + def is_executable(self, path): + '''is the given path executable?''' + return (stat.S_IXUSR & os.stat(path)[stat.ST_MODE] + or stat.S_IXGRP & os.stat(path)[stat.ST_MODE] + or stat.S_IXOTH & os.stat(path)[stat.ST_MODE]) + + def digest_from_file(self, filename, digest_method): + ''' Return hex digest of local file for a given digest_method, or None if file is not present. ''' + if not os.path.exists(filename): + return None + if os.path.isdir(filename): + self.fail_json(msg="attempted to take checksum of directory: %s" % filename) + digest = digest_method + blocksize = 64 * 1024 + infile = open(filename, 'rb') + block = infile.read(blocksize) + while block: + digest.update(block) + block = infile.read(blocksize) + infile.close() + return digest.hexdigest() + + def md5(self, filename): + ''' Return MD5 hex digest of local file using digest_from_file(). + + Do not use this function unless you have no other choice for: + 1) Optional backwards compatibility + 2) Compatibility with a third party protocol + + This function will not work on systems complying with FIPS-140-2. + + Most uses of this function can use the module.sha1 function instead. + ''' + if not _md5: + raise ValueError('MD5 not available. Possibly running in FIPS mode') + return self.digest_from_file(filename, _md5()) + + def sha1(self, filename): + ''' Return SHA1 hex digest of local file using digest_from_file(). ''' + return self.digest_from_file(filename, _sha1()) + + def sha256(self, filename): + ''' Return SHA-256 hex digest of local file using digest_from_file(). ''' + if not HAVE_HASHLIB: + self.fail_json(msg="SHA-256 checksums require hashlib, which is available in Python 2.5 and higher") + return self.digest_from_file(filename, _sha256()) + + def backup_local(self, fn): + '''make a date-marked backup of the specified file, return True or False on success or failure''' + # backups named basename-YYYY-MM-DD@HH:MM~ + ext = time.strftime("%Y-%m-%d@%H:%M~", time.localtime(time.time())) + backupdest = '%s.%s' % (fn, ext) + + try: + shutil.copy2(fn, backupdest) + except shutil.Error, e: + self.fail_json(msg='Could not make backup of %s to %s: %s' % (fn, backupdest, e)) + return backupdest + + def cleanup(self, tmpfile): + if os.path.exists(tmpfile): + try: + os.unlink(tmpfile) + except OSError, e: + sys.stderr.write("could not cleanup %s: %s" % (tmpfile, e)) + + def atomic_move(self, src, dest): + '''atomically move src to dest, copying attributes from dest, returns true on success + it uses os.rename to ensure this as it is an atomic operation, rest of the function is + to work around limitations, corner cases and ensure selinux context is saved if possible''' + context = None + dest_stat = None + if os.path.exists(dest): + try: + dest_stat = os.stat(dest) + os.chmod(src, dest_stat.st_mode & 07777) + os.chown(src, dest_stat.st_uid, dest_stat.st_gid) + except OSError, e: + if e.errno != errno.EPERM: + raise + if self.selinux_enabled(): + context = self.selinux_context(dest) + else: + if self.selinux_enabled(): + context = self.selinux_default_context(dest) + + creating = not os.path.exists(dest) + + try: + login_name = os.getlogin() + except OSError: + # not having a tty can cause the above to fail, so + # just get the LOGNAME environment variable instead + login_name = os.environ.get('LOGNAME', None) + + # if the original login_name doesn't match the currently + # logged-in user, or if the SUDO_USER environment variable + # is set, then this user has switched their credentials + switched_user = login_name and login_name != pwd.getpwuid(os.getuid())[0] or os.environ.get('SUDO_USER') + + try: + # Optimistically try a rename, solves some corner cases and can avoid useless work, throws exception if not atomic. + os.rename(src, dest) + except (IOError,OSError), e: + # only try workarounds for errno 18 (cross device), 1 (not permitted) and 13 (permission denied) + if e.errno != errno.EPERM and e.errno != errno.EXDEV and e.errno != errno.EACCES: + self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e)) + + dest_dir = os.path.dirname(dest) + dest_file = os.path.basename(dest) + try: + tmp_dest = tempfile.NamedTemporaryFile( + prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file) + except (OSError, IOError), e: + self.fail_json(msg='The destination directory (%s) is not writable by the current user.' % dest_dir) + + try: # leaves tmp file behind when sudo and not root + if switched_user and os.getuid() != 0: + # cleanup will happen by 'rm' of tempdir + # copy2 will preserve some metadata + shutil.copy2(src, tmp_dest.name) + else: + shutil.move(src, tmp_dest.name) + if self.selinux_enabled(): + self.set_context_if_different( + tmp_dest.name, context, False) + try: + tmp_stat = os.stat(tmp_dest.name) + if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid): + os.chown(tmp_dest.name, dest_stat.st_uid, dest_stat.st_gid) + except OSError, e: + if e.errno != errno.EPERM: + raise + os.rename(tmp_dest.name, dest) + except (shutil.Error, OSError, IOError), e: + self.cleanup(tmp_dest.name) + self.fail_json(msg='Could not replace file: %s to %s: %s' % (src, dest, e)) + + if creating: + # make sure the file has the correct permissions + # based on the current value of umask + umask = os.umask(0) + os.umask(umask) + os.chmod(dest, 0666 ^ umask) + if switched_user: + os.chown(dest, os.getuid(), os.getgid()) + + if self.selinux_enabled(): + # rename might not preserve context + self.set_context_if_different(dest, context, False) + + def run_command(self, args, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None): + ''' + Execute a command, returns rc, stdout, and stderr. + args is the command to run + If args is a list, the command will be run with shell=False. + If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False + If args is a string and use_unsafe_shell=True it run with shell=True. + Other arguments: + - check_rc (boolean) Whether to call fail_json in case of + non zero RC. Default is False. + - close_fds (boolean) See documentation for subprocess.Popen(). + Default is True. + - executable (string) See documentation for subprocess.Popen(). + Default is None. + - prompt_regex (string) A regex string (not a compiled regex) which + can be used to detect prompts in the stdout + which would otherwise cause the execution + to hang (especially if no input data is + specified) + ''' + + shell = False + if isinstance(args, list): + if use_unsafe_shell: + args = " ".join([pipes.quote(x) for x in args]) + shell = True + elif isinstance(args, basestring) and use_unsafe_shell: + shell = True + elif isinstance(args, basestring): + args = shlex.split(args.encode('utf-8')) + else: + msg = "Argument 'args' to run_command must be list or string" + self.fail_json(rc=257, cmd=args, msg=msg) + + prompt_re = None + if prompt_regex: + try: + prompt_re = re.compile(prompt_regex, re.MULTILINE) + except re.error: + self.fail_json(msg="invalid prompt regular expression given to run_command") + + # expand things like $HOME and ~ + if not shell: + args = [ os.path.expandvars(os.path.expanduser(x)) for x in args ] + + rc = 0 + msg = None + st_in = None + + # Set a temporart env path if a prefix is passed + env=os.environ + if path_prefix: + env['PATH']="%s:%s" % (path_prefix, env['PATH']) + + # create a printable version of the command for use + # in reporting later, which strips out things like + # passwords from the args list + if isinstance(args, list): + clean_args = " ".join(pipes.quote(arg) for arg in args) + else: + clean_args = args + + # all clean strings should return two match groups, + # where the first is the CLI argument and the second + # is the password/key/phrase that will be hidden + clean_re_strings = [ + # this removes things like --password, --pass, --pass-wd, etc. + # optionally followed by an '=' or a space. The password can + # be quoted or not too, though it does not care about quotes + # that are not balanced + # source: http://blog.stevenlevithan.com/archives/match-quoted-string + r'([-]{0,2}pass[-]?(?:word|wd)?[=\s]?)((?:["\'])?(?:[^\s])*(?:\1)?)', + r'^(?P<before>.*:)(?P<password>.*)(?P<after>\@.*)$', + # TODO: add more regex checks here + ] + for re_str in clean_re_strings: + r = re.compile(re_str) + clean_args = r.sub(r'\1********', clean_args) + + if data: + st_in = subprocess.PIPE + + kwargs = dict( + executable=executable, + shell=shell, + close_fds=close_fds, + stdin=st_in, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + if path_prefix: + kwargs['env'] = env + if cwd and os.path.isdir(cwd): + kwargs['cwd'] = cwd + + # store the pwd + prev_dir = os.getcwd() + + # make sure we're in the right working directory + if cwd and os.path.isdir(cwd): + try: + os.chdir(cwd) + except (OSError, IOError), e: + self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, str(e))) + + try: + cmd = subprocess.Popen(args, **kwargs) + + # the communication logic here is essentially taken from that + # of the _communicate() function in ssh.py + + stdout = '' + stderr = '' + rpipes = [cmd.stdout, cmd.stderr] + + if data: + if not binary_data: + data += '\n' + cmd.stdin.write(data) + cmd.stdin.close() + + while True: + rfd, wfd, efd = select.select(rpipes, [], rpipes, 1) + if cmd.stdout in rfd: + dat = os.read(cmd.stdout.fileno(), 9000) + stdout += dat + if dat == '': + rpipes.remove(cmd.stdout) + if cmd.stderr in rfd: + dat = os.read(cmd.stderr.fileno(), 9000) + stderr += dat + if dat == '': + rpipes.remove(cmd.stderr) + # if we're checking for prompts, do it now + if prompt_re: + if prompt_re.search(stdout) and not data: + return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") + # only break out if no pipes are left to read or + # the pipes are completely read and + # the process is terminated + if (not rpipes or not rfd) and cmd.poll() is not None: + break + # No pipes are left to read but process is not yet terminated + # Only then it is safe to wait for the process to be finished + # NOTE: Actually cmd.poll() is always None here if rpipes is empty + elif not rpipes and cmd.poll() == None: + cmd.wait() + # The process is terminated. Since no pipes to read from are + # left, there is no need to call select() again. + break + + cmd.stdout.close() + cmd.stderr.close() + + rc = cmd.returncode + except (OSError, IOError), e: + self.fail_json(rc=e.errno, msg=str(e), cmd=clean_args) + except: + self.fail_json(rc=257, msg=traceback.format_exc(), cmd=clean_args) + + if rc != 0 and check_rc: + msg = stderr.rstrip() + self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) + + # reset the pwd + os.chdir(prev_dir) + + return (rc, stdout, stderr) + + def append_to_file(self, filename, str): + filename = os.path.expandvars(os.path.expanduser(filename)) + fh = open(filename, 'a') + fh.write(str) + fh.close() + + def pretty_bytes(self,size): + ranges = ( + (1<<70L, 'ZB'), + (1<<60L, 'EB'), + (1<<50L, 'PB'), + (1<<40L, 'TB'), + (1<<30L, 'GB'), + (1<<20L, 'MB'), + (1<<10L, 'KB'), + (1, 'Bytes') + ) + for limit, suffix in ranges: + if size >= limit: + break + return '%.2f %s' % (float(size)/ limit, suffix) + +def get_module_path(): + return os.path.dirname(os.path.realpath(__file__)) diff --git a/v2/ansible/module_utils/database.py b/v2/ansible/module_utils/database.py new file mode 100644 index 0000000000..0dd1990d3e --- /dev/null +++ b/v2/ansible/module_utils/database.py @@ -0,0 +1,128 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +class SQLParseError(Exception): + pass + +class UnclosedQuoteError(SQLParseError): + pass + +# maps a type of identifier to the maximum number of dot levels that are +# allowed to specifiy that identifier. For example, a database column can be +# specified by up to 4 levels: database.schema.table.column +_PG_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, schema=2, table=3, column=4, role=1) +_MYSQL_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, table=2, column=3, role=1, vars=1) + +def _find_end_quote(identifier, quote_char): + accumulate = 0 + while True: + try: + quote = identifier.index(quote_char) + except ValueError: + raise UnclosedQuoteError + accumulate = accumulate + quote + try: + next_char = identifier[quote+1] + except IndexError: + return accumulate + if next_char == quote_char: + try: + identifier = identifier[quote+2:] + accumulate = accumulate + 2 + except IndexError: + raise UnclosedQuoteError + else: + return accumulate + + +def _identifier_parse(identifier, quote_char): + if not identifier: + raise SQLParseError('Identifier name unspecified or unquoted trailing dot') + + already_quoted = False + if identifier.startswith(quote_char): + already_quoted = True + try: + end_quote = _find_end_quote(identifier[1:], quote_char=quote_char) + 1 + except UnclosedQuoteError: + already_quoted = False + else: + if end_quote < len(identifier) - 1: + if identifier[end_quote+1] == '.': + dot = end_quote + 1 + first_identifier = identifier[:dot] + next_identifier = identifier[dot+1:] + further_identifiers = _identifier_parse(next_identifier, quote_char) + further_identifiers.insert(0, first_identifier) + else: + raise SQLParseError('User escaped identifiers must escape extra quotes') + else: + further_identifiers = [identifier] + + if not already_quoted: + try: + dot = identifier.index('.') + except ValueError: + identifier = identifier.replace(quote_char, quote_char*2) + identifier = ''.join((quote_char, identifier, quote_char)) + further_identifiers = [identifier] + else: + if dot == 0 or dot >= len(identifier) - 1: + identifier = identifier.replace(quote_char, quote_char*2) + identifier = ''.join((quote_char, identifier, quote_char)) + further_identifiers = [identifier] + else: + first_identifier = identifier[:dot] + next_identifier = identifier[dot+1:] + further_identifiers = _identifier_parse(next_identifier, quote_char) + first_identifier = first_identifier.replace(quote_char, quote_char*2) + first_identifier = ''.join((quote_char, first_identifier, quote_char)) + further_identifiers.insert(0, first_identifier) + + return further_identifiers + + +def pg_quote_identifier(identifier, id_type): + identifier_fragments = _identifier_parse(identifier, quote_char='"') + if len(identifier_fragments) > _PG_IDENTIFIER_TO_DOT_LEVEL[id_type]: + raise SQLParseError('PostgreSQL does not support %s with more than %i dots' % (id_type, _PG_IDENTIFIER_TO_DOT_LEVEL[id_type])) + return '.'.join(identifier_fragments) + +def mysql_quote_identifier(identifier, id_type): + identifier_fragments = _identifier_parse(identifier, quote_char='`') + if len(identifier_fragments) > _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type]: + raise SQLParseError('MySQL does not support %s with more than %i dots' % (id_type, _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type])) + + special_cased_fragments = [] + for fragment in identifier_fragments: + if fragment == '`*`': + special_cased_fragments.append('*') + else: + special_cased_fragments.append(fragment) + + return '.'.join(special_cased_fragments) diff --git a/v2/ansible/module_utils/ec2.py b/v2/ansible/module_utils/ec2.py new file mode 100644 index 0000000000..0f08fead18 --- /dev/null +++ b/v2/ansible/module_utils/ec2.py @@ -0,0 +1,200 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +try: + from distutils.version import LooseVersion + HAS_LOOSE_VERSION = True +except: + HAS_LOOSE_VERSION = False + +AWS_REGIONS = [ + 'ap-northeast-1', + 'ap-southeast-1', + 'ap-southeast-2', + 'cn-north-1', + 'eu-central-1', + 'eu-west-1', + 'eu-central-1', + 'sa-east-1', + 'us-east-1', + 'us-west-1', + 'us-west-2', + 'us-gov-west-1', +] + + +def aws_common_argument_spec(): + return dict( + ec2_url=dict(), + aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True), + aws_access_key=dict(aliases=['ec2_access_key', 'access_key']), + validate_certs=dict(default=True, type='bool'), + security_token=dict(no_log=True), + profile=dict(), + ) + + +def ec2_argument_spec(): + spec = aws_common_argument_spec() + spec.update( + dict( + region=dict(aliases=['aws_region', 'ec2_region'], choices=AWS_REGIONS), + ) + ) + return spec + + +def boto_supports_profile_name(): + return hasattr(boto.ec2.EC2Connection, 'profile_name') + + +def get_aws_connection_info(module): + + # Check module args for credentials, then check environment vars + # access_key + + ec2_url = module.params.get('ec2_url') + access_key = module.params.get('aws_access_key') + secret_key = module.params.get('aws_secret_key') + security_token = module.params.get('security_token') + region = module.params.get('region') + profile_name = module.params.get('profile') + validate_certs = module.params.get('validate_certs') + + if not ec2_url: + if 'EC2_URL' in os.environ: + ec2_url = os.environ['EC2_URL'] + elif 'AWS_URL' in os.environ: + ec2_url = os.environ['AWS_URL'] + + if not access_key: + if 'EC2_ACCESS_KEY' in os.environ: + access_key = os.environ['EC2_ACCESS_KEY'] + elif 'AWS_ACCESS_KEY_ID' in os.environ: + access_key = os.environ['AWS_ACCESS_KEY_ID'] + elif 'AWS_ACCESS_KEY' in os.environ: + access_key = os.environ['AWS_ACCESS_KEY'] + else: + # in case access_key came in as empty string + access_key = None + + if not secret_key: + if 'EC2_SECRET_KEY' in os.environ: + secret_key = os.environ['EC2_SECRET_KEY'] + elif 'AWS_SECRET_ACCESS_KEY' in os.environ: + secret_key = os.environ['AWS_SECRET_ACCESS_KEY'] + elif 'AWS_SECRET_KEY' in os.environ: + secret_key = os.environ['AWS_SECRET_KEY'] + else: + # in case secret_key came in as empty string + secret_key = None + + if not region: + if 'EC2_REGION' in os.environ: + region = os.environ['EC2_REGION'] + elif 'AWS_REGION' in os.environ: + region = os.environ['AWS_REGION'] + else: + # boto.config.get returns None if config not found + region = boto.config.get('Boto', 'aws_region') + if not region: + region = boto.config.get('Boto', 'ec2_region') + + if not security_token: + if 'AWS_SECURITY_TOKEN' in os.environ: + security_token = os.environ['AWS_SECURITY_TOKEN'] + else: + # in case security_token came in as empty string + security_token = None + + boto_params = dict(aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + security_token=security_token) + + # profile_name only works as a key in boto >= 2.24 + # so only set profile_name if passed as an argument + if profile_name: + if not boto_supports_profile_name(): + module.fail_json("boto does not support profile_name before 2.24") + boto_params['profile_name'] = profile_name + + if validate_certs and HAS_LOOSE_VERSION and LooseVersion(boto.Version) >= LooseVersion("2.6.0"): + boto_params['validate_certs'] = validate_certs + + return region, ec2_url, boto_params + + +def get_ec2_creds(module): + ''' for compatibility mode with old modules that don't/can't yet + use ec2_connect method ''' + region, ec2_url, boto_params = get_aws_connection_info(module) + return ec2_url, boto_params['aws_access_key_id'], boto_params['aws_secret_access_key'], region + + +def boto_fix_security_token_in_profile(conn, profile_name): + ''' monkey patch for boto issue boto/boto#2100 ''' + profile = 'profile ' + profile_name + if boto.config.has_option(profile, 'aws_security_token'): + conn.provider.set_security_token(boto.config.get(profile, 'aws_security_token')) + return conn + + +def connect_to_aws(aws_module, region, **params): + conn = aws_module.connect_to_region(region, **params) + if not conn: + if region not in [aws_module_region.name for aws_module_region in aws_module.regions()]: + raise StandardError("Region %s does not seem to be available for aws module %s. If the region definitely exists, you may need to upgrade boto" % (region, aws_module.__name__)) + else: + raise StandardError("Unknown problem connecting to region %s for aws module %s." % (region, aws_module.__name__)) + if params.get('profile_name'): + conn = boto_fix_security_token_in_profile(conn, params['profile_name']) + return conn + + +def ec2_connect(module): + + """ Return an ec2 connection""" + + region, ec2_url, boto_params = get_aws_connection_info(module) + + # If we have a region specified, connect to its endpoint. + if region: + try: + ec2 = connect_to_aws(boto.ec2, region, **boto_params) + except (boto.exception.NoAuthHandlerFound, StandardError), e: + module.fail_json(msg=str(e)) + # Otherwise, no region so we fallback to the old connection method + elif ec2_url: + try: + ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params) + except (boto.exception.NoAuthHandlerFound, StandardError), e: + module.fail_json(msg=str(e)) + else: + module.fail_json(msg="Either region or ec2_url must be specified") + + return ec2 diff --git a/v2/ansible/module_utils/facts.py b/v2/ansible/module_utils/facts.py new file mode 100644 index 0000000000..c2d7b652e1 --- /dev/null +++ b/v2/ansible/module_utils/facts.py @@ -0,0 +1,2572 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os +import stat +import array +import errno +import fcntl +import fnmatch +import glob +import platform +import re +import signal +import socket +import struct +import datetime +import getpass +import pwd +import ConfigParser +import StringIO + +from string import maketrans + +try: + import selinux + HAVE_SELINUX=True +except ImportError: + HAVE_SELINUX=False + +try: + import json +except ImportError: + import simplejson as json + +# -------------------------------------------------------------- +# timeout function to make sure some fact gathering +# steps do not exceed a time limit + +class TimeoutError(Exception): + pass + +def timeout(seconds=10, error_message="Timer expired"): + def decorator(func): + def _handle_timeout(signum, frame): + raise TimeoutError(error_message) + + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _handle_timeout) + signal.alarm(seconds) + try: + result = func(*args, **kwargs) + finally: + signal.alarm(0) + return result + + return wrapper + + return decorator + +# -------------------------------------------------------------- + +class Facts(object): + """ + This class should only attempt to populate those facts that + are mostly generic to all systems. This includes platform facts, + service facts (e.g. ssh keys or selinux), and distribution facts. + Anything that requires extensive code or may have more than one + possible implementation to establish facts for a given topic should + subclass Facts. + """ + + # i86pc is a Solaris and derivatives-ism + _I386RE = re.compile(r'i([3456]86|86pc)') + # For the most part, we assume that platform.dist() will tell the truth. + # This is the fallback to handle unknowns or exceptions + OSDIST_LIST = ( ('/etc/redhat-release', 'RedHat'), + ('/etc/vmware-release', 'VMwareESX'), + ('/etc/openwrt_release', 'OpenWrt'), + ('/etc/system-release', 'OtherLinux'), + ('/etc/alpine-release', 'Alpine'), + ('/etc/release', 'Solaris'), + ('/etc/arch-release', 'Archlinux'), + ('/etc/SuSE-release', 'SuSE'), + ('/etc/os-release', 'SuSE'), + ('/etc/gentoo-release', 'Gentoo'), + ('/etc/os-release', 'Debian'), + ('/etc/lsb-release', 'Mandriva') ) + SELINUX_MODE_DICT = { 1: 'enforcing', 0: 'permissive', -1: 'disabled' } + + # A list of dicts. If there is a platform with more than one + # package manager, put the preferred one last. If there is an + # ansible module, use that as the value for the 'name' key. + PKG_MGRS = [ { 'path' : '/usr/bin/yum', 'name' : 'yum' }, + { 'path' : '/usr/bin/apt-get', 'name' : 'apt' }, + { 'path' : '/usr/bin/zypper', 'name' : 'zypper' }, + { 'path' : '/usr/sbin/urpmi', 'name' : 'urpmi' }, + { 'path' : '/usr/bin/pacman', 'name' : 'pacman' }, + { 'path' : '/bin/opkg', 'name' : 'opkg' }, + { 'path' : '/opt/local/bin/pkgin', 'name' : 'pkgin' }, + { 'path' : '/opt/local/bin/port', 'name' : 'macports' }, + { 'path' : '/sbin/apk', 'name' : 'apk' }, + { 'path' : '/usr/sbin/pkg', 'name' : 'pkgng' }, + { 'path' : '/usr/sbin/swlist', 'name' : 'SD-UX' }, + { 'path' : '/usr/bin/emerge', 'name' : 'portage' }, + { 'path' : '/usr/sbin/pkgadd', 'name' : 'svr4pkg' }, + { 'path' : '/usr/bin/pkg', 'name' : 'pkg' }, + ] + + def __init__(self): + self.facts = {} + self.get_platform_facts() + self.get_distribution_facts() + self.get_cmdline() + self.get_public_ssh_host_keys() + self.get_selinux_facts() + self.get_fips_facts() + self.get_pkg_mgr_facts() + self.get_lsb_facts() + self.get_date_time_facts() + self.get_user_facts() + self.get_local_facts() + self.get_env_facts() + + def populate(self): + return self.facts + + # Platform + # platform.system() can be Linux, Darwin, Java, or Windows + def get_platform_facts(self): + self.facts['system'] = platform.system() + self.facts['kernel'] = platform.release() + self.facts['machine'] = platform.machine() + self.facts['python_version'] = platform.python_version() + self.facts['fqdn'] = socket.getfqdn() + self.facts['hostname'] = platform.node().split('.')[0] + self.facts['nodename'] = platform.node() + self.facts['domain'] = '.'.join(self.facts['fqdn'].split('.')[1:]) + arch_bits = platform.architecture()[0] + self.facts['userspace_bits'] = arch_bits.replace('bit', '') + if self.facts['machine'] == 'x86_64': + self.facts['architecture'] = self.facts['machine'] + if self.facts['userspace_bits'] == '64': + self.facts['userspace_architecture'] = 'x86_64' + elif self.facts['userspace_bits'] == '32': + self.facts['userspace_architecture'] = 'i386' + elif Facts._I386RE.search(self.facts['machine']): + self.facts['architecture'] = 'i386' + if self.facts['userspace_bits'] == '64': + self.facts['userspace_architecture'] = 'x86_64' + elif self.facts['userspace_bits'] == '32': + self.facts['userspace_architecture'] = 'i386' + else: + self.facts['architecture'] = self.facts['machine'] + if self.facts['system'] == 'Linux': + self.get_distribution_facts() + elif self.facts['system'] == 'AIX': + rc, out, err = module.run_command("/usr/sbin/bootinfo -p") + data = out.split('\n') + self.facts['architecture'] = data[0] + + + def get_local_facts(self): + + fact_path = module.params.get('fact_path', None) + if not fact_path or not os.path.exists(fact_path): + return + + local = {} + for fn in sorted(glob.glob(fact_path + '/*.fact')): + # where it will sit under local facts + fact_base = os.path.basename(fn).replace('.fact','') + if stat.S_IXUSR & os.stat(fn)[stat.ST_MODE]: + # run it + # try to read it as json first + # if that fails read it with ConfigParser + # if that fails, skip it + rc, out, err = module.run_command(fn) + else: + out = open(fn).read() + + # load raw json + fact = 'loading %s' % fact_base + try: + fact = json.loads(out) + except ValueError, e: + # load raw ini + cp = ConfigParser.ConfigParser() + try: + cp.readfp(StringIO.StringIO(out)) + except ConfigParser.Error, e: + fact="error loading fact - please check content" + else: + fact = {} + #print cp.sections() + for sect in cp.sections(): + if sect not in fact: + fact[sect] = {} + for opt in cp.options(sect): + val = cp.get(sect, opt) + fact[sect][opt]=val + + local[fact_base] = fact + if not local: + return + self.facts['local'] = local + + # platform.dist() is deprecated in 2.6 + # in 2.6 and newer, you should use platform.linux_distribution() + def get_distribution_facts(self): + + # A list with OS Family members + OS_FAMILY = dict( + RedHat = 'RedHat', Fedora = 'RedHat', CentOS = 'RedHat', Scientific = 'RedHat', + SLC = 'RedHat', Ascendos = 'RedHat', CloudLinux = 'RedHat', PSBM = 'RedHat', + OracleLinux = 'RedHat', OVS = 'RedHat', OEL = 'RedHat', Amazon = 'RedHat', + XenServer = 'RedHat', Ubuntu = 'Debian', Debian = 'Debian', SLES = 'Suse', + SLED = 'Suse', OpenSuSE = 'Suse', SuSE = 'Suse', Gentoo = 'Gentoo', Funtoo = 'Gentoo', + Archlinux = 'Archlinux', Mandriva = 'Mandrake', Mandrake = 'Mandrake', + Solaris = 'Solaris', Nexenta = 'Solaris', OmniOS = 'Solaris', OpenIndiana = 'Solaris', + SmartOS = 'Solaris', AIX = 'AIX', Alpine = 'Alpine', MacOSX = 'Darwin', + FreeBSD = 'FreeBSD', HPUX = 'HP-UX' + ) + + # TODO: Rewrite this to use the function references in a dict pattern + # as it's much cleaner than this massive if-else + if self.facts['system'] == 'AIX': + self.facts['distribution'] = 'AIX' + rc, out, err = module.run_command("/usr/bin/oslevel") + data = out.split('.') + self.facts['distribution_version'] = data[0] + self.facts['distribution_release'] = data[1] + elif self.facts['system'] == 'HP-UX': + self.facts['distribution'] = 'HP-UX' + rc, out, err = module.run_command("/usr/sbin/swlist |egrep 'HPUX.*OE.*[AB].[0-9]+\.[0-9]+'", use_unsafe_shell=True) + data = re.search('HPUX.*OE.*([AB].[0-9]+\.[0-9]+)\.([0-9]+).*', out) + if data: + self.facts['distribution_version'] = data.groups()[0] + self.facts['distribution_release'] = data.groups()[1] + elif self.facts['system'] == 'Darwin': + self.facts['distribution'] = 'MacOSX' + rc, out, err = module.run_command("/usr/bin/sw_vers -productVersion") + data = out.split()[-1] + self.facts['distribution_version'] = data + elif self.facts['system'] == 'FreeBSD': + self.facts['distribution'] = 'FreeBSD' + self.facts['distribution_release'] = platform.release() + self.facts['distribution_version'] = platform.version() + elif self.facts['system'] == 'OpenBSD': + self.facts['distribution'] = 'OpenBSD' + self.facts['distribution_release'] = platform.release() + rc, out, err = module.run_command("/sbin/sysctl -n kern.version") + match = re.match('OpenBSD\s[0-9]+.[0-9]+-(\S+)\s.*', out) + if match: + self.facts['distribution_version'] = match.groups()[0] + else: + self.facts['distribution_version'] = 'release' + else: + dist = platform.dist() + self.facts['distribution'] = dist[0].capitalize() or 'NA' + self.facts['distribution_version'] = dist[1] or 'NA' + self.facts['distribution_major_version'] = dist[1].split('.')[0] or 'NA' + self.facts['distribution_release'] = dist[2] or 'NA' + # Try to handle the exceptions now ... + for (path, name) in Facts.OSDIST_LIST: + if os.path.exists(path): + if os.path.getsize(path) > 0: + if self.facts['distribution'] in ('Fedora', ): + # Once we determine the value is one of these distros + # we trust the values are always correct + break + elif name == 'RedHat': + data = get_file_content(path) + if 'Red Hat' in data: + self.facts['distribution'] = name + else: + self.facts['distribution'] = data.split()[0] + break + elif name == 'OtherLinux': + data = get_file_content(path) + if 'Amazon' in data: + self.facts['distribution'] = 'Amazon' + self.facts['distribution_version'] = data.split()[-1] + break + elif name == 'OpenWrt': + data = get_file_content(path) + if 'OpenWrt' in data: + self.facts['distribution'] = name + version = re.search('DISTRIB_RELEASE="(.*)"', data) + if version: + self.facts['distribution_version'] = version.groups()[0] + release = re.search('DISTRIB_CODENAME="(.*)"', data) + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif name == 'Alpine': + data = get_file_content(path) + self.facts['distribution'] = name + self.facts['distribution_version'] = data + break + elif name == 'Solaris': + data = get_file_content(path).split('\n')[0] + if 'Solaris' in data: + ora_prefix = '' + if 'Oracle Solaris' in data: + data = data.replace('Oracle ','') + ora_prefix = 'Oracle ' + self.facts['distribution'] = data.split()[0] + self.facts['distribution_version'] = data.split()[1] + self.facts['distribution_release'] = ora_prefix + data + break + + uname_rc, uname_out, uname_err = module.run_command(['uname', '-v']) + distribution_version = None + if 'SmartOS' in data: + self.facts['distribution'] = 'SmartOS' + if os.path.exists('/etc/product'): + product_data = dict([l.split(': ', 1) for l in get_file_content('/etc/product').split('\n') if ': ' in l]) + if 'Image' in product_data: + distribution_version = product_data.get('Image').split()[-1] + elif 'OpenIndiana' in data: + self.facts['distribution'] = 'OpenIndiana' + elif 'OmniOS' in data: + self.facts['distribution'] = 'OmniOS' + distribution_version = data.split()[-1] + elif uname_rc == 0 and 'NexentaOS_' in uname_out: + self.facts['distribution'] = 'Nexenta' + distribution_version = data.split()[-1].lstrip('v') + + if self.facts['distribution'] in ('SmartOS', 'OpenIndiana', 'OmniOS', 'Nexenta'): + self.facts['distribution_release'] = data.strip() + if distribution_version is not None: + self.facts['distribution_version'] = distribution_version + elif uname_rc == 0: + self.facts['distribution_version'] = uname_out.split('\n')[0].strip() + break + + elif name == 'SuSE': + data = get_file_content(path) + if 'suse' in data.lower(): + if path == '/etc/os-release': + release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) + distdata = get_file_content(path).split('\n')[0] + self.facts['distribution'] = distdata.split('=')[1] + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif path == '/etc/SuSE-release': + data = data.splitlines() + distdata = get_file_content(path).split('\n')[0] + self.facts['distribution'] = distdata.split()[0] + for line in data: + release = re.search('CODENAME *= *([^\n]+)', line) + if release: + self.facts['distribution_release'] = release.groups()[0].strip() + break + elif name == 'Debian': + data = get_file_content(path) + if 'Debian' in data: + release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif name == 'Mandriva': + data = get_file_content(path) + if 'Mandriva' in data: + version = re.search('DISTRIB_RELEASE="(.*)"', data) + if version: + self.facts['distribution_version'] = version.groups()[0] + release = re.search('DISTRIB_CODENAME="(.*)"', data) + if release: + self.facts['distribution_release'] = release.groups()[0] + self.facts['distribution'] = name + break + else: + self.facts['distribution'] = name + + self.facts['os_family'] = self.facts['distribution'] + if self.facts['distribution'] in OS_FAMILY: + self.facts['os_family'] = OS_FAMILY[self.facts['distribution']] + + def get_cmdline(self): + data = get_file_content('/proc/cmdline') + if data: + self.facts['cmdline'] = {} + try: + for piece in shlex.split(data): + item = piece.split('=', 1) + if len(item) == 1: + self.facts['cmdline'][item[0]] = True + else: + self.facts['cmdline'][item[0]] = item[1] + except ValueError, e: + pass + + def get_public_ssh_host_keys(self): + dsa_filename = '/etc/ssh/ssh_host_dsa_key.pub' + rsa_filename = '/etc/ssh/ssh_host_rsa_key.pub' + ecdsa_filename = '/etc/ssh/ssh_host_ecdsa_key.pub' + + if self.facts['system'] == 'Darwin': + dsa_filename = '/etc/ssh_host_dsa_key.pub' + rsa_filename = '/etc/ssh_host_rsa_key.pub' + ecdsa_filename = '/etc/ssh_host_ecdsa_key.pub' + dsa = get_file_content(dsa_filename) + rsa = get_file_content(rsa_filename) + ecdsa = get_file_content(ecdsa_filename) + if dsa is None: + dsa = 'NA' + else: + self.facts['ssh_host_key_dsa_public'] = dsa.split()[1] + if rsa is None: + rsa = 'NA' + else: + self.facts['ssh_host_key_rsa_public'] = rsa.split()[1] + if ecdsa is None: + ecdsa = 'NA' + else: + self.facts['ssh_host_key_ecdsa_public'] = ecdsa.split()[1] + + def get_pkg_mgr_facts(self): + self.facts['pkg_mgr'] = 'unknown' + for pkg in Facts.PKG_MGRS: + if os.path.exists(pkg['path']): + self.facts['pkg_mgr'] = pkg['name'] + if self.facts['system'] == 'OpenBSD': + self.facts['pkg_mgr'] = 'openbsd_pkg' + + def get_lsb_facts(self): + lsb_path = module.get_bin_path('lsb_release') + if lsb_path: + rc, out, err = module.run_command([lsb_path, "-a"]) + if rc == 0: + self.facts['lsb'] = {} + for line in out.split('\n'): + if len(line) < 1: + continue + value = line.split(':', 1)[1].strip() + if 'LSB Version:' in line: + self.facts['lsb']['release'] = value + elif 'Distributor ID:' in line: + self.facts['lsb']['id'] = value + elif 'Description:' in line: + self.facts['lsb']['description'] = value + elif 'Release:' in line: + self.facts['lsb']['release'] = value + elif 'Codename:' in line: + self.facts['lsb']['codename'] = value + if 'lsb' in self.facts and 'release' in self.facts['lsb']: + self.facts['lsb']['major_release'] = self.facts['lsb']['release'].split('.')[0] + elif lsb_path is None and os.path.exists('/etc/lsb-release'): + self.facts['lsb'] = {} + f = open('/etc/lsb-release', 'r') + try: + for line in f.readlines(): + value = line.split('=',1)[1].strip() + if 'DISTRIB_ID' in line: + self.facts['lsb']['id'] = value + elif 'DISTRIB_RELEASE' in line: + self.facts['lsb']['release'] = value + elif 'DISTRIB_DESCRIPTION' in line: + self.facts['lsb']['description'] = value + elif 'DISTRIB_CODENAME' in line: + self.facts['lsb']['codename'] = value + finally: + f.close() + else: + return self.facts + + if 'lsb' in self.facts and 'release' in self.facts['lsb']: + self.facts['lsb']['major_release'] = self.facts['lsb']['release'].split('.')[0] + + + def get_selinux_facts(self): + if not HAVE_SELINUX: + self.facts['selinux'] = False + return + self.facts['selinux'] = {} + if not selinux.is_selinux_enabled(): + self.facts['selinux']['status'] = 'disabled' + else: + self.facts['selinux']['status'] = 'enabled' + try: + self.facts['selinux']['policyvers'] = selinux.security_policyvers() + except OSError, e: + self.facts['selinux']['policyvers'] = 'unknown' + try: + (rc, configmode) = selinux.selinux_getenforcemode() + if rc == 0: + self.facts['selinux']['config_mode'] = Facts.SELINUX_MODE_DICT.get(configmode, 'unknown') + else: + self.facts['selinux']['config_mode'] = 'unknown' + except OSError, e: + self.facts['selinux']['config_mode'] = 'unknown' + try: + mode = selinux.security_getenforce() + self.facts['selinux']['mode'] = Facts.SELINUX_MODE_DICT.get(mode, 'unknown') + except OSError, e: + self.facts['selinux']['mode'] = 'unknown' + try: + (rc, policytype) = selinux.selinux_getpolicytype() + if rc == 0: + self.facts['selinux']['type'] = policytype + else: + self.facts['selinux']['type'] = 'unknown' + except OSError, e: + self.facts['selinux']['type'] = 'unknown' + + + def get_fips_facts(self): + self.facts['fips'] = False + data = get_file_content('/proc/sys/crypto/fips_enabled') + if data and data == '1': + self.facts['fips'] = True + + + def get_date_time_facts(self): + self.facts['date_time'] = {} + + now = datetime.datetime.now() + self.facts['date_time']['year'] = now.strftime('%Y') + self.facts['date_time']['month'] = now.strftime('%m') + self.facts['date_time']['weekday'] = now.strftime('%A') + self.facts['date_time']['day'] = now.strftime('%d') + self.facts['date_time']['hour'] = now.strftime('%H') + self.facts['date_time']['minute'] = now.strftime('%M') + self.facts['date_time']['second'] = now.strftime('%S') + self.facts['date_time']['epoch'] = now.strftime('%s') + if self.facts['date_time']['epoch'] == '' or self.facts['date_time']['epoch'][0] == '%': + self.facts['date_time']['epoch'] = str(int(time.time())) + self.facts['date_time']['date'] = now.strftime('%Y-%m-%d') + self.facts['date_time']['time'] = now.strftime('%H:%M:%S') + self.facts['date_time']['iso8601_micro'] = now.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.facts['date_time']['iso8601'] = now.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + self.facts['date_time']['tz'] = time.strftime("%Z") + self.facts['date_time']['tz_offset'] = time.strftime("%z") + + + # User + def get_user_facts(self): + self.facts['user_id'] = getpass.getuser() + pwent = pwd.getpwnam(getpass.getuser()) + self.facts['user_uid'] = pwent.pw_uid + self.facts['user_gid'] = pwent.pw_gid + self.facts['user_gecos'] = pwent.pw_gecos + self.facts['user_dir'] = pwent.pw_dir + self.facts['user_shell'] = pwent.pw_shell + + def get_env_facts(self): + self.facts['env'] = {} + for k,v in os.environ.iteritems(): + self.facts['env'][k] = v + +class Hardware(Facts): + """ + This is a generic Hardware subclass of Facts. This should be further + subclassed to implement per platform. If you subclass this, it + should define: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor (a list) + - processor_cores + - processor_count + + All subclasses MUST define platform. + """ + platform = 'Generic' + + def __new__(cls, *arguments, **keyword): + subclass = cls + for sc in Hardware.__subclasses__(): + if sc.platform == platform.system(): + subclass = sc + return super(cls, subclass).__new__(subclass, *arguments, **keyword) + + def __init__(self): + Facts.__init__(self) + + def populate(self): + return self.facts + +class LinuxHardware(Hardware): + """ + Linux-specific subclass of Hardware. Defines memory and CPU facts: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor (a list) + - processor_cores + - processor_count + + In addition, it also defines number of DMI facts and device facts. + """ + + platform = 'Linux' + + # Originally only had these four as toplevelfacts + ORIGINAL_MEMORY_FACTS = frozenset(('MemTotal', 'SwapTotal', 'MemFree', 'SwapFree')) + # Now we have all of these in a dict structure + MEMORY_FACTS = ORIGINAL_MEMORY_FACTS.union(('Buffers', 'Cached', 'SwapCached')) + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.get_cpu_facts() + self.get_memory_facts() + self.get_dmi_facts() + self.get_device_facts() + try: + self.get_mount_facts() + except TimeoutError: + pass + return self.facts + + def get_memory_facts(self): + if not os.access("/proc/meminfo", os.R_OK): + return + + memstats = {} + for line in open("/proc/meminfo").readlines(): + data = line.split(":", 1) + key = data[0] + if key in self.ORIGINAL_MEMORY_FACTS: + val = data[1].strip().split(' ')[0] + self.facts["%s_mb" % key.lower()] = long(val) / 1024 + + if key in self.MEMORY_FACTS: + val = data[1].strip().split(' ')[0] + memstats[key.lower()] = long(val) / 1024 + + if None not in (memstats.get('memtotal'), memstats.get('memfree')): + memstats['real:used'] = memstats['memtotal'] - memstats['memfree'] + if None not in (memstats.get('cached'), memstats.get('memfree'), memstats.get('buffers')): + memstats['nocache:free'] = memstats['cached'] + memstats['memfree'] + memstats['buffers'] + if None not in (memstats.get('memtotal'), memstats.get('nocache:free')): + memstats['nocache:used'] = memstats['memtotal'] - memstats['nocache:free'] + if None not in (memstats.get('swaptotal'), memstats.get('swapfree')): + memstats['swap:used'] = memstats['swaptotal'] - memstats['swapfree'] + + self.facts['memory_mb'] = { + 'real' : { + 'total': memstats.get('memtotal'), + 'used': memstats.get('real:used'), + 'free': memstats.get('memfree'), + }, + 'nocache' : { + 'free': memstats.get('nocache:free'), + 'used': memstats.get('nocache:used'), + }, + 'swap' : { + 'total': memstats.get('swaptotal'), + 'free': memstats.get('swapfree'), + 'used': memstats.get('swap:used'), + 'cached': memstats.get('swapcached'), + }, + } + + def get_cpu_facts(self): + i = 0 + vendor_id_occurrence = 0 + model_name_occurrence = 0 + physid = 0 + coreid = 0 + sockets = {} + cores = {} + + xen = False + xen_paravirt = False + try: + if os.path.exists('/proc/xen'): + xen = True + elif open('/sys/hypervisor/type').readline().strip() == 'xen': + xen = True + except IOError: + pass + + if not os.access("/proc/cpuinfo", os.R_OK): + return + self.facts['processor'] = [] + for line in open("/proc/cpuinfo").readlines(): + data = line.split(":", 1) + key = data[0].strip() + + if xen: + if key == 'flags': + # Check for vme cpu flag, Xen paravirt does not expose this. + # Need to detect Xen paravirt because it exposes cpuinfo + # differently than Xen HVM or KVM and causes reporting of + # only a single cpu core. + if 'vme' not in data: + xen_paravirt = True + + # model name is for Intel arch, Processor (mind the uppercase P) + # works for some ARM devices, like the Sheevaplug. + if key == 'model name' or key == 'Processor' or key == 'vendor_id': + if 'processor' not in self.facts: + self.facts['processor'] = [] + self.facts['processor'].append(data[1].strip()) + if key == 'vendor_id': + vendor_id_occurrence += 1 + if key == 'model name': + model_name_occurrence += 1 + i += 1 + elif key == 'physical id': + physid = data[1].strip() + if physid not in sockets: + sockets[physid] = 1 + elif key == 'core id': + coreid = data[1].strip() + if coreid not in sockets: + cores[coreid] = 1 + elif key == 'cpu cores': + sockets[physid] = int(data[1].strip()) + elif key == 'siblings': + cores[coreid] = int(data[1].strip()) + elif key == '# processors': + self.facts['processor_cores'] = int(data[1].strip()) + + if vendor_id_occurrence == model_name_occurrence: + i = vendor_id_occurrence + + if self.facts['architecture'] != 's390x': + if xen_paravirt: + self.facts['processor_count'] = i + self.facts['processor_cores'] = i + self.facts['processor_threads_per_core'] = 1 + self.facts['processor_vcpus'] = i + else: + self.facts['processor_count'] = sockets and len(sockets) or i + self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1 + self.facts['processor_threads_per_core'] = ((cores.values() and + cores.values()[0] or 1) / self.facts['processor_cores']) + self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] * + self.facts['processor_count'] * self.facts['processor_cores']) + + def get_dmi_facts(self): + ''' learn dmi facts from system + + Try /sys first for dmi related facts. + If that is not available, fall back to dmidecode executable ''' + + if os.path.exists('/sys/devices/virtual/dmi/id/product_name'): + # Use kernel DMI info, if available + + # DMI SPEC -- http://www.dmtf.org/sites/default/files/standards/documents/DSP0134_2.7.0.pdf + FORM_FACTOR = [ "Unknown", "Other", "Unknown", "Desktop", + "Low Profile Desktop", "Pizza Box", "Mini Tower", "Tower", + "Portable", "Laptop", "Notebook", "Hand Held", "Docking Station", + "All In One", "Sub Notebook", "Space-saving", "Lunch Box", + "Main Server Chassis", "Expansion Chassis", "Sub Chassis", + "Bus Expansion Chassis", "Peripheral Chassis", "RAID Chassis", + "Rack Mount Chassis", "Sealed-case PC", "Multi-system", + "CompactPCI", "AdvancedTCA", "Blade" ] + + DMI_DICT = { + 'bios_date': '/sys/devices/virtual/dmi/id/bios_date', + 'bios_version': '/sys/devices/virtual/dmi/id/bios_version', + 'form_factor': '/sys/devices/virtual/dmi/id/chassis_type', + 'product_name': '/sys/devices/virtual/dmi/id/product_name', + 'product_serial': '/sys/devices/virtual/dmi/id/product_serial', + 'product_uuid': '/sys/devices/virtual/dmi/id/product_uuid', + 'product_version': '/sys/devices/virtual/dmi/id/product_version', + 'system_vendor': '/sys/devices/virtual/dmi/id/sys_vendor' + } + + for (key,path) in DMI_DICT.items(): + data = get_file_content(path) + if data is not None: + if key == 'form_factor': + try: + self.facts['form_factor'] = FORM_FACTOR[int(data)] + except IndexError, e: + self.facts['form_factor'] = 'unknown (%s)' % data + else: + self.facts[key] = data + else: + self.facts[key] = 'NA' + + else: + # Fall back to using dmidecode, if available + dmi_bin = module.get_bin_path('dmidecode') + DMI_DICT = { + 'bios_date': 'bios-release-date', + 'bios_version': 'bios-version', + 'form_factor': 'chassis-type', + 'product_name': 'system-product-name', + 'product_serial': 'system-serial-number', + 'product_uuid': 'system-uuid', + 'product_version': 'system-version', + 'system_vendor': 'system-manufacturer' + } + for (k, v) in DMI_DICT.items(): + if dmi_bin is not None: + (rc, out, err) = module.run_command('%s -s %s' % (dmi_bin, v)) + if rc == 0: + # Strip out commented lines (specific dmidecode output) + thisvalue = ''.join([ line for line in out.split('\n') if not line.startswith('#') ]) + try: + json.dumps(thisvalue) + except UnicodeDecodeError: + thisvalue = "NA" + + self.facts[k] = thisvalue + else: + self.facts[k] = 'NA' + else: + self.facts[k] = 'NA' + + @timeout(10) + def get_mount_facts(self): + self.facts['mounts'] = [] + mtab = get_file_content('/etc/mtab', '') + for line in mtab.split('\n'): + if line.startswith('/'): + fields = line.rstrip('\n').split() + if(fields[2] != 'none'): + size_total = None + size_available = None + try: + statvfs_result = os.statvfs(fields[1]) + size_total = statvfs_result.f_bsize * statvfs_result.f_blocks + size_available = statvfs_result.f_bsize * (statvfs_result.f_bavail) + except OSError, e: + continue + lsblkPath = module.get_bin_path("lsblk") + rc, out, err = module.run_command("%s -ln --output UUID %s" % (lsblkPath, fields[0]), use_unsafe_shell=True) + + if rc == 0: + uuid = out.strip() + else: + uuid = 'NA' + + self.facts['mounts'].append( + {'mount': fields[1], + 'device':fields[0], + 'fstype': fields[2], + 'options': fields[3], + # statvfs data + 'size_total': size_total, + 'size_available': size_available, + 'uuid': uuid, + }) + + def get_device_facts(self): + self.facts['devices'] = {} + lspci = module.get_bin_path('lspci') + if lspci: + rc, pcidata, err = module.run_command([lspci, '-D']) + else: + pcidata = None + + try: + block_devs = os.listdir("/sys/block") + except OSError: + return + + for block in block_devs: + virtual = 1 + sysfs_no_links = 0 + try: + path = os.readlink(os.path.join("/sys/block/", block)) + except OSError, e: + if e.errno == errno.EINVAL: + path = block + sysfs_no_links = 1 + else: + continue + if "virtual" in path: + continue + sysdir = os.path.join("/sys/block", path) + if sysfs_no_links == 1: + for folder in os.listdir(sysdir): + if "device" in folder: + virtual = 0 + break + if virtual: + continue + d = {} + diskname = os.path.basename(sysdir) + for key in ['vendor', 'model']: + d[key] = get_file_content(sysdir + "/device/" + key) + + for key,test in [ ('removable','/removable'), \ + ('support_discard','/queue/discard_granularity'), + ]: + d[key] = get_file_content(sysdir + test) + + d['partitions'] = {} + for folder in os.listdir(sysdir): + m = re.search("(" + diskname + "\d+)", folder) + if m: + part = {} + partname = m.group(1) + part_sysdir = sysdir + "/" + partname + + part['start'] = get_file_content(part_sysdir + "/start",0) + part['sectors'] = get_file_content(part_sysdir + "/size",0) + part['sectorsize'] = get_file_content(part_sysdir + "/queue/physical_block_size") + if not part['sectorsize']: + part['sectorsize'] = get_file_content(part_sysdir + "/queue/hw_sector_size",512) + part['size'] = module.pretty_bytes((float(part['sectors']) * float(part['sectorsize']))) + d['partitions'][partname] = part + + d['rotational'] = get_file_content(sysdir + "/queue/rotational") + d['scheduler_mode'] = "" + scheduler = get_file_content(sysdir + "/queue/scheduler") + if scheduler is not None: + m = re.match(".*?(\[(.*)\])", scheduler) + if m: + d['scheduler_mode'] = m.group(2) + + d['sectors'] = get_file_content(sysdir + "/size") + if not d['sectors']: + d['sectors'] = 0 + d['sectorsize'] = get_file_content(sysdir + "/queue/physical_block_size") + if not d['sectorsize']: + d['sectorsize'] = get_file_content(sysdir + "/queue/hw_sector_size",512) + d['size'] = module.pretty_bytes(float(d['sectors']) * float(d['sectorsize'])) + + d['host'] = "" + + # domains are numbered (0 to ffff), bus (0 to ff), slot (0 to 1f), and function (0 to 7). + m = re.match(".+/([a-f0-9]{4}:[a-f0-9]{2}:[0|1][a-f0-9]\.[0-7])/", sysdir) + if m and pcidata: + pciid = m.group(1) + did = re.escape(pciid) + m = re.search("^" + did + "\s(.*)$", pcidata, re.MULTILINE) + d['host'] = m.group(1) + + d['holders'] = [] + if os.path.isdir(sysdir + "/holders"): + for folder in os.listdir(sysdir + "/holders"): + if not folder.startswith("dm-"): + continue + name = get_file_content(sysdir + "/holders/" + folder + "/dm/name") + if name: + d['holders'].append(name) + else: + d['holders'].append(folder) + + self.facts['devices'][diskname] = d + + +class SunOSHardware(Hardware): + """ + In addition to the generic memory and cpu facts, this also sets + swap_reserved_mb and swap_allocated_mb that is available from *swap -s*. + """ + platform = 'SunOS' + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.get_cpu_facts() + self.get_memory_facts() + return self.facts + + def get_cpu_facts(self): + physid = 0 + sockets = {} + rc, out, err = module.run_command("/usr/bin/kstat cpu_info") + self.facts['processor'] = [] + for line in out.split('\n'): + if len(line) < 1: + continue + data = line.split(None, 1) + key = data[0].strip() + # "brand" works on Solaris 10 & 11. "implementation" for Solaris 9. + if key == 'module:': + brand = '' + elif key == 'brand': + brand = data[1].strip() + elif key == 'clock_MHz': + clock_mhz = data[1].strip() + elif key == 'implementation': + processor = brand or data[1].strip() + # Add clock speed to description for SPARC CPU + if self.facts['machine'] != 'i86pc': + processor += " @ " + clock_mhz + "MHz" + if 'processor' not in self.facts: + self.facts['processor'] = [] + self.facts['processor'].append(processor) + elif key == 'chip_id': + physid = data[1].strip() + if physid not in sockets: + sockets[physid] = 1 + else: + sockets[physid] += 1 + # Counting cores on Solaris can be complicated. + # https://blogs.oracle.com/mandalika/entry/solaris_show_me_the_cpu + # Treat 'processor_count' as physical sockets and 'processor_cores' as + # virtual CPUs visisble to Solaris. Not a true count of cores for modern SPARC as + # these processors have: sockets -> cores -> threads/virtual CPU. + if len(sockets) > 0: + self.facts['processor_count'] = len(sockets) + self.facts['processor_cores'] = reduce(lambda x, y: x + y, sockets.values()) + else: + self.facts['processor_cores'] = 'NA' + self.facts['processor_count'] = len(self.facts['processor']) + + def get_memory_facts(self): + rc, out, err = module.run_command(["/usr/sbin/prtconf"]) + for line in out.split('\n'): + if 'Memory size' in line: + self.facts['memtotal_mb'] = line.split()[2] + rc, out, err = module.run_command("/usr/sbin/swap -s") + allocated = long(out.split()[1][:-1]) + reserved = long(out.split()[5][:-1]) + used = long(out.split()[8][:-1]) + free = long(out.split()[10][:-1]) + self.facts['swapfree_mb'] = free / 1024 + self.facts['swaptotal_mb'] = (free + used) / 1024 + self.facts['swap_allocated_mb'] = allocated / 1024 + self.facts['swap_reserved_mb'] = reserved / 1024 + +class OpenBSDHardware(Hardware): + """ + OpenBSD-specific subclass of Hardware. Defines memory, CPU and device facts: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor (a list) + - processor_cores + - processor_count + - processor_speed + - devices + """ + platform = 'OpenBSD' + DMESG_BOOT = '/var/run/dmesg.boot' + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.sysctl = self.get_sysctl() + self.get_memory_facts() + self.get_processor_facts() + self.get_device_facts() + return self.facts + + def get_sysctl(self): + rc, out, err = module.run_command(["/sbin/sysctl", "hw"]) + if rc != 0: + return dict() + sysctl = dict() + for line in out.splitlines(): + (key, value) = line.split('=') + sysctl[key] = value.strip() + return sysctl + + def get_memory_facts(self): + # Get free memory. vmstat output looks like: + # procs memory page disks traps cpu + # r b w avm fre flt re pi po fr sr wd0 fd0 int sys cs us sy id + # 0 0 0 47512 28160 51 0 0 0 0 0 1 0 116 89 17 0 1 99 + rc, out, err = module.run_command("/usr/bin/vmstat") + if rc == 0: + self.facts['memfree_mb'] = long(out.splitlines()[-1].split()[4]) / 1024 + self.facts['memtotal_mb'] = long(self.sysctl['hw.usermem']) / 1024 / 1024 + + # Get swapctl info. swapctl output looks like: + # total: 69268 1K-blocks allocated, 0 used, 69268 available + # And for older OpenBSD: + # total: 69268k bytes allocated = 0k used, 69268k available + rc, out, err = module.run_command("/sbin/swapctl -sk") + if rc == 0: + swaptrans = maketrans(' ', ' ') + data = out.split() + self.facts['swapfree_mb'] = long(data[-2].translate(swaptrans, "kmg")) / 1024 + self.facts['swaptotal_mb'] = long(data[1].translate(swaptrans, "kmg")) / 1024 + + def get_processor_facts(self): + processor = [] + dmesg_boot = get_file_content(OpenBSDHardware.DMESG_BOOT) + if not dmesg_boot: + rc, dmesg_boot, err = module.run_command("/sbin/dmesg") + i = 0 + for line in dmesg_boot.splitlines(): + if line.split(' ', 1)[0] == 'cpu%i:' % i: + processor.append(line.split(' ', 1)[1]) + i = i + 1 + processor_count = i + self.facts['processor'] = processor + self.facts['processor_count'] = processor_count + # I found no way to figure out the number of Cores per CPU in OpenBSD + self.facts['processor_cores'] = 'NA' + + def get_device_facts(self): + devices = [] + devices.extend(self.sysctl['hw.disknames'].split(',')) + self.facts['devices'] = devices + +class FreeBSDHardware(Hardware): + """ + FreeBSD-specific subclass of Hardware. Defines memory and CPU facts: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor (a list) + - processor_cores + - processor_count + - devices + """ + platform = 'FreeBSD' + DMESG_BOOT = '/var/run/dmesg.boot' + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.get_cpu_facts() + self.get_memory_facts() + self.get_dmi_facts() + self.get_device_facts() + try: + self.get_mount_facts() + except TimeoutError: + pass + return self.facts + + def get_cpu_facts(self): + self.facts['processor'] = [] + rc, out, err = module.run_command("/sbin/sysctl -n hw.ncpu") + self.facts['processor_count'] = out.strip() + + dmesg_boot = get_file_content(FreeBSDHardware.DMESG_BOOT) + if not dmesg_boot: + rc, dmesg_boot, err = module.run_command("/sbin/dmesg") + for line in dmesg_boot.split('\n'): + if 'CPU:' in line: + cpu = re.sub(r'CPU:\s+', r"", line) + self.facts['processor'].append(cpu.strip()) + if 'Logical CPUs per core' in line: + self.facts['processor_cores'] = line.split()[4] + + + def get_memory_facts(self): + rc, out, err = module.run_command("/sbin/sysctl vm.stats") + for line in out.split('\n'): + data = line.split() + if 'vm.stats.vm.v_page_size' in line: + pagesize = long(data[1]) + if 'vm.stats.vm.v_page_count' in line: + pagecount = long(data[1]) + if 'vm.stats.vm.v_free_count' in line: + freecount = long(data[1]) + self.facts['memtotal_mb'] = pagesize * pagecount / 1024 / 1024 + self.facts['memfree_mb'] = pagesize * freecount / 1024 / 1024 + # Get swapinfo. swapinfo output looks like: + # Device 1M-blocks Used Avail Capacity + # /dev/ada0p3 314368 0 314368 0% + # + rc, out, err = module.run_command("/usr/sbin/swapinfo -m") + lines = out.split('\n') + if len(lines[-1]) == 0: + lines.pop() + data = lines[-1].split() + self.facts['swaptotal_mb'] = data[1] + self.facts['swapfree_mb'] = data[3] + + @timeout(10) + def get_mount_facts(self): + self.facts['mounts'] = [] + fstab = get_file_content('/etc/fstab') + if fstab: + for line in fstab.split('\n'): + if line.startswith('#') or line.strip() == '': + continue + fields = re.sub(r'\s+',' ',line.rstrip('\n')).split() + self.facts['mounts'].append({'mount': fields[1], 'device': fields[0], 'fstype' : fields[2], 'options': fields[3]}) + + def get_device_facts(self): + sysdir = '/dev' + self.facts['devices'] = {} + drives = re.compile('(ada?\d+|da\d+|a?cd\d+)') #TODO: rc, disks, err = module.run_command("/sbin/sysctl kern.disks") + slices = re.compile('(ada?\d+s\d+\w*|da\d+s\d+\w*)') + if os.path.isdir(sysdir): + dirlist = sorted(os.listdir(sysdir)) + for device in dirlist: + d = drives.match(device) + if d: + self.facts['devices'][d.group(1)] = [] + s = slices.match(device) + if s: + self.facts['devices'][d.group(1)].append(s.group(1)) + + def get_dmi_facts(self): + ''' learn dmi facts from system + + Use dmidecode executable if available''' + + # Fall back to using dmidecode, if available + dmi_bin = module.get_bin_path('dmidecode') + DMI_DICT = dict( + bios_date='bios-release-date', + bios_version='bios-version', + form_factor='chassis-type', + product_name='system-product-name', + product_serial='system-serial-number', + product_uuid='system-uuid', + product_version='system-version', + system_vendor='system-manufacturer' + ) + for (k, v) in DMI_DICT.items(): + if dmi_bin is not None: + (rc, out, err) = module.run_command('%s -s %s' % (dmi_bin, v)) + if rc == 0: + # Strip out commented lines (specific dmidecode output) + self.facts[k] = ''.join([ line for line in out.split('\n') if not line.startswith('#') ]) + try: + json.dumps(self.facts[k]) + except UnicodeDecodeError: + self.facts[k] = 'NA' + else: + self.facts[k] = 'NA' + else: + self.facts[k] = 'NA' + + +class NetBSDHardware(Hardware): + """ + NetBSD-specific subclass of Hardware. Defines memory and CPU facts: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor (a list) + - processor_cores + - processor_count + - devices + """ + platform = 'NetBSD' + MEMORY_FACTS = ['MemTotal', 'SwapTotal', 'MemFree', 'SwapFree'] + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.get_cpu_facts() + self.get_memory_facts() + try: + self.get_mount_facts() + except TimeoutError: + pass + return self.facts + + def get_cpu_facts(self): + + i = 0 + physid = 0 + sockets = {} + if not os.access("/proc/cpuinfo", os.R_OK): + return + self.facts['processor'] = [] + for line in open("/proc/cpuinfo").readlines(): + data = line.split(":", 1) + key = data[0].strip() + # model name is for Intel arch, Processor (mind the uppercase P) + # works for some ARM devices, like the Sheevaplug. + if key == 'model name' or key == 'Processor': + if 'processor' not in self.facts: + self.facts['processor'] = [] + self.facts['processor'].append(data[1].strip()) + i += 1 + elif key == 'physical id': + physid = data[1].strip() + if physid not in sockets: + sockets[physid] = 1 + elif key == 'cpu cores': + sockets[physid] = int(data[1].strip()) + if len(sockets) > 0: + self.facts['processor_count'] = len(sockets) + self.facts['processor_cores'] = reduce(lambda x, y: x + y, sockets.values()) + else: + self.facts['processor_count'] = i + self.facts['processor_cores'] = 'NA' + + def get_memory_facts(self): + if not os.access("/proc/meminfo", os.R_OK): + return + for line in open("/proc/meminfo").readlines(): + data = line.split(":", 1) + key = data[0] + if key in NetBSDHardware.MEMORY_FACTS: + val = data[1].strip().split(' ')[0] + self.facts["%s_mb" % key.lower()] = long(val) / 1024 + + @timeout(10) + def get_mount_facts(self): + self.facts['mounts'] = [] + fstab = get_file_content('/etc/fstab') + if fstab: + for line in fstab.split('\n'): + if line.startswith('#') or line.strip() == '': + continue + fields = re.sub(r'\s+',' ',line.rstrip('\n')).split() + self.facts['mounts'].append({'mount': fields[1], 'device': fields[0], 'fstype' : fields[2], 'options': fields[3]}) + +class AIX(Hardware): + """ + AIX-specific subclass of Hardware. Defines memory and CPU facts: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor (a list) + - processor_cores + - processor_count + """ + platform = 'AIX' + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.get_cpu_facts() + self.get_memory_facts() + self.get_dmi_facts() + return self.facts + + def get_cpu_facts(self): + self.facts['processor'] = [] + + + rc, out, err = module.run_command("/usr/sbin/lsdev -Cc processor") + if out: + i = 0 + for line in out.split('\n'): + + if 'Available' in line: + if i == 0: + data = line.split(' ') + cpudev = data[0] + + i += 1 + self.facts['processor_count'] = int(i) + + rc, out, err = module.run_command("/usr/sbin/lsattr -El " + cpudev + " -a type") + + data = out.split(' ') + self.facts['processor'] = data[1] + + rc, out, err = module.run_command("/usr/sbin/lsattr -El " + cpudev + " -a smt_threads") + + data = out.split(' ') + self.facts['processor_cores'] = int(data[1]) + + def get_memory_facts(self): + pagesize = 4096 + rc, out, err = module.run_command("/usr/bin/vmstat -v") + for line in out.split('\n'): + data = line.split() + if 'memory pages' in line: + pagecount = long(data[0]) + if 'free pages' in line: + freecount = long(data[0]) + self.facts['memtotal_mb'] = pagesize * pagecount / 1024 / 1024 + self.facts['memfree_mb'] = pagesize * freecount / 1024 / 1024 + # Get swapinfo. swapinfo output looks like: + # Device 1M-blocks Used Avail Capacity + # /dev/ada0p3 314368 0 314368 0% + # + rc, out, err = module.run_command("/usr/sbin/lsps -s") + if out: + lines = out.split('\n') + data = lines[1].split() + swaptotal_mb = long(data[0].rstrip('MB')) + percused = int(data[1].rstrip('%')) + self.facts['swaptotal_mb'] = swaptotal_mb + self.facts['swapfree_mb'] = long(swaptotal_mb * ( 100 - percused ) / 100) + + def get_dmi_facts(self): + rc, out, err = module.run_command("/usr/sbin/lsattr -El sys0 -a fwversion") + data = out.split() + self.facts['firmware_version'] = data[1].strip('IBM,') + +class HPUX(Hardware): + """ + HP-UX-specifig subclass of Hardware. Defines memory and CPU facts: + - memfree_mb + - memtotal_mb + - swapfree_mb + - swaptotal_mb + - processor + - processor_cores + - processor_count + - model + - firmware + """ + + platform = 'HP-UX' + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.get_cpu_facts() + self.get_memory_facts() + self.get_hw_facts() + return self.facts + + def get_cpu_facts(self): + if self.facts['architecture'] == '9000/800': + rc, out, err = module.run_command("ioscan -FkCprocessor | wc -l", use_unsafe_shell=True) + self.facts['processor_count'] = int(out.strip()) + #Working with machinfo mess + elif self.facts['architecture'] == 'ia64': + if self.facts['distribution_version'] == "B.11.23": + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep 'Number of CPUs'", use_unsafe_shell=True) + self.facts['processor_count'] = int(out.strip().split('=')[1]) + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep 'processor family'", use_unsafe_shell=True) + self.facts['processor'] = re.search('.*(Intel.*)', out).groups()[0].strip() + rc, out, err = module.run_command("ioscan -FkCprocessor | wc -l", use_unsafe_shell=True) + self.facts['processor_cores'] = int(out.strip()) + if self.facts['distribution_version'] == "B.11.31": + #if machinfo return cores strings release B.11.31 > 1204 + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep core | wc -l", use_unsafe_shell=True) + if out.strip()== '0': + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep Intel", use_unsafe_shell=True) + self.facts['processor_count'] = int(out.strip().split(" ")[0]) + #If hyperthreading is active divide cores by 2 + rc, out, err = module.run_command("/usr/sbin/psrset | grep LCPU", use_unsafe_shell=True) + data = re.sub(' +',' ',out).strip().split(' ') + if len(data) == 1: + hyperthreading = 'OFF' + else: + hyperthreading = data[1] + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep logical", use_unsafe_shell=True) + data = out.strip().split(" ") + if hyperthreading == 'ON': + self.facts['processor_cores'] = int(data[0])/2 + else: + if len(data) == 1: + self.facts['processor_cores'] = self.facts['processor_count'] + else: + self.facts['processor_cores'] = int(data[0]) + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep Intel |cut -d' ' -f4-", use_unsafe_shell=True) + self.facts['processor'] = out.strip() + else: + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | egrep 'socket[s]?$' | tail -1", use_unsafe_shell=True) + self.facts['processor_count'] = int(out.strip().split(" ")[0]) + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep -e '[0-9] core' | tail -1", use_unsafe_shell=True) + self.facts['processor_cores'] = int(out.strip().split(" ")[0]) + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep Intel", use_unsafe_shell=True) + self.facts['processor'] = out.strip() + + def get_memory_facts(self): + pagesize = 4096 + rc, out, err = module.run_command("/usr/bin/vmstat | tail -1", use_unsafe_shell=True) + data = int(re.sub(' +',' ',out).split(' ')[5].strip()) + self.facts['memfree_mb'] = pagesize * data / 1024 / 1024 + if self.facts['architecture'] == '9000/800': + try: + rc, out, err = module.run_command("grep Physical /var/adm/syslog/syslog.log") + data = re.search('.*Physical: ([0-9]*) Kbytes.*',out).groups()[0].strip() + self.facts['memtotal_mb'] = int(data) / 1024 + except AttributeError: + #For systems where memory details aren't sent to syslog or the log has rotated, use parsed + #adb output. Unfortunately /dev/kmem doesn't have world-read, so this only works as root. + if os.access("/dev/kmem", os.R_OK): + rc, out, err = module.run_command("echo 'phys_mem_pages/D' | adb -k /stand/vmunix /dev/kmem | tail -1 | awk '{print $2}'", use_unsafe_shell=True) + if not err: + data = out + self.facts['memtotal_mb'] = int(data) / 256 + else: + rc, out, err = module.run_command("/usr/contrib/bin/machinfo | grep Memory", use_unsafe_shell=True) + data = re.search('Memory[\ :=]*([0-9]*).*MB.*',out).groups()[0].strip() + self.facts['memtotal_mb'] = int(data) + rc, out, err = module.run_command("/usr/sbin/swapinfo -m -d -f -q") + self.facts['swaptotal_mb'] = int(out.strip()) + rc, out, err = module.run_command("/usr/sbin/swapinfo -m -d -f | egrep '^dev|^fs'", use_unsafe_shell=True) + swap = 0 + for line in out.strip().split('\n'): + swap += int(re.sub(' +',' ',line).split(' ')[3].strip()) + self.facts['swapfree_mb'] = swap + + def get_hw_facts(self): + rc, out, err = module.run_command("model") + self.facts['model'] = out.strip() + if self.facts['architecture'] == 'ia64': + separator = ':' + if self.facts['distribution_version'] == "B.11.23": + separator = '=' + rc, out, err = module.run_command("/usr/contrib/bin/machinfo |grep -i 'Firmware revision' | grep -v BMC", use_unsafe_shell=True) + self.facts['firmware_version'] = out.split(separator)[1].strip() + + +class Darwin(Hardware): + """ + Darwin-specific subclass of Hardware. Defines memory and CPU facts: + - processor + - processor_cores + - memtotal_mb + - memfree_mb + - model + - osversion + - osrevision + """ + platform = 'Darwin' + + def __init__(self): + Hardware.__init__(self) + + def populate(self): + self.sysctl = self.get_sysctl() + self.get_mac_facts() + self.get_cpu_facts() + self.get_memory_facts() + return self.facts + + def get_sysctl(self): + rc, out, err = module.run_command(["/usr/sbin/sysctl", "hw", "machdep", "kern"]) + if rc != 0: + return dict() + sysctl = dict() + for line in out.splitlines(): + if line.rstrip("\n"): + (key, value) = re.split(' = |: ', line, maxsplit=1) + sysctl[key] = value.strip() + return sysctl + + def get_system_profile(self): + rc, out, err = module.run_command(["/usr/sbin/system_profiler", "SPHardwareDataType"]) + if rc != 0: + return dict() + system_profile = dict() + for line in out.splitlines(): + if ': ' in line: + (key, value) = line.split(': ', 1) + system_profile[key.strip()] = ' '.join(value.strip().split()) + return system_profile + + def get_mac_facts(self): + rc, out, err = module.run_command("sysctl hw.model") + if rc == 0: + self.facts['model'] = out.splitlines()[-1].split()[1] + self.facts['osversion'] = self.sysctl['kern.osversion'] + self.facts['osrevision'] = self.sysctl['kern.osrevision'] + + def get_cpu_facts(self): + if 'machdep.cpu.brand_string' in self.sysctl: # Intel + self.facts['processor'] = self.sysctl['machdep.cpu.brand_string'] + self.facts['processor_cores'] = self.sysctl['machdep.cpu.core_count'] + else: # PowerPC + system_profile = self.get_system_profile() + self.facts['processor'] = '%s @ %s' % (system_profile['Processor Name'], system_profile['Processor Speed']) + self.facts['processor_cores'] = self.sysctl['hw.physicalcpu'] + + def get_memory_facts(self): + self.facts['memtotal_mb'] = long(self.sysctl['hw.memsize']) / 1024 / 1024 + + rc, out, err = module.run_command("sysctl hw.usermem") + if rc == 0: + self.facts['memfree_mb'] = long(out.splitlines()[-1].split()[1]) / 1024 / 1024 + +class Network(Facts): + """ + This is a generic Network subclass of Facts. This should be further + subclassed to implement per platform. If you subclass this, + you must define: + - interfaces (a list of interface names) + - interface_<name> dictionary of ipv4, ipv6, and mac address information. + + All subclasses MUST define platform. + """ + platform = 'Generic' + + IPV6_SCOPE = { '0' : 'global', + '10' : 'host', + '20' : 'link', + '40' : 'admin', + '50' : 'site', + '80' : 'organization' } + + def __new__(cls, *arguments, **keyword): + subclass = cls + for sc in Network.__subclasses__(): + if sc.platform == platform.system(): + subclass = sc + return super(cls, subclass).__new__(subclass, *arguments, **keyword) + + def __init__(self, module): + self.module = module + Facts.__init__(self) + + def populate(self): + return self.facts + +class LinuxNetwork(Network): + """ + This is a Linux-specific subclass of Network. It defines + - interfaces (a list of interface names) + - interface_<name> dictionary of ipv4, ipv6, and mac address information. + - all_ipv4_addresses and all_ipv6_addresses: lists of all configured addresses. + - ipv4_address and ipv6_address: the first non-local address for each family. + """ + platform = 'Linux' + + def __init__(self, module): + Network.__init__(self, module) + + def populate(self): + ip_path = self.module.get_bin_path('ip') + if ip_path is None: + return self.facts + default_ipv4, default_ipv6 = self.get_default_interfaces(ip_path) + interfaces, ips = self.get_interfaces_info(ip_path, default_ipv4, default_ipv6) + self.facts['interfaces'] = interfaces.keys() + for iface in interfaces: + self.facts[iface] = interfaces[iface] + self.facts['default_ipv4'] = default_ipv4 + self.facts['default_ipv6'] = default_ipv6 + self.facts['all_ipv4_addresses'] = ips['all_ipv4_addresses'] + self.facts['all_ipv6_addresses'] = ips['all_ipv6_addresses'] + return self.facts + + def get_default_interfaces(self, ip_path): + # Use the commands: + # ip -4 route get 8.8.8.8 -> Google public DNS + # ip -6 route get 2404:6800:400a:800::1012 -> ipv6.google.com + # to find out the default outgoing interface, address, and gateway + command = dict( + v4 = [ip_path, '-4', 'route', 'get', '8.8.8.8'], + v6 = [ip_path, '-6', 'route', 'get', '2404:6800:400a:800::1012'] + ) + interface = dict(v4 = {}, v6 = {}) + for v in 'v4', 'v6': + if v == 'v6' and self.facts['os_family'] == 'RedHat' \ + and self.facts['distribution_version'].startswith('4.'): + continue + if v == 'v6' and not socket.has_ipv6: + continue + rc, out, err = module.run_command(command[v]) + if not out: + # v6 routing may result in + # RTNETLINK answers: Invalid argument + continue + words = out.split('\n')[0].split() + # A valid output starts with the queried address on the first line + if len(words) > 0 and words[0] == command[v][-1]: + for i in range(len(words) - 1): + if words[i] == 'dev': + interface[v]['interface'] = words[i+1] + elif words[i] == 'src': + interface[v]['address'] = words[i+1] + elif words[i] == 'via' and words[i+1] != command[v][-1]: + interface[v]['gateway'] = words[i+1] + return interface['v4'], interface['v6'] + + def get_interfaces_info(self, ip_path, default_ipv4, default_ipv6): + interfaces = {} + ips = dict( + all_ipv4_addresses = [], + all_ipv6_addresses = [], + ) + + for path in glob.glob('/sys/class/net/*'): + if not os.path.isdir(path): + continue + device = os.path.basename(path) + interfaces[device] = { 'device': device } + if os.path.exists(os.path.join(path, 'address')): + macaddress = open(os.path.join(path, 'address')).read().strip() + if macaddress and macaddress != '00:00:00:00:00:00': + interfaces[device]['macaddress'] = macaddress + if os.path.exists(os.path.join(path, 'mtu')): + interfaces[device]['mtu'] = int(open(os.path.join(path, 'mtu')).read().strip()) + if os.path.exists(os.path.join(path, 'operstate')): + interfaces[device]['active'] = open(os.path.join(path, 'operstate')).read().strip() != 'down' +# if os.path.exists(os.path.join(path, 'carrier')): +# interfaces[device]['link'] = open(os.path.join(path, 'carrier')).read().strip() == '1' + if os.path.exists(os.path.join(path, 'device','driver', 'module')): + interfaces[device]['module'] = os.path.basename(os.path.realpath(os.path.join(path, 'device', 'driver', 'module'))) + if os.path.exists(os.path.join(path, 'type')): + type = open(os.path.join(path, 'type')).read().strip() + if type == '1': + interfaces[device]['type'] = 'ether' + elif type == '512': + interfaces[device]['type'] = 'ppp' + elif type == '772': + interfaces[device]['type'] = 'loopback' + if os.path.exists(os.path.join(path, 'bridge')): + interfaces[device]['type'] = 'bridge' + interfaces[device]['interfaces'] = [ os.path.basename(b) for b in glob.glob(os.path.join(path, 'brif', '*')) ] + if os.path.exists(os.path.join(path, 'bridge', 'bridge_id')): + interfaces[device]['id'] = open(os.path.join(path, 'bridge', 'bridge_id')).read().strip() + if os.path.exists(os.path.join(path, 'bridge', 'stp_state')): + interfaces[device]['stp'] = open(os.path.join(path, 'bridge', 'stp_state')).read().strip() == '1' + if os.path.exists(os.path.join(path, 'bonding')): + interfaces[device]['type'] = 'bonding' + interfaces[device]['slaves'] = open(os.path.join(path, 'bonding', 'slaves')).read().split() + interfaces[device]['mode'] = open(os.path.join(path, 'bonding', 'mode')).read().split()[0] + interfaces[device]['miimon'] = open(os.path.join(path, 'bonding', 'miimon')).read().split()[0] + interfaces[device]['lacp_rate'] = open(os.path.join(path, 'bonding', 'lacp_rate')).read().split()[0] + primary = open(os.path.join(path, 'bonding', 'primary')).read() + if primary: + interfaces[device]['primary'] = primary + path = os.path.join(path, 'bonding', 'all_slaves_active') + if os.path.exists(path): + interfaces[device]['all_slaves_active'] = open(path).read() == '1' + + # Check whether an interface is in promiscuous mode + if os.path.exists(os.path.join(path,'flags')): + promisc_mode = False + # The second byte indicates whether the interface is in promiscuous mode. + # 1 = promisc + # 0 = no promisc + data = int(open(os.path.join(path, 'flags')).read().strip(),16) + promisc_mode = (data & 0x0100 > 0) + interfaces[device]['promisc'] = promisc_mode + + def parse_ip_output(output, secondary=False): + for line in output.split('\n'): + if not line: + continue + words = line.split() + if words[0] == 'inet': + if '/' in words[1]: + address, netmask_length = words[1].split('/') + else: + # pointopoint interfaces do not have a prefix + address = words[1] + netmask_length = "32" + address_bin = struct.unpack('!L', socket.inet_aton(address))[0] + netmask_bin = (1<<32) - (1<<32>>int(netmask_length)) + netmask = socket.inet_ntoa(struct.pack('!L', netmask_bin)) + network = socket.inet_ntoa(struct.pack('!L', address_bin & netmask_bin)) + iface = words[-1] + if iface != device: + interfaces[iface] = {} + if not secondary and "ipv4" not in interfaces[iface]: + interfaces[iface]['ipv4'] = {'address': address, + 'netmask': netmask, + 'network': network} + else: + if "ipv4_secondaries" not in interfaces[iface]: + interfaces[iface]["ipv4_secondaries"] = [] + interfaces[iface]["ipv4_secondaries"].append({ + 'address': address, + 'netmask': netmask, + 'network': network, + }) + + # add this secondary IP to the main device + if secondary: + if "ipv4_secondaries" not in interfaces[device]: + interfaces[device]["ipv4_secondaries"] = [] + interfaces[device]["ipv4_secondaries"].append({ + 'address': address, + 'netmask': netmask, + 'network': network, + }) + + # If this is the default address, update default_ipv4 + if 'address' in default_ipv4 and default_ipv4['address'] == address: + default_ipv4['netmask'] = netmask + default_ipv4['network'] = network + default_ipv4['macaddress'] = macaddress + default_ipv4['mtu'] = interfaces[device]['mtu'] + default_ipv4['type'] = interfaces[device].get("type", "unknown") + default_ipv4['alias'] = words[-1] + if not address.startswith('127.'): + ips['all_ipv4_addresses'].append(address) + elif words[0] == 'inet6': + address, prefix = words[1].split('/') + scope = words[3] + if 'ipv6' not in interfaces[device]: + interfaces[device]['ipv6'] = [] + interfaces[device]['ipv6'].append({ + 'address' : address, + 'prefix' : prefix, + 'scope' : scope + }) + # If this is the default address, update default_ipv6 + if 'address' in default_ipv6 and default_ipv6['address'] == address: + default_ipv6['prefix'] = prefix + default_ipv6['scope'] = scope + default_ipv6['macaddress'] = macaddress + default_ipv6['mtu'] = interfaces[device]['mtu'] + default_ipv6['type'] = interfaces[device].get("type", "unknown") + if not address == '::1': + ips['all_ipv6_addresses'].append(address) + + ip_path = module.get_bin_path("ip") + + args = [ip_path, 'addr', 'show', 'primary', device] + rc, stdout, stderr = self.module.run_command(args) + primary_data = stdout + + args = [ip_path, 'addr', 'show', 'secondary', device] + rc, stdout, stderr = self.module.run_command(args) + secondary_data = stdout + + parse_ip_output(primary_data) + parse_ip_output(secondary_data, secondary=True) + + # replace : by _ in interface name since they are hard to use in template + new_interfaces = {} + for i in interfaces: + if ':' in i: + new_interfaces[i.replace(':','_')] = interfaces[i] + else: + new_interfaces[i] = interfaces[i] + return new_interfaces, ips + +class GenericBsdIfconfigNetwork(Network): + """ + This is a generic BSD subclass of Network using the ifconfig command. + It defines + - interfaces (a list of interface names) + - interface_<name> dictionary of ipv4, ipv6, and mac address information. + - all_ipv4_addresses and all_ipv6_addresses: lists of all configured addresses. + It currently does not define + - default_ipv4 and default_ipv6 + - type, mtu and network on interfaces + """ + platform = 'Generic_BSD_Ifconfig' + + def __init__(self, module): + Network.__init__(self, module) + + def populate(self): + + ifconfig_path = module.get_bin_path('ifconfig') + + if ifconfig_path is None: + return self.facts + route_path = module.get_bin_path('route') + + if route_path is None: + return self.facts + + default_ipv4, default_ipv6 = self.get_default_interfaces(route_path) + interfaces, ips = self.get_interfaces_info(ifconfig_path) + self.merge_default_interface(default_ipv4, interfaces, 'ipv4') + self.merge_default_interface(default_ipv6, interfaces, 'ipv6') + self.facts['interfaces'] = interfaces.keys() + + for iface in interfaces: + self.facts[iface] = interfaces[iface] + + self.facts['default_ipv4'] = default_ipv4 + self.facts['default_ipv6'] = default_ipv6 + self.facts['all_ipv4_addresses'] = ips['all_ipv4_addresses'] + self.facts['all_ipv6_addresses'] = ips['all_ipv6_addresses'] + + return self.facts + + def get_default_interfaces(self, route_path): + + # Use the commands: + # route -n get 8.8.8.8 -> Google public DNS + # route -n get -inet6 2404:6800:400a:800::1012 -> ipv6.google.com + # to find out the default outgoing interface, address, and gateway + + command = dict( + v4 = [route_path, '-n', 'get', '8.8.8.8'], + v6 = [route_path, '-n', 'get', '-inet6', '2404:6800:400a:800::1012'] + ) + + interface = dict(v4 = {}, v6 = {}) + + for v in 'v4', 'v6': + + if v == 'v6' and not socket.has_ipv6: + continue + rc, out, err = module.run_command(command[v]) + if not out: + # v6 routing may result in + # RTNETLINK answers: Invalid argument + continue + lines = out.split('\n') + for line in lines: + words = line.split() + # Collect output from route command + if len(words) > 1: + if words[0] == 'interface:': + interface[v]['interface'] = words[1] + if words[0] == 'gateway:': + interface[v]['gateway'] = words[1] + + return interface['v4'], interface['v6'] + + def get_interfaces_info(self, ifconfig_path): + interfaces = {} + current_if = {} + ips = dict( + all_ipv4_addresses = [], + all_ipv6_addresses = [], + ) + # FreeBSD, DragonflyBSD, NetBSD, OpenBSD and OS X all implicitly add '-a' + # when running the command 'ifconfig'. + # Solaris must explicitly run the command 'ifconfig -a'. + rc, out, err = module.run_command([ifconfig_path, '-a']) + + for line in out.split('\n'): + + if line: + words = line.split() + + if words[0] == 'pass': + continue + elif re.match('^\S', line) and len(words) > 3: + current_if = self.parse_interface_line(words) + interfaces[ current_if['device'] ] = current_if + elif words[0].startswith('options='): + self.parse_options_line(words, current_if, ips) + elif words[0] == 'nd6': + self.parse_nd6_line(words, current_if, ips) + elif words[0] == 'ether': + self.parse_ether_line(words, current_if, ips) + elif words[0] == 'media:': + self.parse_media_line(words, current_if, ips) + elif words[0] == 'status:': + self.parse_status_line(words, current_if, ips) + elif words[0] == 'lladdr': + self.parse_lladdr_line(words, current_if, ips) + elif words[0] == 'inet': + self.parse_inet_line(words, current_if, ips) + elif words[0] == 'inet6': + self.parse_inet6_line(words, current_if, ips) + else: + self.parse_unknown_line(words, current_if, ips) + + return interfaces, ips + + def parse_interface_line(self, words): + device = words[0][0:-1] + current_if = {'device': device, 'ipv4': [], 'ipv6': [], 'type': 'unknown'} + current_if['flags'] = self.get_options(words[1]) + current_if['macaddress'] = 'unknown' # will be overwritten later + + if len(words) >= 5 : # Newer FreeBSD versions + current_if['metric'] = words[3] + current_if['mtu'] = words[5] + else: + current_if['mtu'] = words[3] + + return current_if + + def parse_options_line(self, words, current_if, ips): + # Mac has options like this... + current_if['options'] = self.get_options(words[0]) + + def parse_nd6_line(self, words, current_if, ips): + # FreBSD has options like this... + current_if['options'] = self.get_options(words[1]) + + def parse_ether_line(self, words, current_if, ips): + current_if['macaddress'] = words[1] + + def parse_media_line(self, words, current_if, ips): + # not sure if this is useful - we also drop information + current_if['media'] = words[1] + if len(words) > 2: + current_if['media_select'] = words[2] + if len(words) > 3: + current_if['media_type'] = words[3][1:] + if len(words) > 4: + current_if['media_options'] = self.get_options(words[4]) + + def parse_status_line(self, words, current_if, ips): + current_if['status'] = words[1] + + def parse_lladdr_line(self, words, current_if, ips): + current_if['lladdr'] = words[1] + + def parse_inet_line(self, words, current_if, ips): + address = {'address': words[1]} + # deal with hex netmask + if re.match('([0-9a-f]){8}', words[3]) and len(words[3]) == 8: + words[3] = '0x' + words[3] + if words[3].startswith('0x'): + address['netmask'] = socket.inet_ntoa(struct.pack('!L', int(words[3], base=16))) + else: + # otherwise assume this is a dotted quad + address['netmask'] = words[3] + # calculate the network + address_bin = struct.unpack('!L', socket.inet_aton(address['address']))[0] + netmask_bin = struct.unpack('!L', socket.inet_aton(address['netmask']))[0] + address['network'] = socket.inet_ntoa(struct.pack('!L', address_bin & netmask_bin)) + # broadcast may be given or we need to calculate + if len(words) > 5: + address['broadcast'] = words[5] + else: + address['broadcast'] = socket.inet_ntoa(struct.pack('!L', address_bin | (~netmask_bin & 0xffffffff))) + # add to our list of addresses + if not words[1].startswith('127.'): + ips['all_ipv4_addresses'].append(address['address']) + current_if['ipv4'].append(address) + + def parse_inet6_line(self, words, current_if, ips): + address = {'address': words[1]} + if (len(words) >= 4) and (words[2] == 'prefixlen'): + address['prefix'] = words[3] + if (len(words) >= 6) and (words[4] == 'scopeid'): + address['scope'] = words[5] + localhost6 = ['::1', '::1/128', 'fe80::1%lo0'] + if address['address'] not in localhost6: + ips['all_ipv6_addresses'].append(address['address']) + current_if['ipv6'].append(address) + + def parse_unknown_line(self, words, current_if, ips): + # we are going to ignore unknown lines here - this may be + # a bad idea - but you can override it in your subclass + pass + + def get_options(self, option_string): + start = option_string.find('<') + 1 + end = option_string.rfind('>') + if (start > 0) and (end > 0) and (end > start + 1): + option_csv = option_string[start:end] + return option_csv.split(',') + else: + return [] + + def merge_default_interface(self, defaults, interfaces, ip_type): + if not 'interface' in defaults.keys(): + return + if not defaults['interface'] in interfaces: + return + ifinfo = interfaces[defaults['interface']] + # copy all the interface values across except addresses + for item in ifinfo.keys(): + if item != 'ipv4' and item != 'ipv6': + defaults[item] = ifinfo[item] + if len(ifinfo[ip_type]) > 0: + for item in ifinfo[ip_type][0].keys(): + defaults[item] = ifinfo[ip_type][0][item] + +class DarwinNetwork(GenericBsdIfconfigNetwork, Network): + """ + This is the Mac OS X/Darwin Network Class. + It uses the GenericBsdIfconfigNetwork unchanged + """ + platform = 'Darwin' + + # media line is different to the default FreeBSD one + def parse_media_line(self, words, current_if, ips): + # not sure if this is useful - we also drop information + current_if['media'] = 'Unknown' # Mac does not give us this + current_if['media_select'] = words[1] + if len(words) > 2: + current_if['media_type'] = words[2][1:] + if len(words) > 3: + current_if['media_options'] = self.get_options(words[3]) + + +class FreeBSDNetwork(GenericBsdIfconfigNetwork, Network): + """ + This is the FreeBSD Network Class. + It uses the GenericBsdIfconfigNetwork unchanged. + """ + platform = 'FreeBSD' + +class AIXNetwork(GenericBsdIfconfigNetwork, Network): + """ + This is the AIX Network Class. + It uses the GenericBsdIfconfigNetwork unchanged. + """ + platform = 'AIX' + + # AIX 'ifconfig -a' does not have three words in the interface line + def get_interfaces_info(self, ifconfig_path): + interfaces = {} + current_if = {} + ips = dict( + all_ipv4_addresses = [], + all_ipv6_addresses = [], + ) + rc, out, err = module.run_command([ifconfig_path, '-a']) + + for line in out.split('\n'): + + if line: + words = line.split() + + # only this condition differs from GenericBsdIfconfigNetwork + if re.match('^\w*\d*:', line): + current_if = self.parse_interface_line(words) + interfaces[ current_if['device'] ] = current_if + elif words[0].startswith('options='): + self.parse_options_line(words, current_if, ips) + elif words[0] == 'nd6': + self.parse_nd6_line(words, current_if, ips) + elif words[0] == 'ether': + self.parse_ether_line(words, current_if, ips) + elif words[0] == 'media:': + self.parse_media_line(words, current_if, ips) + elif words[0] == 'status:': + self.parse_status_line(words, current_if, ips) + elif words[0] == 'lladdr': + self.parse_lladdr_line(words, current_if, ips) + elif words[0] == 'inet': + self.parse_inet_line(words, current_if, ips) + elif words[0] == 'inet6': + self.parse_inet6_line(words, current_if, ips) + else: + self.parse_unknown_line(words, current_if, ips) + + return interfaces, ips + + # AIX 'ifconfig -a' does not inform about MTU, so remove current_if['mtu'] here + def parse_interface_line(self, words): + device = words[0][0:-1] + current_if = {'device': device, 'ipv4': [], 'ipv6': [], 'type': 'unknown'} + current_if['flags'] = self.get_options(words[1]) + current_if['macaddress'] = 'unknown' # will be overwritten later + return current_if + +class OpenBSDNetwork(GenericBsdIfconfigNetwork, Network): + """ + This is the OpenBSD Network Class. + It uses the GenericBsdIfconfigNetwork. + """ + platform = 'OpenBSD' + + # Return macaddress instead of lladdr + def parse_lladdr_line(self, words, current_if, ips): + current_if['macaddress'] = words[1] + +class SunOSNetwork(GenericBsdIfconfigNetwork, Network): + """ + This is the SunOS Network Class. + It uses the GenericBsdIfconfigNetwork. + + Solaris can have different FLAGS and MTU for IPv4 and IPv6 on the same interface + so these facts have been moved inside the 'ipv4' and 'ipv6' lists. + """ + platform = 'SunOS' + + # Solaris 'ifconfig -a' will print interfaces twice, once for IPv4 and again for IPv6. + # MTU and FLAGS also may differ between IPv4 and IPv6 on the same interface. + # 'parse_interface_line()' checks for previously seen interfaces before defining + # 'current_if' so that IPv6 facts don't clobber IPv4 facts (or vice versa). + def get_interfaces_info(self, ifconfig_path): + interfaces = {} + current_if = {} + ips = dict( + all_ipv4_addresses = [], + all_ipv6_addresses = [], + ) + rc, out, err = module.run_command([ifconfig_path, '-a']) + + for line in out.split('\n'): + + if line: + words = line.split() + + if re.match('^\S', line) and len(words) > 3: + current_if = self.parse_interface_line(words, current_if, interfaces) + interfaces[ current_if['device'] ] = current_if + elif words[0].startswith('options='): + self.parse_options_line(words, current_if, ips) + elif words[0] == 'nd6': + self.parse_nd6_line(words, current_if, ips) + elif words[0] == 'ether': + self.parse_ether_line(words, current_if, ips) + elif words[0] == 'media:': + self.parse_media_line(words, current_if, ips) + elif words[0] == 'status:': + self.parse_status_line(words, current_if, ips) + elif words[0] == 'lladdr': + self.parse_lladdr_line(words, current_if, ips) + elif words[0] == 'inet': + self.parse_inet_line(words, current_if, ips) + elif words[0] == 'inet6': + self.parse_inet6_line(words, current_if, ips) + else: + self.parse_unknown_line(words, current_if, ips) + + # 'parse_interface_line' and 'parse_inet*_line' leave two dicts in the + # ipv4/ipv6 lists which is ugly and hard to read. + # This quick hack merges the dictionaries. Purely cosmetic. + for iface in interfaces: + for v in 'ipv4', 'ipv6': + combined_facts = {} + for facts in interfaces[iface][v]: + combined_facts.update(facts) + if len(combined_facts.keys()) > 0: + interfaces[iface][v] = [combined_facts] + + return interfaces, ips + + def parse_interface_line(self, words, current_if, interfaces): + device = words[0][0:-1] + if device not in interfaces.keys(): + current_if = {'device': device, 'ipv4': [], 'ipv6': [], 'type': 'unknown'} + else: + current_if = interfaces[device] + flags = self.get_options(words[1]) + v = 'ipv4' + if 'IPv6' in flags: + v = 'ipv6' + current_if[v].append({'flags': flags, 'mtu': words[3]}) + current_if['macaddress'] = 'unknown' # will be overwritten later + return current_if + + # Solaris displays single digit octets in MAC addresses e.g. 0:1:2:d:e:f + # Add leading zero to each octet where needed. + def parse_ether_line(self, words, current_if, ips): + macaddress = '' + for octet in words[1].split(':'): + octet = ('0' + octet)[-2:None] + macaddress += (octet + ':') + current_if['macaddress'] = macaddress[0:-1] + +class Virtual(Facts): + """ + This is a generic Virtual subclass of Facts. This should be further + subclassed to implement per platform. If you subclass this, + you should define: + - virtualization_type + - virtualization_role + - container (e.g. solaris zones, freebsd jails, linux containers) + + All subclasses MUST define platform. + """ + + def __new__(cls, *arguments, **keyword): + subclass = cls + for sc in Virtual.__subclasses__(): + if sc.platform == platform.system(): + subclass = sc + return super(cls, subclass).__new__(subclass, *arguments, **keyword) + + def __init__(self): + Facts.__init__(self) + + def populate(self): + return self.facts + +class LinuxVirtual(Virtual): + """ + This is a Linux-specific subclass of Virtual. It defines + - virtualization_type + - virtualization_role + """ + platform = 'Linux' + + def __init__(self): + Virtual.__init__(self) + + def populate(self): + self.get_virtual_facts() + return self.facts + + # For more information, check: http://people.redhat.com/~rjones/virt-what/ + def get_virtual_facts(self): + if os.path.exists("/proc/xen"): + self.facts['virtualization_type'] = 'xen' + self.facts['virtualization_role'] = 'guest' + try: + for line in open('/proc/xen/capabilities'): + if "control_d" in line: + self.facts['virtualization_role'] = 'host' + except IOError: + pass + return + + if os.path.exists('/proc/vz'): + self.facts['virtualization_type'] = 'openvz' + if os.path.exists('/proc/bc'): + self.facts['virtualization_role'] = 'host' + else: + self.facts['virtualization_role'] = 'guest' + return + + if os.path.exists('/proc/1/cgroup'): + for line in open('/proc/1/cgroup').readlines(): + if re.search('/docker/', line): + self.facts['virtualization_type'] = 'docker' + self.facts['virtualization_role'] = 'guest' + return + if re.search('/lxc/', line): + self.facts['virtualization_type'] = 'lxc' + self.facts['virtualization_role'] = 'guest' + return + + product_name = get_file_content('/sys/devices/virtual/dmi/id/product_name') + + if product_name in ['KVM', 'Bochs']: + self.facts['virtualization_type'] = 'kvm' + self.facts['virtualization_role'] = 'guest' + return + + if product_name == 'RHEV Hypervisor': + self.facts['virtualization_type'] = 'RHEV' + self.facts['virtualization_role'] = 'guest' + return + + if product_name == 'VMware Virtual Platform': + self.facts['virtualization_type'] = 'VMware' + self.facts['virtualization_role'] = 'guest' + return + + bios_vendor = get_file_content('/sys/devices/virtual/dmi/id/bios_vendor') + + if bios_vendor == 'Xen': + self.facts['virtualization_type'] = 'xen' + self.facts['virtualization_role'] = 'guest' + return + + if bios_vendor == 'innotek GmbH': + self.facts['virtualization_type'] = 'virtualbox' + self.facts['virtualization_role'] = 'guest' + return + + sys_vendor = get_file_content('/sys/devices/virtual/dmi/id/sys_vendor') + + # FIXME: This does also match hyperv + if sys_vendor == 'Microsoft Corporation': + self.facts['virtualization_type'] = 'VirtualPC' + self.facts['virtualization_role'] = 'guest' + return + + if sys_vendor == 'Parallels Software International Inc.': + self.facts['virtualization_type'] = 'parallels' + self.facts['virtualization_role'] = 'guest' + return + + if sys_vendor == 'QEMU': + self.facts['virtualization_type'] = 'kvm' + self.facts['virtualization_role'] = 'guest' + return + + if os.path.exists('/proc/self/status'): + for line in open('/proc/self/status').readlines(): + if re.match('^VxID: \d+', line): + self.facts['virtualization_type'] = 'linux_vserver' + if re.match('^VxID: 0', line): + self.facts['virtualization_role'] = 'host' + else: + self.facts['virtualization_role'] = 'guest' + return + + if os.path.exists('/proc/cpuinfo'): + for line in open('/proc/cpuinfo').readlines(): + if re.match('^model name.*QEMU Virtual CPU', line): + self.facts['virtualization_type'] = 'kvm' + elif re.match('^vendor_id.*User Mode Linux', line): + self.facts['virtualization_type'] = 'uml' + elif re.match('^model name.*UML', line): + self.facts['virtualization_type'] = 'uml' + elif re.match('^vendor_id.*PowerVM Lx86', line): + self.facts['virtualization_type'] = 'powervm_lx86' + elif re.match('^vendor_id.*IBM/S390', line): + self.facts['virtualization_type'] = 'PR/SM' + lscpu = module.get_bin_path('lscpu') + if lscpu: + rc, out, err = module.run_command(["lscpu"]) + if rc == 0: + for line in out.split("\n"): + data = line.split(":", 1) + key = data[0].strip() + if key == 'Hypervisor': + self.facts['virtualization_type'] = data[1].strip() + else: + self.facts['virtualization_type'] = 'ibm_systemz' + else: + continue + if self.facts['virtualization_type'] == 'PR/SM': + self.facts['virtualization_role'] = 'LPAR' + else: + self.facts['virtualization_role'] = 'guest' + return + + # Beware that we can have both kvm and virtualbox running on a single system + if os.path.exists("/proc/modules") and os.access('/proc/modules', os.R_OK): + modules = [] + for line in open("/proc/modules").readlines(): + data = line.split(" ", 1) + modules.append(data[0]) + + if 'kvm' in modules: + self.facts['virtualization_type'] = 'kvm' + self.facts['virtualization_role'] = 'host' + return + + if 'vboxdrv' in modules: + self.facts['virtualization_type'] = 'virtualbox' + self.facts['virtualization_role'] = 'host' + return + + # If none of the above matches, return 'NA' for virtualization_type + # and virtualization_role. This allows for proper grouping. + self.facts['virtualization_type'] = 'NA' + self.facts['virtualization_role'] = 'NA' + return + + +class HPUXVirtual(Virtual): + """ + This is a HP-UX specific subclass of Virtual. It defines + - virtualization_type + - virtualization_role + """ + platform = 'HP-UX' + + def __init__(self): + Virtual.__init__(self) + + def populate(self): + self.get_virtual_facts() + return self.facts + + def get_virtual_facts(self): + if os.path.exists('/usr/sbin/vecheck'): + rc, out, err = module.run_command("/usr/sbin/vecheck") + if rc == 0: + self.facts['virtualization_type'] = 'guest' + self.facts['virtualization_role'] = 'HP vPar' + if os.path.exists('/opt/hpvm/bin/hpvminfo'): + rc, out, err = module.run_command("/opt/hpvm/bin/hpvminfo") + if rc == 0 and re.match('.*Running.*HPVM vPar.*', out): + self.facts['virtualization_type'] = 'guest' + self.facts['virtualization_role'] = 'HPVM vPar' + elif rc == 0 and re.match('.*Running.*HPVM guest.*', out): + self.facts['virtualization_type'] = 'guest' + self.facts['virtualization_role'] = 'HPVM IVM' + elif rc == 0 and re.match('.*Running.*HPVM host.*', out): + self.facts['virtualization_type'] = 'host' + self.facts['virtualization_role'] = 'HPVM' + if os.path.exists('/usr/sbin/parstatus'): + rc, out, err = module.run_command("/usr/sbin/parstatus") + if rc == 0: + self.facts['virtualization_type'] = 'guest' + self.facts['virtualization_role'] = 'HP nPar' + + +class SunOSVirtual(Virtual): + """ + This is a SunOS-specific subclass of Virtual. It defines + - virtualization_type + - virtualization_role + - container + """ + platform = 'SunOS' + + def __init__(self): + Virtual.__init__(self) + + def populate(self): + self.get_virtual_facts() + return self.facts + + def get_virtual_facts(self): + rc, out, err = module.run_command("/usr/sbin/prtdiag") + for line in out.split('\n'): + if 'VMware' in line: + self.facts['virtualization_type'] = 'vmware' + self.facts['virtualization_role'] = 'guest' + if 'Parallels' in line: + self.facts['virtualization_type'] = 'parallels' + self.facts['virtualization_role'] = 'guest' + if 'VirtualBox' in line: + self.facts['virtualization_type'] = 'virtualbox' + self.facts['virtualization_role'] = 'guest' + if 'HVM domU' in line: + self.facts['virtualization_type'] = 'xen' + self.facts['virtualization_role'] = 'guest' + # Check if it's a zone + if os.path.exists("/usr/bin/zonename"): + rc, out, err = module.run_command("/usr/bin/zonename") + if out.rstrip() != "global": + self.facts['container'] = 'zone' + # Check if it's a branded zone (i.e. Solaris 8/9 zone) + if os.path.isdir('/.SUNWnative'): + self.facts['container'] = 'zone' + # If it's a zone check if we can detect if our global zone is itself virtualized. + # Relies on the "guest tools" (e.g. vmware tools) to be installed + if 'container' in self.facts and self.facts['container'] == 'zone': + rc, out, err = module.run_command("/usr/sbin/modinfo") + for line in out.split('\n'): + if 'VMware' in line: + self.facts['virtualization_type'] = 'vmware' + self.facts['virtualization_role'] = 'guest' + if 'VirtualBox' in line: + self.facts['virtualization_type'] = 'virtualbox' + self.facts['virtualization_role'] = 'guest' + +def get_file_content(path, default=None): + data = default + if os.path.exists(path) and os.access(path, os.R_OK): + data = open(path).read().strip() + if len(data) == 0: + data = default + return data + +def ansible_facts(module): + facts = {} + facts.update(Facts().populate()) + facts.update(Hardware().populate()) + facts.update(Network(module).populate()) + facts.update(Virtual().populate()) + return facts + +# =========================================== + +def get_all_facts(module): + + setup_options = dict(module_setup=True) + facts = ansible_facts(module) + + for (k, v) in facts.items(): + setup_options["ansible_%s" % k.replace('-', '_')] = v + + # Look for the path to the facter and ohai binary and set + # the variable to that path. + + facter_path = module.get_bin_path('facter') + ohai_path = module.get_bin_path('ohai') + + # if facter is installed, and we can use --json because + # ruby-json is ALSO installed, include facter data in the JSON + + if facter_path is not None: + rc, out, err = module.run_command(facter_path + " --json") + facter = True + try: + facter_ds = json.loads(out) + except: + facter = False + if facter: + for (k,v) in facter_ds.items(): + setup_options["facter_%s" % k] = v + + # ditto for ohai + + if ohai_path is not None: + rc, out, err = module.run_command(ohai_path) + ohai = True + try: + ohai_ds = json.loads(out) + except: + ohai = False + if ohai: + for (k,v) in ohai_ds.items(): + k2 = "ohai_%s" % k.replace('-', '_') + setup_options[k2] = v + + setup_result = { 'ansible_facts': {} } + + for (k,v) in setup_options.items(): + if module.params['filter'] == '*' or fnmatch.fnmatch(k, module.params['filter']): + setup_result['ansible_facts'][k] = v + + # hack to keep --verbose from showing all the setup module results + setup_result['verbose_override'] = True + + return setup_result + diff --git a/v2/ansible/module_utils/gce.py b/v2/ansible/module_utils/gce.py new file mode 100644 index 0000000000..37a4bf1dea --- /dev/null +++ b/v2/ansible/module_utils/gce.py @@ -0,0 +1,93 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Franck Cuny <franck.cuny@gmail.com>, 2014 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +import pprint + +USER_AGENT_PRODUCT="Ansible-gce" +USER_AGENT_VERSION="v1" + +def gce_connect(module, provider=None): + """Return a Google Cloud Engine connection.""" + service_account_email = module.params.get('service_account_email', None) + pem_file = module.params.get('pem_file', None) + project_id = module.params.get('project_id', None) + + # If any of the values are not given as parameters, check the appropriate + # environment variables. + if not service_account_email: + service_account_email = os.environ.get('GCE_EMAIL', None) + if not project_id: + project_id = os.environ.get('GCE_PROJECT', None) + if not pem_file: + pem_file = os.environ.get('GCE_PEM_FILE_PATH', None) + + # If we still don't have one or more of our credentials, attempt to + # get the remaining values from the libcloud secrets file. + if service_account_email is None or pem_file is None: + try: + import secrets + except ImportError: + secrets = None + + if hasattr(secrets, 'GCE_PARAMS'): + if not service_account_email: + service_account_email = secrets.GCE_PARAMS[0] + if not pem_file: + pem_file = secrets.GCE_PARAMS[1] + keyword_params = getattr(secrets, 'GCE_KEYWORD_PARAMS', {}) + if not project_id: + project_id = keyword_params.get('project', None) + + # If we *still* don't have the credentials we need, then it's time to + # just fail out. + if service_account_email is None or pem_file is None or project_id is None: + module.fail_json(msg='Missing GCE connection parameters in libcloud ' + 'secrets file.') + return None + + # Allow for passing in libcloud Google DNS (e.g, Provider.GOOGLE) + if provider is None: + provider = Provider.GCE + + try: + gce = get_driver(provider)(service_account_email, pem_file, + datacenter=module.params.get('zone', None), + project=project_id) + gce.connection.user_agent_append("%s/%s" % ( + USER_AGENT_PRODUCT, USER_AGENT_VERSION)) + except (RuntimeError, ValueError), e: + module.fail_json(msg=str(e), changed=False) + except Exception, e: + module.fail_json(msg=unexpected_error_msg(e), changed=False) + + return gce + +def unexpected_error_msg(error): + """Create an error string based on passed in error.""" + return 'Unexpected response: ' + pprint.pformat(vars(error)) diff --git a/v2/ansible/module_utils/known_hosts.py b/v2/ansible/module_utils/known_hosts.py new file mode 100644 index 0000000000..99dbf2c03a --- /dev/null +++ b/v2/ansible/module_utils/known_hosts.py @@ -0,0 +1,176 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import hmac +import urlparse + +try: + from hashlib import sha1 +except ImportError: + import sha as sha1 + +HASHED_KEY_MAGIC = "|1|" + +def add_git_host_key(module, url, accept_hostkey=True, create_dir=True): + + """ idempotently add a git url hostkey """ + + fqdn = get_fqdn(url) + + if fqdn: + known_host = check_hostkey(module, fqdn) + if not known_host: + if accept_hostkey: + rc, out, err = add_host_key(module, fqdn, create_dir=create_dir) + if rc != 0: + module.fail_json(msg="failed to add %s hostkey: %s" % (fqdn, out + err)) + else: + module.fail_json(msg="%s has an unknown hostkey. Set accept_hostkey to True or manually add the hostkey prior to running the git module" % fqdn) + +def get_fqdn(repo_url): + + """ chop the hostname out of a giturl """ + + result = None + if "@" in repo_url and "://" not in repo_url: + # most likely a git@ or ssh+git@ type URL + repo_url = repo_url.split("@", 1)[1] + if ":" in repo_url: + repo_url = repo_url.split(":")[0] + result = repo_url + elif "/" in repo_url: + repo_url = repo_url.split("/")[0] + result = repo_url + elif "://" in repo_url: + # this should be something we can parse with urlparse + parts = urlparse.urlparse(repo_url) + if 'ssh' not in parts[0] and 'git' not in parts[0]: + # don't try and scan a hostname that's not ssh + return None + # parts[1] will be empty on python2.4 on ssh:// or git:// urls, so + # ensure we actually have a parts[1] before continuing. + if parts[1] != '': + result = parts[1] + if ":" in result: + result = result.split(":")[0] + if "@" in result: + result = result.split("@", 1)[1] + + return result + +def check_hostkey(module, fqdn): + return not not_in_host_file(module, fqdn) + +# this is a variant of code found in connection_plugins/paramiko.py and we should modify +# the paramiko code to import and use this. + +def not_in_host_file(self, host): + + + if 'USER' in os.environ: + user_host_file = os.path.expandvars("~${USER}/.ssh/known_hosts") + else: + user_host_file = "~/.ssh/known_hosts" + user_host_file = os.path.expanduser(user_host_file) + + host_file_list = [] + host_file_list.append(user_host_file) + host_file_list.append("/etc/ssh/ssh_known_hosts") + host_file_list.append("/etc/ssh/ssh_known_hosts2") + + hfiles_not_found = 0 + for hf in host_file_list: + if not os.path.exists(hf): + hfiles_not_found += 1 + continue + + try: + host_fh = open(hf) + except IOError, e: + hfiles_not_found += 1 + continue + else: + data = host_fh.read() + host_fh.close() + + for line in data.split("\n"): + if line is None or " " not in line: + continue + tokens = line.split() + if tokens[0].find(HASHED_KEY_MAGIC) == 0: + # this is a hashed known host entry + try: + (kn_salt,kn_host) = tokens[0][len(HASHED_KEY_MAGIC):].split("|",2) + hash = hmac.new(kn_salt.decode('base64'), digestmod=sha1) + hash.update(host) + if hash.digest() == kn_host.decode('base64'): + return False + except: + # invalid hashed host key, skip it + continue + else: + # standard host file entry + if host in tokens[0]: + return False + + return True + + +def add_host_key(module, fqdn, key_type="rsa", create_dir=False): + + """ use ssh-keyscan to add the hostkey """ + + result = False + keyscan_cmd = module.get_bin_path('ssh-keyscan', True) + + if 'USER' in os.environ: + user_ssh_dir = os.path.expandvars("~${USER}/.ssh/") + user_host_file = os.path.expandvars("~${USER}/.ssh/known_hosts") + else: + user_ssh_dir = "~/.ssh/" + user_host_file = "~/.ssh/known_hosts" + user_ssh_dir = os.path.expanduser(user_ssh_dir) + + if not os.path.exists(user_ssh_dir): + if create_dir: + try: + os.makedirs(user_ssh_dir, 0700) + except: + module.fail_json(msg="failed to create host key directory: %s" % user_ssh_dir) + else: + module.fail_json(msg="%s does not exist" % user_ssh_dir) + elif not os.path.isdir(user_ssh_dir): + module.fail_json(msg="%s is not a directory" % user_ssh_dir) + + this_cmd = "%s -t %s %s" % (keyscan_cmd, key_type, fqdn) + + rc, out, err = module.run_command(this_cmd) + module.append_to_file(user_host_file, out) + + return rc, out, err + diff --git a/v2/ansible/module_utils/openstack.py b/v2/ansible/module_utils/openstack.py new file mode 100644 index 0000000000..64f9543714 --- /dev/null +++ b/v2/ansible/module_utils/openstack.py @@ -0,0 +1,69 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + + +def openstack_argument_spec(): + # Consume standard OpenStack environment variables. + # This is mainly only useful for ad-hoc command line operation as + # in playbooks one would assume variables would be used appropriately + OS_AUTH_URL=os.environ.get('OS_AUTH_URL', 'http://127.0.0.1:35357/v2.0/') + OS_PASSWORD=os.environ.get('OS_PASSWORD', None) + OS_REGION_NAME=os.environ.get('OS_REGION_NAME', None) + OS_USERNAME=os.environ.get('OS_USERNAME', 'admin') + OS_TENANT_NAME=os.environ.get('OS_TENANT_NAME', OS_USERNAME) + + spec = dict( + login_username = dict(default=OS_USERNAME), + auth_url = dict(default=OS_AUTH_URL), + region_name = dict(default=OS_REGION_NAME), + availability_zone = dict(default=None), + ) + if OS_PASSWORD: + spec['login_password'] = dict(default=OS_PASSWORD) + else: + spec['login_password'] = dict(required=True) + if OS_TENANT_NAME: + spec['login_tenant_name'] = dict(default=OS_TENANT_NAME) + else: + spec['login_tenant_name'] = dict(required=True) + return spec + +def openstack_find_nova_addresses(addresses, ext_tag, key_name=None): + + ret = [] + for (k, v) in addresses.iteritems(): + if key_name and k == key_name: + ret.extend([addrs['addr'] for addrs in v]) + else: + for interface_spec in v: + if 'OS-EXT-IPS:type' in interface_spec and interface_spec['OS-EXT-IPS:type'] == ext_tag: + ret.append(interface_spec['addr']) + return ret + diff --git a/v2/ansible/module_utils/powershell.ps1 b/v2/ansible/module_utils/powershell.ps1 new file mode 100644 index 0000000000..c097c69768 --- /dev/null +++ b/v2/ansible/module_utils/powershell.ps1 @@ -0,0 +1,144 @@ +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2014, and others +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +# Helper function to parse Ansible JSON arguments from a file passed as +# the single argument to the module +# Example: $params = Parse-Args $args +Function Parse-Args($arguments) +{ + $parameters = New-Object psobject; + If ($arguments.Length -gt 0) + { + $parameters = Get-Content $arguments[0] | ConvertFrom-Json; + } + $parameters; +} + +# Helper function to set an "attribute" on a psobject instance in powershell. +# This is a convenience to make adding Members to the object easier and +# slightly more pythonic +# Example: Set-Attr $result "changed" $true +Function Set-Attr($obj, $name, $value) +{ + # If the provided $obj is undefined, define one to be nice + If (-not $obj.GetType) + { + $obj = New-Object psobject + } + + $obj | Add-Member -Force -MemberType NoteProperty -Name $name -Value $value +} + +# Helper function to convert a powershell object to JSON to echo it, exiting +# the script +# Example: Exit-Json $result +Function Exit-Json($obj) +{ + # If the provided $obj is undefined, define one to be nice + If (-not $obj.GetType) + { + $obj = New-Object psobject + } + + echo $obj | ConvertTo-Json -Depth 99 + Exit +} + +# Helper function to add the "msg" property and "failed" property, convert the +# powershell object to JSON and echo it, exiting the script +# Example: Fail-Json $result "This is the failure message" +Function Fail-Json($obj, $message = $null) +{ + # If we weren't given 2 args, and the only arg was a string, create a new + # psobject and use the arg as the failure message + If ($message -eq $null -and $obj.GetType().Name -eq "String") + { + $message = $obj + $obj = New-Object psobject + } + # If the first args is undefined or not an object, make it an object + ElseIf (-not $obj.GetType -or $obj.GetType().Name -ne "PSCustomObject") + { + $obj = New-Object psobject + } + + Set-Attr $obj "msg" $message + Set-Attr $obj "failed" $true + echo $obj | ConvertTo-Json -Depth 99 + Exit 1 +} + +# Helper function to get an "attribute" from a psobject instance in powershell. +# This is a convenience to make getting Members from an object easier and +# slightly more pythonic +# Example: $attr = Get-Attr $response "code" -default "1" +#Note that if you use the failifempty option, you do need to specify resultobject as well. +Function Get-Attr($obj, $name, $default = $null,$resultobj, $failifempty=$false, $emptyattributefailmessage) +{ + # Check if the provided Member $name exists in $obj and return it or the + # default + If ($obj.$name.GetType) + { + $obj.$name + } + Elseif($failifempty -eq $false) + { + $default + } + else + { + if (!$emptyattributefailmessage) {$emptyattributefailmessage = "Missing required argument: $name"} + Fail-Json -obj $resultobj -message $emptyattributefailmessage + } + return +} + +# Helper filter/pipeline function to convert a value to boolean following current +# Ansible practices +# Example: $is_true = "true" | ConvertTo-Bool +Function ConvertTo-Bool +{ + param( + [parameter(valuefrompipeline=$true)] + $obj + ) + + $boolean_strings = "yes", "on", "1", "true", 1 + $obj_string = [string]$obj + + if (($obj.GetType().Name -eq "Boolean" -and $obj) -or $boolean_strings -contains $obj_string.ToLower()) + { + $true + } + Else + { + $false + } + return +} + diff --git a/v2/ansible/module_utils/rax.py b/v2/ansible/module_utils/rax.py new file mode 100644 index 0000000000..75363b1aac --- /dev/null +++ b/v2/ansible/module_utils/rax.py @@ -0,0 +1,277 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by +# Ansible still belong to the author of the module, and may assign their own +# license to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from uuid import UUID + + +FINAL_STATUSES = ('ACTIVE', 'ERROR') +VOLUME_STATUS = ('available', 'attaching', 'creating', 'deleting', 'in-use', + 'error', 'error_deleting') + +CLB_ALGORITHMS = ['RANDOM', 'LEAST_CONNECTIONS', 'ROUND_ROBIN', + 'WEIGHTED_LEAST_CONNECTIONS', 'WEIGHTED_ROUND_ROBIN'] +CLB_PROTOCOLS = ['DNS_TCP', 'DNS_UDP', 'FTP', 'HTTP', 'HTTPS', 'IMAPS', + 'IMAPv4', 'LDAP', 'LDAPS', 'MYSQL', 'POP3', 'POP3S', 'SMTP', + 'TCP', 'TCP_CLIENT_FIRST', 'UDP', 'UDP_STREAM', 'SFTP'] + +NON_CALLABLES = (basestring, bool, dict, int, list, type(None)) +PUBLIC_NET_ID = "00000000-0000-0000-0000-000000000000" +SERVICE_NET_ID = "11111111-1111-1111-1111-111111111111" + + +def rax_slugify(value): + """Prepend a key with rax_ and normalize the key name""" + return 'rax_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_')) + + +def rax_clb_node_to_dict(obj): + """Function to convert a CLB Node object to a dict""" + if not obj: + return {} + node = obj.to_dict() + node['id'] = obj.id + node['weight'] = obj.weight + return node + + +def rax_to_dict(obj, obj_type='standard'): + """Generic function to convert a pyrax object to a dict + + obj_type values: + standard + clb + server + + """ + instance = {} + for key in dir(obj): + value = getattr(obj, key) + if obj_type == 'clb' and key == 'nodes': + instance[key] = [] + for node in value: + instance[key].append(rax_clb_node_to_dict(node)) + elif (isinstance(value, list) and len(value) > 0 and + not isinstance(value[0], NON_CALLABLES)): + instance[key] = [] + for item in value: + instance[key].append(rax_to_dict(item)) + elif (isinstance(value, NON_CALLABLES) and not key.startswith('_')): + if obj_type == 'server': + key = rax_slugify(key) + instance[key] = value + + if obj_type == 'server': + for attr in ['id', 'accessIPv4', 'name', 'status']: + instance[attr] = instance.get(rax_slugify(attr)) + + return instance + + +def rax_find_image(module, rax_module, image): + cs = rax_module.cloudservers + try: + UUID(image) + except ValueError: + try: + image = cs.images.find(human_id=image) + except(cs.exceptions.NotFound, + cs.exceptions.NoUniqueMatch): + try: + image = cs.images.find(name=image) + except (cs.exceptions.NotFound, + cs.exceptions.NoUniqueMatch): + module.fail_json(msg='No matching image found (%s)' % + image) + + return rax_module.utils.get_id(image) + + +def rax_find_volume(module, rax_module, name): + cbs = rax_module.cloud_blockstorage + try: + UUID(name) + volume = cbs.get(name) + except ValueError: + try: + volume = cbs.find(name=name) + except rax_module.exc.NotFound: + volume = None + except Exception, e: + module.fail_json(msg='%s' % e) + return volume + + +def rax_find_network(module, rax_module, network): + cnw = rax_module.cloud_networks + try: + UUID(network) + except ValueError: + if network.lower() == 'public': + return cnw.get_server_networks(PUBLIC_NET_ID) + elif network.lower() == 'private': + return cnw.get_server_networks(SERVICE_NET_ID) + else: + try: + network_obj = cnw.find_network_by_label(network) + except (rax_module.exceptions.NetworkNotFound, + rax_module.exceptions.NetworkLabelNotUnique): + module.fail_json(msg='No matching network found (%s)' % + network) + else: + return cnw.get_server_networks(network_obj) + else: + return cnw.get_server_networks(network) + + +def rax_find_server(module, rax_module, server): + cs = rax_module.cloudservers + try: + UUID(server) + server = cs.servers.get(server) + except ValueError: + servers = cs.servers.list(search_opts=dict(name='^%s$' % server)) + if not servers: + module.fail_json(msg='No Server was matched by name, ' + 'try using the Server ID instead') + if len(servers) > 1: + module.fail_json(msg='Multiple servers matched by name, ' + 'try using the Server ID instead') + + # We made it this far, grab the first and hopefully only server + # in the list + server = servers[0] + return server + + +def rax_find_loadbalancer(module, rax_module, loadbalancer): + clb = rax_module.cloud_loadbalancers + try: + found = clb.get(loadbalancer) + except: + found = [] + for lb in clb.list(): + if loadbalancer == lb.name: + found.append(lb) + + if not found: + module.fail_json(msg='No loadbalancer was matched') + + if len(found) > 1: + module.fail_json(msg='Multiple loadbalancers matched') + + # We made it this far, grab the first and hopefully only item + # in the list + found = found[0] + + return found + + +def rax_argument_spec(): + return dict( + api_key=dict(type='str', aliases=['password'], no_log=True), + auth_endpoint=dict(type='str'), + credentials=dict(type='str', aliases=['creds_file']), + env=dict(type='str'), + identity_type=dict(type='str', default='rackspace'), + region=dict(type='str'), + tenant_id=dict(type='str'), + tenant_name=dict(type='str'), + username=dict(type='str'), + verify_ssl=dict(choices=BOOLEANS, type='bool'), + ) + + +def rax_required_together(): + return [['api_key', 'username']] + + +def setup_rax_module(module, rax_module, region_required=True): + rax_module.USER_AGENT = 'ansible/%s %s' % (ANSIBLE_VERSION, + rax_module.USER_AGENT) + + api_key = module.params.get('api_key') + auth_endpoint = module.params.get('auth_endpoint') + credentials = module.params.get('credentials') + env = module.params.get('env') + identity_type = module.params.get('identity_type') + region = module.params.get('region') + tenant_id = module.params.get('tenant_id') + tenant_name = module.params.get('tenant_name') + username = module.params.get('username') + verify_ssl = module.params.get('verify_ssl') + + if env is not None: + rax_module.set_environment(env) + + rax_module.set_setting('identity_type', identity_type) + if verify_ssl is not None: + rax_module.set_setting('verify_ssl', verify_ssl) + if auth_endpoint is not None: + rax_module.set_setting('auth_endpoint', auth_endpoint) + if tenant_id is not None: + rax_module.set_setting('tenant_id', tenant_id) + if tenant_name is not None: + rax_module.set_setting('tenant_name', tenant_name) + + try: + username = username or os.environ.get('RAX_USERNAME') + if not username: + username = rax_module.get_setting('keyring_username') + if username: + api_key = 'USE_KEYRING' + if not api_key: + api_key = os.environ.get('RAX_API_KEY') + credentials = (credentials or os.environ.get('RAX_CREDENTIALS') or + os.environ.get('RAX_CREDS_FILE')) + region = (region or os.environ.get('RAX_REGION') or + rax_module.get_setting('region')) + except KeyError, e: + module.fail_json(msg='Unable to load %s' % e.message) + + try: + if api_key and username: + if api_key == 'USE_KEYRING': + rax_module.keyring_auth(username, region=region) + else: + rax_module.set_credentials(username, api_key=api_key, + region=region) + elif credentials: + credentials = os.path.expanduser(credentials) + rax_module.set_credential_file(credentials, region=region) + else: + raise Exception('No credentials supplied!') + except Exception, e: + module.fail_json(msg='%s' % e.message) + + if region_required and region not in rax_module.regions: + module.fail_json(msg='%s is not a valid region, must be one of: %s' % + (region, ','.join(rax_module.regions))) + + return rax_module diff --git a/v2/ansible/module_utils/redhat.py b/v2/ansible/module_utils/redhat.py new file mode 100644 index 0000000000..bf19ccf390 --- /dev/null +++ b/v2/ansible/module_utils/redhat.py @@ -0,0 +1,280 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), James Laska +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import re +import types +import ConfigParser +import shlex + + +class RegistrationBase(object): + def __init__(self, module, username=None, password=None): + self.module = module + self.username = username + self.password = password + + def configure(self): + raise NotImplementedError("Must be implemented by a sub-class") + + def enable(self): + # Remove any existing redhat.repo + redhat_repo = '/etc/yum.repos.d/redhat.repo' + if os.path.isfile(redhat_repo): + os.unlink(redhat_repo) + + def register(self): + raise NotImplementedError("Must be implemented by a sub-class") + + def unregister(self): + raise NotImplementedError("Must be implemented by a sub-class") + + def unsubscribe(self): + raise NotImplementedError("Must be implemented by a sub-class") + + def update_plugin_conf(self, plugin, enabled=True): + plugin_conf = '/etc/yum/pluginconf.d/%s.conf' % plugin + if os.path.isfile(plugin_conf): + cfg = ConfigParser.ConfigParser() + cfg.read([plugin_conf]) + if enabled: + cfg.set('main', 'enabled', 1) + else: + cfg.set('main', 'enabled', 0) + fd = open(plugin_conf, 'rwa+') + cfg.write(fd) + fd.close() + + def subscribe(self, **kwargs): + raise NotImplementedError("Must be implemented by a sub-class") + + +class Rhsm(RegistrationBase): + def __init__(self, module, username=None, password=None): + RegistrationBase.__init__(self, module, username, password) + self.config = self._read_config() + self.module = module + + def _read_config(self, rhsm_conf='/etc/rhsm/rhsm.conf'): + ''' + Load RHSM configuration from /etc/rhsm/rhsm.conf. + Returns: + * ConfigParser object + ''' + + # Read RHSM defaults ... + cp = ConfigParser.ConfigParser() + cp.read(rhsm_conf) + + # Add support for specifying a default value w/o having to standup some configuration + # Yeah, I know this should be subclassed ... but, oh well + def get_option_default(self, key, default=''): + sect, opt = key.split('.', 1) + if self.has_section(sect) and self.has_option(sect, opt): + return self.get(sect, opt) + else: + return default + + cp.get_option = types.MethodType(get_option_default, cp, ConfigParser.ConfigParser) + + return cp + + def enable(self): + ''' + Enable the system to receive updates from subscription-manager. + This involves updating affected yum plugins and removing any + conflicting yum repositories. + ''' + RegistrationBase.enable(self) + self.update_plugin_conf('rhnplugin', False) + self.update_plugin_conf('subscription-manager', True) + + def configure(self, **kwargs): + ''' + Configure the system as directed for registration with RHN + Raises: + * Exception - if error occurs while running command + ''' + args = ['subscription-manager', 'config'] + + # Pass supplied **kwargs as parameters to subscription-manager. Ignore + # non-configuration parameters and replace '_' with '.'. For example, + # 'server_hostname' becomes '--system.hostname'. + for k,v in kwargs.items(): + if re.search(r'^(system|rhsm)_', k): + args.append('--%s=%s' % (k.replace('_','.'), v)) + + self.module.run_command(args, check_rc=True) + + @property + def is_registered(self): + ''' + Determine whether the current system + Returns: + * Boolean - whether the current system is currently registered to + RHN. + ''' + # Quick version... + if False: + return os.path.isfile('/etc/pki/consumer/cert.pem') and \ + os.path.isfile('/etc/pki/consumer/key.pem') + + args = ['subscription-manager', 'identity'] + rc, stdout, stderr = self.module.run_command(args, check_rc=False) + if rc == 0: + return True + else: + return False + + def register(self, username, password, autosubscribe, activationkey): + ''' + Register the current system to the provided RHN server + Raises: + * Exception - if error occurs while running command + ''' + args = ['subscription-manager', 'register'] + + # Generate command arguments + if activationkey: + args.append('--activationkey "%s"' % activationkey) + else: + if autosubscribe: + args.append('--autosubscribe') + if username: + args.extend(['--username', username]) + if password: + args.extend(['--password', password]) + + # Do the needful... + rc, stderr, stdout = self.module.run_command(args, check_rc=True) + + def unsubscribe(self): + ''' + Unsubscribe a system from all subscribed channels + Raises: + * Exception - if error occurs while running command + ''' + args = ['subscription-manager', 'unsubscribe', '--all'] + rc, stderr, stdout = self.module.run_command(args, check_rc=True) + + def unregister(self): + ''' + Unregister a currently registered system + Raises: + * Exception - if error occurs while running command + ''' + args = ['subscription-manager', 'unregister'] + rc, stderr, stdout = self.module.run_command(args, check_rc=True) + + def subscribe(self, regexp): + ''' + Subscribe current system to available pools matching the specified + regular expression + Raises: + * Exception - if error occurs while running command + ''' + + # Available pools ready for subscription + available_pools = RhsmPools(self.module) + + for pool in available_pools.filter(regexp): + pool.subscribe() + + +class RhsmPool(object): + ''' + Convenience class for housing subscription information + ''' + + def __init__(self, module, **kwargs): + self.module = module + for k,v in kwargs.items(): + setattr(self, k, v) + + def __str__(self): + return str(self.__getattribute__('_name')) + + def subscribe(self): + args = "subscription-manager subscribe --pool %s" % self.PoolId + rc, stdout, stderr = self.module.run_command(args, check_rc=True) + if rc == 0: + return True + else: + return False + + +class RhsmPools(object): + """ + This class is used for manipulating pools subscriptions with RHSM + """ + def __init__(self, module): + self.module = module + self.products = self._load_product_list() + + def __iter__(self): + return self.products.__iter__() + + def _load_product_list(self): + """ + Loads list of all available pools for system in data structure + """ + args = "subscription-manager list --available" + rc, stdout, stderr = self.module.run_command(args, check_rc=True) + + products = [] + for line in stdout.split('\n'): + # Remove leading+trailing whitespace + line = line.strip() + # An empty line implies the end of an output group + if len(line) == 0: + continue + # If a colon ':' is found, parse + elif ':' in line: + (key, value) = line.split(':',1) + key = key.strip().replace(" ", "") # To unify + value = value.strip() + if key in ['ProductName', 'SubscriptionName']: + # Remember the name for later processing + products.append(RhsmPool(self.module, _name=value, key=value)) + elif products: + # Associate value with most recently recorded product + products[-1].__setattr__(key, value) + # FIXME - log some warning? + #else: + # warnings.warn("Unhandled subscription key/value: %s/%s" % (key,value)) + return products + + def filter(self, regexp='^$'): + ''' + Return a list of RhsmPools whose name matches the provided regular expression + ''' + r = re.compile(regexp) + for product in self.products: + if r.search(product._name): + yield product + diff --git a/v2/ansible/module_utils/splitter.py b/v2/ansible/module_utils/splitter.py new file mode 100644 index 0000000000..899fa8cd92 --- /dev/null +++ b/v2/ansible/module_utils/splitter.py @@ -0,0 +1,201 @@ +# (c) 2014 James Cammarata, <jcammarata@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +def _get_quote_state(token, quote_char): + ''' + the goal of this block is to determine if the quoted string + is unterminated in which case it needs to be put back together + ''' + # the char before the current one, used to see if + # the current character is escaped + prev_char = None + for idx, cur_char in enumerate(token): + if idx > 0: + prev_char = token[idx-1] + if cur_char in '"\'' and prev_char != '\\': + if quote_char: + if cur_char == quote_char: + quote_char = None + else: + quote_char = cur_char + return quote_char + +def _count_jinja2_blocks(token, cur_depth, open_token, close_token): + ''' + this function counts the number of opening/closing blocks for a + given opening/closing type and adjusts the current depth for that + block based on the difference + ''' + num_open = token.count(open_token) + num_close = token.count(close_token) + if num_open != num_close: + cur_depth += (num_open - num_close) + if cur_depth < 0: + cur_depth = 0 + return cur_depth + +def split_args(args): + ''' + Splits args on whitespace, but intelligently reassembles + those that may have been split over a jinja2 block or quotes. + + When used in a remote module, we won't ever have to be concerned about + jinja2 blocks, however this function is/will be used in the + core portions as well before the args are templated. + + example input: a=b c="foo bar" + example output: ['a=b', 'c="foo bar"'] + + Basically this is a variation shlex that has some more intelligence for + how Ansible needs to use it. + ''' + + # the list of params parsed out of the arg string + # this is going to be the result value when we are donei + params = [] + + # here we encode the args, so we have a uniform charset to + # work with, and split on white space + args = args.strip() + try: + args = args.encode('utf-8') + do_decode = True + except UnicodeDecodeError: + do_decode = False + items = args.split('\n') + + # iterate over the tokens, and reassemble any that may have been + # split on a space inside a jinja2 block. + # ex if tokens are "{{", "foo", "}}" these go together + + # These variables are used + # to keep track of the state of the parsing, since blocks and quotes + # may be nested within each other. + + quote_char = None + inside_quotes = False + print_depth = 0 # used to count nested jinja2 {{ }} blocks + block_depth = 0 # used to count nested jinja2 {% %} blocks + comment_depth = 0 # used to count nested jinja2 {# #} blocks + + # now we loop over each split chunk, coalescing tokens if the white space + # split occurred within quotes or a jinja2 block of some kind + for itemidx,item in enumerate(items): + + # we split on spaces and newlines separately, so that we + # can tell which character we split on for reassembly + # inside quotation characters + tokens = item.strip().split(' ') + + line_continuation = False + for idx,token in enumerate(tokens): + + # if we hit a line continuation character, but + # we're not inside quotes, ignore it and continue + # on to the next token while setting a flag + if token == '\\' and not inside_quotes: + line_continuation = True + continue + + # store the previous quoting state for checking later + was_inside_quotes = inside_quotes + quote_char = _get_quote_state(token, quote_char) + inside_quotes = quote_char is not None + + # multiple conditions may append a token to the list of params, + # so we keep track with this flag to make sure it only happens once + # append means add to the end of the list, don't append means concatenate + # it to the end of the last token + appended = False + + # if we're inside quotes now, but weren't before, append the token + # to the end of the list, since we'll tack on more to it later + # otherwise, if we're inside any jinja2 block, inside quotes, or we were + # inside quotes (but aren't now) concat this token to the last param + if inside_quotes and not was_inside_quotes: + params.append(token) + appended = True + elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes: + if idx == 0 and not inside_quotes and was_inside_quotes: + params[-1] = "%s%s" % (params[-1], token) + elif len(tokens) > 1: + spacer = '' + if idx > 0: + spacer = ' ' + params[-1] = "%s%s%s" % (params[-1], spacer, token) + else: + spacer = '' + if not params[-1].endswith('\n') and idx == 0: + spacer = '\n' + params[-1] = "%s%s%s" % (params[-1], spacer, token) + appended = True + + # if the number of paired block tags is not the same, the depth has changed, so we calculate that here + # and may append the current token to the params (if we haven't previously done so) + prev_print_depth = print_depth + print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}") + if print_depth != prev_print_depth and not appended: + params.append(token) + appended = True + + prev_block_depth = block_depth + block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}") + if block_depth != prev_block_depth and not appended: + params.append(token) + appended = True + + prev_comment_depth = comment_depth + comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}") + if comment_depth != prev_comment_depth and not appended: + params.append(token) + appended = True + + # finally, if we're at zero depth for all blocks and not inside quotes, and have not + # yet appended anything to the list of params, we do so now + if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '': + params.append(token) + + # if this was the last token in the list, and we have more than + # one item (meaning we split on newlines), add a newline back here + # to preserve the original structure + if len(items) > 1 and itemidx != len(items) - 1 and not line_continuation: + if not params[-1].endswith('\n') or item == '': + params[-1] += '\n' + + # always clear the line continuation flag + line_continuation = False + + # If we're done and things are not at zero depth or we're still inside quotes, + # raise an error to indicate that the args were unbalanced + if print_depth or block_depth or comment_depth or inside_quotes: + raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes") + + # finally, we decode each param back to the unicode it was in the arg string + if do_decode: + params = [x.decode('utf-8') for x in params] + + return params + +def is_quoted(data): + return len(data) > 0 and (data[0] == '"' and data[-1] == '"' or data[0] == "'" and data[-1] == "'") + +def unquote(data): + ''' removes first and last quotes from a string, if the string starts and ends with the same quotes ''' + if is_quoted(data): + return data[1:-1] + return data + diff --git a/v2/ansible/module_utils/urls.py b/v2/ansible/module_utils/urls.py new file mode 100644 index 0000000000..962b868ee0 --- /dev/null +++ b/v2/ansible/module_utils/urls.py @@ -0,0 +1,480 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan <michael.dehaan@gmail.com>, 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +try: + import urllib + HAS_URLLIB = True +except: + HAS_URLLIB = False + +try: + import urllib2 + HAS_URLLIB2 = True +except: + HAS_URLLIB2 = False + +try: + import urlparse + HAS_URLPARSE = True +except: + HAS_URLPARSE = False + +try: + import ssl + HAS_SSL=True +except: + HAS_SSL=False + +import httplib +import os +import re +import socket +import tempfile + + +# This is a dummy cacert provided for Mac OS since you need at least 1 +# ca cert, regardless of validity, for Python on Mac OS to use the +# keychain functionality in OpenSSL for validating SSL certificates. +# See: http://mercurial.selenic.com/wiki/CACertificates#Mac_OS_X_10.6_and_higher +DUMMY_CA_CERT = """-----BEGIN CERTIFICATE----- +MIICvDCCAiWgAwIBAgIJAO8E12S7/qEpMA0GCSqGSIb3DQEBBQUAMEkxCzAJBgNV +BAYTAlVTMRcwFQYDVQQIEw5Ob3J0aCBDYXJvbGluYTEPMA0GA1UEBxMGRHVyaGFt +MRAwDgYDVQQKEwdBbnNpYmxlMB4XDTE0MDMxODIyMDAyMloXDTI0MDMxNTIyMDAy +MlowSTELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMQ8wDQYD +VQQHEwZEdXJoYW0xEDAOBgNVBAoTB0Fuc2libGUwgZ8wDQYJKoZIhvcNAQEBBQAD +gY0AMIGJAoGBANtvpPq3IlNlRbCHhZAcP6WCzhc5RbsDqyh1zrkmLi0GwcQ3z/r9 +gaWfQBYhHpobK2Tiq11TfraHeNB3/VfNImjZcGpN8Fl3MWwu7LfVkJy3gNNnxkA1 +4Go0/LmIvRFHhbzgfuo9NFgjPmmab9eqXJceqZIlz2C8xA7EeG7ku0+vAgMBAAGj +gaswgagwHQYDVR0OBBYEFPnN1nPRqNDXGlCqCvdZchRNi/FaMHkGA1UdIwRyMHCA +FPnN1nPRqNDXGlCqCvdZchRNi/FaoU2kSzBJMQswCQYDVQQGEwJVUzEXMBUGA1UE +CBMOTm9ydGggQ2Fyb2xpbmExDzANBgNVBAcTBkR1cmhhbTEQMA4GA1UEChMHQW5z +aWJsZYIJAO8E12S7/qEpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEA +MUB80IR6knq9K/tY+hvPsZer6eFMzO3JGkRFBh2kn6JdMDnhYGX7AXVHGflrwNQH +qFy+aenWXsC0ZvrikFxbQnX8GVtDADtVznxOi7XzFw7JOxdsVrpXgSN0eh0aMzvV +zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg= +-----END CERTIFICATE----- +""" + +class CustomHTTPSConnection(httplib.HTTPSConnection): + def connect(self): + "Connect to a host on a given (SSL) port." + + if hasattr(self, 'source_address'): + sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address) + else: + sock = socket.create_connection((self.host, self.port), self.timeout) + if self._tunnel_host: + self.sock = sock + self._tunnel() + self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1) + +class CustomHTTPSHandler(urllib2.HTTPSHandler): + + def https_open(self, req): + return self.do_open(CustomHTTPSConnection, req) + + https_request = urllib2.AbstractHTTPHandler.do_request_ + +def generic_urlparse(parts): + ''' + Returns a dictionary of url parts as parsed by urlparse, + but accounts for the fact that older versions of that + library do not support named attributes (ie. .netloc) + ''' + generic_parts = dict() + if hasattr(parts, 'netloc'): + # urlparse is newer, just read the fields straight + # from the parts object + generic_parts['scheme'] = parts.scheme + generic_parts['netloc'] = parts.netloc + generic_parts['path'] = parts.path + generic_parts['params'] = parts.params + generic_parts['query'] = parts.query + generic_parts['fragment'] = parts.fragment + generic_parts['username'] = parts.username + generic_parts['password'] = parts.password + generic_parts['hostname'] = parts.hostname + generic_parts['port'] = parts.port + else: + # we have to use indexes, and then parse out + # the other parts not supported by indexing + generic_parts['scheme'] = parts[0] + generic_parts['netloc'] = parts[1] + generic_parts['path'] = parts[2] + generic_parts['params'] = parts[3] + generic_parts['query'] = parts[4] + generic_parts['fragment'] = parts[5] + # get the username, password, etc. + try: + netloc_re = re.compile(r'^((?:\w)+(?::(?:\w)+)?@)?([A-Za-z0-9.-]+)(:\d+)?$') + (auth, hostname, port) = netloc_re.match(parts[1]) + if port: + # the capture group for the port will include the ':', + # so remove it and convert the port to an integer + port = int(port[1:]) + if auth: + # the capture group above inclues the @, so remove it + # and then split it up based on the first ':' found + auth = auth[:-1] + username, password = auth.split(':', 1) + generic_parts['username'] = username + generic_parts['password'] = password + generic_parts['hostname'] = hostnme + generic_parts['port'] = port + except: + generic_parts['username'] = None + generic_parts['password'] = None + generic_parts['hostname'] = None + generic_parts['port'] = None + return generic_parts + +class RequestWithMethod(urllib2.Request): + ''' + Workaround for using DELETE/PUT/etc with urllib2 + Originally contained in library/net_infrastructure/dnsmadeeasy + ''' + + def __init__(self, url, method, data=None, headers={}): + self._method = method + urllib2.Request.__init__(self, url, data, headers) + + def get_method(self): + if self._method: + return self._method + else: + return urllib2.Request.get_method(self) + + +class SSLValidationHandler(urllib2.BaseHandler): + ''' + A custom handler class for SSL validation. + + Based on: + http://stackoverflow.com/questions/1087227/validate-ssl-certificates-with-python + http://techknack.net/python-urllib2-handlers/ + ''' + CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n" + + def __init__(self, module, hostname, port): + self.module = module + self.hostname = hostname + self.port = port + + def get_ca_certs(self): + # tries to find a valid CA cert in one of the + # standard locations for the current distribution + + ca_certs = [] + paths_checked = [] + platform = get_platform() + distribution = get_distribution() + + # build a list of paths to check for .crt/.pem files + # based on the platform type + paths_checked.append('/etc/ssl/certs') + if platform == 'Linux': + paths_checked.append('/etc/pki/ca-trust/extracted/pem') + paths_checked.append('/etc/pki/tls/certs') + paths_checked.append('/usr/share/ca-certificates/cacert.org') + elif platform == 'FreeBSD': + paths_checked.append('/usr/local/share/certs') + elif platform == 'OpenBSD': + paths_checked.append('/etc/ssl') + elif platform == 'NetBSD': + ca_certs.append('/etc/openssl/certs') + elif platform == 'SunOS': + paths_checked.append('/opt/local/etc/openssl/certs') + + # fall back to a user-deployed cert in a standard + # location if the OS platform one is not available + paths_checked.append('/etc/ansible') + + tmp_fd, tmp_path = tempfile.mkstemp() + + # Write the dummy ca cert if we are running on Mac OS X + if platform == 'Darwin': + os.write(tmp_fd, DUMMY_CA_CERT) + # Default Homebrew path for OpenSSL certs + paths_checked.append('/usr/local/etc/openssl') + + # for all of the paths, find any .crt or .pem files + # and compile them into single temp file for use + # in the ssl check to speed up the test + for path in paths_checked: + if os.path.exists(path) and os.path.isdir(path): + dir_contents = os.listdir(path) + for f in dir_contents: + full_path = os.path.join(path, f) + if os.path.isfile(full_path) and os.path.splitext(f)[1] in ('.crt','.pem'): + try: + cert_file = open(full_path, 'r') + os.write(tmp_fd, cert_file.read()) + os.write(tmp_fd, '\n') + cert_file.close() + except: + pass + + return (tmp_path, paths_checked) + + def validate_proxy_response(self, response, valid_codes=[200]): + ''' + make sure we get back a valid code from the proxy + ''' + try: + (http_version, resp_code, msg) = re.match(r'(HTTP/\d\.\d) (\d\d\d) (.*)', response).groups() + if int(resp_code) not in valid_codes: + raise Exception + except: + self.module.fail_json(msg='Connection to proxy failed') + + def detect_no_proxy(self, url): + ''' + Detect if the 'no_proxy' environment variable is set and honor those locations. + ''' + env_no_proxy = os.environ.get('no_proxy') + if env_no_proxy: + env_no_proxy = env_no_proxy.split(',') + netloc = urlparse.urlparse(url).netloc + + for host in env_no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # Our requested URL matches something in no_proxy, so don't + # use the proxy for this + return False + return True + + def http_request(self, req): + tmp_ca_cert_path, paths_checked = self.get_ca_certs() + https_proxy = os.environ.get('https_proxy') + + # Detect if 'no_proxy' environment variable is set and if our URL is included + use_proxy = self.detect_no_proxy(req.get_full_url()) + + if not use_proxy: + # ignore proxy settings for this host request + return req + + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if https_proxy: + proxy_parts = generic_urlparse(urlparse.urlparse(https_proxy)) + s.connect((proxy_parts.get('hostname'), proxy_parts.get('port'))) + if proxy_parts.get('scheme') == 'http': + s.sendall(self.CONNECT_COMMAND % (self.hostname, self.port)) + if proxy_parts.get('username'): + credentials = "%s:%s" % (proxy_parts.get('username',''), proxy_parts.get('password','')) + s.sendall('Proxy-Authorization: Basic %s\r\n' % credentials.encode('base64').strip()) + s.sendall('\r\n') + connect_result = s.recv(4096) + self.validate_proxy_response(connect_result) + ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED) + else: + self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme')) + else: + s.connect((self.hostname, self.port)) + ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED) + # close the ssl connection + #ssl_s.unwrap() + s.close() + except (ssl.SSLError, socket.error), e: + # fail if we tried all of the certs but none worked + if 'connection refused' in str(e).lower(): + self.module.fail_json(msg='Failed to connect to %s:%s.' % (self.hostname, self.port)) + else: + self.module.fail_json( + msg='Failed to validate the SSL certificate for %s:%s. ' % (self.hostname, self.port) + \ + 'Use validate_certs=no or make sure your managed systems have a valid CA certificate installed. ' + \ + 'Paths checked for this platform: %s' % ", ".join(paths_checked) + ) + try: + # cleanup the temp file created, don't worry + # if it fails for some reason + os.remove(tmp_ca_cert_path) + except: + pass + + return req + + https_request = http_request + + +def url_argument_spec(): + ''' + Creates an argument spec that can be used with any module + that will be requesting content via urllib/urllib2 + ''' + return dict( + url = dict(), + force = dict(default='no', aliases=['thirsty'], type='bool'), + http_agent = dict(default='ansible-httpget'), + use_proxy = dict(default='yes', type='bool'), + validate_certs = dict(default='yes', type='bool'), + url_username = dict(required=False), + url_password = dict(required=False), + ) + + +def fetch_url(module, url, data=None, headers=None, method=None, + use_proxy=True, force=False, last_mod_time=None, timeout=10): + ''' + Fetches a file from an HTTP/FTP server using urllib2 + ''' + + if not HAS_URLLIB: + module.fail_json(msg='urllib is not installed') + if not HAS_URLLIB2: + module.fail_json(msg='urllib2 is not installed') + elif not HAS_URLPARSE: + module.fail_json(msg='urlparse is not installed') + + r = None + handlers = [] + info = dict(url=url) + + distribution = get_distribution() + # Get validate_certs from the module params + validate_certs = module.params.get('validate_certs', True) + + # FIXME: change the following to use the generic_urlparse function + # to remove the indexed references for 'parsed' + parsed = urlparse.urlparse(url) + if parsed[0] == 'https': + if not HAS_SSL and validate_certs: + if distribution == 'Redhat': + module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended. You can also install python-ssl from EPEL') + else: + module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended') + + elif validate_certs: + # do the cert validation + netloc = parsed[1] + if '@' in netloc: + netloc = netloc.split('@', 1)[1] + if ':' in netloc: + hostname, port = netloc.split(':', 1) + else: + hostname = netloc + port = 443 + # create the SSL validation handler and + # add it to the list of handlers + ssl_handler = SSLValidationHandler(module, hostname, port) + handlers.append(ssl_handler) + + if parsed[0] != 'ftp': + username = module.params.get('url_username', '') + if username: + password = module.params.get('url_password', '') + netloc = parsed[1] + elif '@' in parsed[1]: + credentials, netloc = parsed[1].split('@', 1) + if ':' in credentials: + username, password = credentials.split(':', 1) + else: + username = credentials + password = '' + + parsed = list(parsed) + parsed[1] = netloc + + # reconstruct url without credentials + url = urlparse.urlunparse(parsed) + + if username: + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + + # this creates a password manager + passman.add_password(None, netloc, username, password) + + # because we have put None at the start it will always + # use this username/password combination for urls + # for which `theurl` is a super-url + authhandler = urllib2.HTTPBasicAuthHandler(passman) + + # create the AuthHandler + handlers.append(authhandler) + + if not use_proxy: + proxyhandler = urllib2.ProxyHandler({}) + handlers.append(proxyhandler) + + # pre-2.6 versions of python cannot use the custom https + # handler, since the socket class is lacking this method + if hasattr(socket, 'create_connection'): + handlers.append(CustomHTTPSHandler) + + opener = urllib2.build_opener(*handlers) + urllib2.install_opener(opener) + + if method: + if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'): + module.fail_json(msg='invalid HTTP request method; %s' % method.upper()) + request = RequestWithMethod(url, method.upper(), data) + else: + request = urllib2.Request(url, data) + + # add the custom agent header, to help prevent issues + # with sites that block the default urllib agent string + request.add_header('User-agent', module.params.get('http_agent')) + + # if we're ok with getting a 304, set the timestamp in the + # header, otherwise make sure we don't get a cached copy + if last_mod_time and not force: + tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') + request.add_header('If-Modified-Since', tstamp) + else: + request.add_header('cache-control', 'no-cache') + + # user defined headers now, which may override things we've set above + if headers: + if not isinstance(headers, dict): + module.fail_json("headers provided to fetch_url() must be a dict") + for header in headers: + request.add_header(header, headers[header]) + + try: + if sys.version_info < (2,6,0): + # urlopen in python prior to 2.6.0 did not + # have a timeout parameter + r = urllib2.urlopen(request, None) + else: + r = urllib2.urlopen(request, None, timeout) + info.update(r.info()) + info['url'] = r.geturl() # The URL goes in too, because of redirects. + info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200)) + except urllib2.HTTPError, e: + info.update(dict(msg=str(e), status=e.code)) + except urllib2.URLError, e: + code = int(getattr(e, 'code', -1)) + info.update(dict(msg="Request failed: %s" % str(e), status=code)) + except socket.error, e: + info.update(dict(msg="Connection failure: %s" % str(e), status=-1)) + except Exception, e: + info.update(dict(msg="An unknown error occurred: %s" % str(e), status=-1)) + + return r, info + diff --git a/v2/ansible/modules/core b/v2/ansible/modules/core -Subproject cb69744bcee4b4217d83b4a30006635ba69e2aa +Subproject 1394920cd3e440f5806463d0c1cfbe4a4b94f42 diff --git a/v2/ansible/modules/extras b/v2/ansible/modules/extras -Subproject 8a4f07eecd2bb877f51b7b04b5352efa6076cce +Subproject ed35fc69bf3cf280cdc3d272d2aec419e47a07b diff --git a/v2/ansible/new_inventory/__init__.py b/v2/ansible/new_inventory/__init__.py new file mode 100644 index 0000000000..bcf87c9ef8 --- /dev/null +++ b/v2/ansible/new_inventory/__init__.py @@ -0,0 +1,341 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible import constants as C +from ansible.inventory.group import Group +from ansible.inventory.host import Host +from ansible.inventory.aggregate import InventoryAggregateParser + +class Inventory: + ''' + Create hosts and groups from inventory + + Retrieve the hosts and groups that ansible knows about from this class. + + Retrieve raw variables (non-expanded) from the Group and Host classes + returned from here. + ''' + + def __init__(self, inventory_list=C.DEFAULT_HOST_LIST): + ''' + :kwarg inventory_list: A list of inventory sources. This may be file + names which will be parsed as ini-like files, executable scripts + which return inventory data as json, directories of both of the above, + or hostnames. Files and directories are + :kwarg vault_password: Password to use if any of the inventory sources + are in an ansible vault + ''' + + self._restricted_to = None + self._filter_pattern = None + + parser = InventoryAggregateParser(inventory_list) + parser.parse() + + self._basedir = parser.basedir + self._hosts = parser.hosts + self._groups = parser.groups + + def get_hosts(self): + ''' + Return the list of hosts, after filtering based on any set pattern + and restricting the results based on the set host restrictions. + ''' + + if self._filter_pattern: + hosts = self._filter_hosts() + else: + hosts = self._hosts[:] + + if self._restricted_to is not None: + # this will preserve the order of hosts after intersecting them + res_set = set(hosts).intersection(self._restricted_to) + return [h for h in hosts if h in res_set] + else: + return hosts[:] + + def get_groups(self): + ''' + Retrieve the Group objects known to the Inventory + ''' + + return self._groups[:] + + def get_host(self, hostname): + ''' + Retrieve the Host object for a hostname + ''' + + for host in self._hosts: + if host.name == hostname: + return host + + return None + + def get_group(self, groupname): + ''' + Retrieve the Group object for a groupname + ''' + + for group in self._groups: + if group.name == group_name: + return group + + return None + + def add_group(self, group): + ''' + Add a new group to the inventory + ''' + + if group not in self._groups: + self._groups.append(group) + + def set_filter_pattern(self, pattern='all'): + ''' + Sets a pattern upon which hosts/groups will be filtered. + This pattern can contain logical groupings such as unions, + intersections and negations using special syntax. + ''' + + self._filter_pattern = pattern + + def set_host_restriction(self, restriction): + ''' + Restrict operations to hosts in the given list + ''' + + assert isinstance(restriction, list) + self._restricted_to = restriction[:] + + def remove_host_restriction(self): + ''' + Remove the restriction on hosts, if any. + ''' + + self._restricted_to = None + + def _filter_hosts(self): + """ + Limits inventory results to a subset of inventory that matches a given + list of patterns, such as to select a subset of a hosts selection that also + belongs to a certain geographic group or numeric slice. + + Corresponds to --limit parameter to ansible-playbook + + :arg patterns: The pattern to limit with. If this is None it + clears the subset. Multiple patterns may be specified as a comma, + semicolon, or colon separated string. + """ + + hosts = [] + + pattern_regular = [] + pattern_intersection = [] + pattern_exclude = [] + + patterns = self._pattern.replace(";",":").split(":") + for p in patterns: + if p.startswith("!"): + pattern_exclude.append(p) + elif p.startswith("&"): + pattern_intersection.append(p) + elif p: + pattern_regular.append(p) + + # if no regular pattern was given, hence only exclude and/or intersection + # make that magically work + if pattern_regular == []: + pattern_regular = ['all'] + + # when applying the host selectors, run those without the "&" or "!" + # first, then the &s, then the !s. + patterns = pattern_regular + pattern_intersection + pattern_exclude + + for p in patterns: + intersect = False + negate = False + if p.startswith('&'): + intersect = True + elif p.startswith('!'): + p = p[1:] + negate = True + + target = self._resolve_pattern(p) + if isinstance(target, Host): + if negate and target in hosts: + # remove it + hosts.remove(target) + elif target not in hosts: + # for both union and intersections, we just append it + hosts.append(target) + else: + if intersect: + hosts = [ h for h in hosts if h not in target ] + elif negate: + hosts = [ h for h in hosts if h in target ] + else: + to_append = [ h for h in target if h.name not in [ y.name for y in hosts ] ] + hosts.extend(to_append) + + return hosts + + def _resolve_pattern(self, pattern): + target = self.get_host(pattern) + if target: + return target + else: + (name, enumeration_details) = self._enumeration_info(pattern) + hpat = self._hosts_in_unenumerated_pattern(name) + result = self._apply_ranges(pattern, hpat) + return result + + def _enumeration_info(self, pattern): + """ + returns (pattern, limits) taking a regular pattern and finding out + which parts of it correspond to start/stop offsets. limits is + a tuple of (start, stop) or None + """ + + # Do not parse regexes for enumeration info + if pattern.startswith('~'): + return (pattern, None) + + # The regex used to match on the range, which can be [x] or [x-y]. + pattern_re = re.compile("^(.*)\[([-]?[0-9]+)(?:(?:-)([0-9]+))?\](.*)$") + m = pattern_re.match(pattern) + if m: + (target, first, last, rest) = m.groups() + first = int(first) + if last: + if first < 0: + raise errors.AnsibleError("invalid range: negative indices cannot be used as the first item in a range") + last = int(last) + else: + last = first + return (target, (first, last)) + else: + return (pattern, None) + + def _apply_ranges(self, pat, hosts): + """ + given a pattern like foo, that matches hosts, return all of hosts + given a pattern like foo[0:5], where foo matches hosts, return the first 6 hosts + """ + + # If there are no hosts to select from, just return the + # empty set. This prevents trying to do selections on an empty set. + # issue#6258 + if not hosts: + return hosts + + (loose_pattern, limits) = self._enumeration_info(pat) + if not limits: + return hosts + + (left, right) = limits + + if left == '': + left = 0 + if right == '': + right = 0 + left=int(left) + right=int(right) + try: + if left != right: + return hosts[left:right] + else: + return [ hosts[left] ] + except IndexError: + raise errors.AnsibleError("no hosts matching the pattern '%s' were found" % pat) + + def _hosts_in_unenumerated_pattern(self, pattern): + """ Get all host names matching the pattern """ + + results = [] + hosts = [] + hostnames = set() + + # ignore any negative checks here, this is handled elsewhere + pattern = pattern.replace("!","").replace("&", "") + + def __append_host_to_results(host): + if host not in results and host.name not in hostnames: + hostnames.add(host.name) + results.append(host) + + groups = self.get_groups() + for group in groups: + if pattern == 'all': + for host in group.get_hosts(): + __append_host_to_results(host) + else: + if self._match(group.name, pattern): + for host in group.get_hosts(): + __append_host_to_results(host) + else: + matching_hosts = self._match_list(group.get_hosts(), 'name', pattern) + for host in matching_hosts: + __append_host_to_results(host) + + if pattern in ["localhost", "127.0.0.1"] and len(results) == 0: + new_host = self._create_implicit_localhost(pattern) + results.append(new_host) + return results + + def _create_implicit_localhost(self, pattern): + new_host = Host(pattern) + new_host._connection = 'local' + new_host.set_variable("ansible_python_interpreter", sys.executable) + ungrouped = self.get_group("ungrouped") + if ungrouped is None: + self.add_group(Group('ungrouped')) + ungrouped = self.get_group('ungrouped') + self.get_group('all').add_child_group(ungrouped) + ungrouped.add_host(new_host) + return new_host + + def is_file(self): + ''' + Did inventory come from a file? + + :returns: True if the inventory is file based, False otherwise + ''' + pass + + def src(self): + ''' + What's the complete path to the inventory file? + + :returns: Complete path to the inventory file. None if inventory is + not file-based + ''' + pass + + def basedir(self): + ''' + What directory from which the inventory was read. + ''' + + return self._basedir + diff --git a/v2/ansible/new_inventory/aggregate.py b/v2/ansible/new_inventory/aggregate.py new file mode 100644 index 0000000000..6bdf2ddcb6 --- /dev/null +++ b/v2/ansible/new_inventory/aggregate.py @@ -0,0 +1,61 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from . import InventoryParser +#from . ini import InventoryIniParser +#from . script import InventoryScriptParser + +class InventoryAggregateParser(InventoryParser): + + def __init__(self, inven_sources): + self.inven_source = inven_sources + self.hosts = dict() + self.groups = dict() + + def reset_parser(self): + super(InventoryAggregateParser, self).reset_parser() + + def parse(self, refresh=False): + # InventoryDirectoryParser is a InventoryAggregateParser so we avoid + # a circular import by importing here + from . directory import InventoryAggregateParser + if super(InventoryAggregateParser, self).parse(refresh): + return self.parsed + + for entry in self.inven_sources: + if os.path.sep in entry: + # file or directory + if os.path.isdir(entry): + parser = directory.InventoryDirectoryParser(filename=entry) + elif utils.is_executable(entry): + parser = InventoryScriptParser(filename=entry) + else: + parser = InventoryIniParser(filename=entry) + else: + # hostname + parser = HostnameParser(hostname=entry) + hosts, groups = parser.parse() + self._merge(self.hosts, hosts) + self._merge(self.groups, groups) diff --git a/v2/ansible/executor/template_engine.py b/v2/ansible/new_inventory/group.py index 785fc45992..785fc45992 100644 --- a/v2/ansible/executor/template_engine.py +++ b/v2/ansible/new_inventory/group.py diff --git a/v2/ansible/new_inventory/host.py b/v2/ansible/new_inventory/host.py new file mode 100644 index 0000000000..78f190c423 --- /dev/null +++ b/v2/ansible/new_inventory/host.py @@ -0,0 +1,51 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +class Host: + def __init__(self, name): + self._name = name + self._connection = None + self._ipv4_address = '' + self._ipv6_address = '' + self._port = 22 + self._vars = dict() + + def __repr__(self): + return self.get_name() + + def get_name(self): + return self._name + + def get_groups(self): + return [] + + def set_variable(self, name, value): + ''' sets a variable for this host ''' + + self._vars[name] = value + + def get_vars(self): + ''' returns all variables for this host ''' + + all_vars = self._vars.copy() + all_vars.update(dict(inventory_hostname=self._name)) + return all_vars + diff --git a/v2/ansible/parsing/__init__.py b/v2/ansible/parsing/__init__.py index 785fc45992..f8a3e96746 100644 --- a/v2/ansible/parsing/__init__.py +++ b/v2/ansible/parsing/__init__.py @@ -19,3 +19,187 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import json +import os + +from yaml import load, YAMLError + +from ansible.errors import AnsibleParserError +from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR +from ansible.parsing.vault import VaultLib +from ansible.parsing.splitter import unquote +from ansible.parsing.yaml.loader import AnsibleLoader +from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject +from ansible.utils.path import unfrackpath + +class DataLoader(): + + ''' + The DataLoader class is used to load and parse YAML or JSON content, + either from a given file name or from a string that was previously + read in through other means. A Vault password can be specified, and + any vault-encrypted files will be decrypted. + + Data read from files will also be cached, so the file will never be + read from disk more than once. + + Usage: + + dl = DataLoader() + (or) + dl = DataLoader(vault_password='foo') + + ds = dl.load('...') + ds = dl.load_from_file('/path/to/file') + ''' + + def __init__(self, vault_password=None): + self._basedir = '.' + self._vault_password = vault_password + self._FILE_CACHE = dict() + + self._vault = VaultLib(password=vault_password) + + def load(self, data, file_name='<string>', show_content=True): + ''' + Creates a python datastructure from the given data, which can be either + a JSON or YAML string. + ''' + + try: + # we first try to load this data as JSON + return json.loads(data) + except: + try: + # if loading JSON failed for any reason, we go ahead + # and try to parse it as YAML instead + return self._safe_load(data, file_name=file_name) + except YAMLError as yaml_exc: + self._handle_error(yaml_exc, file_name, show_content) + + def load_from_file(self, file_name): + ''' Loads data from a file, which can contain either JSON or YAML. ''' + + file_name = self.path_dwim(file_name) + + # if the file has already been read in and cached, we'll + # return those results to avoid more file/vault operations + if file_name in self._FILE_CACHE: + return self._FILE_CACHE[file_name] + + # read the file contents and load the data structure from them + (file_data, show_content) = self._get_file_contents(file_name) + parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content) + + # cache the file contents for next time + self._FILE_CACHE[file_name] = parsed_data + + return parsed_data + + def path_exists(self, path): + return os.path.exists(path) + + def is_directory(self, path): + return os.path.isdir(path) + + def is_file(self, path): + return os.path.isfile(path) + + def _safe_load(self, stream, file_name=None): + ''' Implements yaml.safe_load(), except using our custom loader class. ''' + + loader = AnsibleLoader(stream, file_name) + try: + return loader.get_single_data() + finally: + loader.dispose() + + def _get_file_contents(self, file_name): + ''' + Reads the file contents from the given file name, and will decrypt them + if they are found to be vault-encrypted. + ''' + + if not self.path_exists(file_name) or not self.is_file(file_name): + raise AnsibleParserError("the file_name '%s' does not exist, or is not readable" % file_name) + + show_content = True + try: + with open(file_name, 'r') as f: + data = f.read() + if self._vault.is_encrypted(data): + data = self._vault.decrypt(data) + show_content = False + return (data, show_content) + except (IOError, OSError) as e: + raise AnsibleParserError("an error occured while trying to read the file '%s': %s" % (file_name, str(e))) + + def _handle_error(self, yaml_exc, file_name, show_content): + ''' + Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the + file name/position where a YAML exception occured, and raises an AnsibleParserError + to display the syntax exception information. + ''' + + # if the YAML exception contains a problem mark, use it to construct + # an object the error class can use to display the faulty line + err_obj = None + if hasattr(yaml_exc, 'problem_mark'): + err_obj = AnsibleBaseYAMLObject() + err_obj.set_position_info(file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) + + raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content) + + def get_basedir(self): + ''' returns the current basedir ''' + return self._basedir + + def set_basedir(self, basedir): + ''' sets the base directory, used to find files when a relative path is given ''' + + if basedir is not None: + self._basedir = basedir + + def path_dwim(self, given): + ''' + make relative paths work like folks expect. + ''' + + given = unquote(given) + + if given.startswith("/"): + return os.path.abspath(given) + elif given.startswith("~"): + return os.path.abspath(os.path.expanduser(given)) + else: + return os.path.abspath(os.path.join(self._basedir, given)) + + def path_dwim_relative(self, role_path, dirname, source): + ''' find one file in a directory one level up in a dir named dirname relative to current ''' + + basedir = os.path.dirname(role_path) + if os.path.islink(basedir): + basedir = unfrackpath(basedir) + template2 = os.path.join(basedir, dirname, source) + else: + template2 = os.path.join(basedir, '..', dirname, source) + + source1 = os.path.join(role_path, dirname, source) + if os.path.exists(source1): + return source1 + + cur_basedir = self._basedir + self.set_basedir(basedir) + source2 = self.path_dwim(template2) + if os.path.exists(source2): + self.set_basedir(cur_basedir) + return source2 + + obvious_local_path = self.path_dwim(source) + if os.path.exists(obvious_local_path): + self.set_basedir(cur_basedir) + return obvious_local_path + + self.set_basedir(cur_basedir) + return source2 # which does not exist + diff --git a/v2/ansible/parsing/mod_args.py b/v2/ansible/parsing/mod_args.py index 0bb1c3fa2b..eddc093ef3 100644 --- a/v2/ansible/parsing/mod_args.py +++ b/v2/ansible/parsing/mod_args.py @@ -20,9 +20,10 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from six import iteritems, string_types +from types import NoneType from ansible.errors import AnsibleParserError -from ansible.plugins import module_finder +from ansible.plugins import module_loader from ansible.parsing.splitter import parse_kv class ModuleArgsParser: @@ -54,6 +55,15 @@ class ModuleArgsParser: src: a dest: b + # extra gross, but also legal. in this case, the args specified + # will act as 'defaults' and will be overriden by any args specified + # in one of the other formats (complex args under the action, or + # parsed from the k=v string + - command: 'pwd' + args: + chdir: '/tmp' + + This class has some of the logic to canonicalize these into the form - module: <module_name> @@ -103,29 +113,39 @@ class ModuleArgsParser: return (action, args) - def _normalize_parameters(self, thing, action=None): + def _normalize_parameters(self, thing, action=None, additional_args=dict()): ''' arguments can be fuzzy. Deal with all the forms. ''' - args = dict() + # final args are the ones we'll eventually return, so first update + # them with any additional args specified, which have lower priority + # than those which may be parsed/normalized next + final_args = dict() + if additional_args: + final_args.update(additional_args) # how we normalize depends if we figured out what the module name is # yet. If we have already figured it out, it's an 'old style' invocation. # otherwise, it's not if action is not None: - args = self._normalize_old_style_args(thing) + args = self._normalize_old_style_args(thing, action) else: (action, args) = self._normalize_new_style_args(thing) # this can occasionally happen, simplify - if 'args' in args: + if args and 'args' in args: args = args['args'] - return (action, args) + # finally, update the args we're going to return with the ones + # which were normalized above + if args: + final_args.update(args) - def _normalize_old_style_args(self, thing): + return (action, final_args) + + def _normalize_old_style_args(self, thing, action): ''' deals with fuzziness in old-style (action/local_action) module invocations returns tuple of (module_name, dictionary_args) @@ -143,9 +163,13 @@ class ModuleArgsParser: args = thing elif isinstance(thing, string_types): # form is like: local_action: copy src=a dest=b ... pretty common - args = parse_kv(thing) + check_raw = action in ('command', 'shell', 'script') + args = parse_kv(thing, check_raw=check_raw) + elif isinstance(thing, NoneType): + # this can happen with modules which take no params, like ping: + args = None else: - raise AnsibleParsingError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds) + raise AnsibleParserError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds) return args def _normalize_new_style_args(self, thing): @@ -176,11 +200,12 @@ class ModuleArgsParser: elif isinstance(thing, string_types): # form is like: copy: src=a dest=b ... common shorthand throughout ansible (action, args) = self._split_module_string(thing) - args = parse_kv(args) + check_raw = action in ('command', 'shell', 'script') + args = parse_kv(args, check_raw=check_raw) else: # need a dict or a string, so giving up - raise AnsibleParsingError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds) + raise AnsibleParserError("unexpected parameter type in action: %s" % type(thing), obj=self._task_ds) return (action, args) @@ -202,13 +227,20 @@ class ModuleArgsParser: # We can have one of action, local_action, or module specified # + + # this is the 'extra gross' scenario detailed above, so we grab + # the args and pass them in as additional arguments, which can/will + # be overwritten via dict updates from the other arg sources below + # FIXME: add test cases for this + additional_args = self._task_ds.get('args', dict()) + # action if 'action' in self._task_ds: # an old school 'action' statement thing = self._task_ds['action'] delegate_to = None - action, args = self._normalize_parameters(thing) + action, args = self._normalize_parameters(thing, additional_args=additional_args) # local_action if 'local_action' in self._task_ds: @@ -218,23 +250,27 @@ class ModuleArgsParser: raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds) thing = self._task_ds.get('local_action', '') delegate_to = 'localhost' - action, args = self._normalize_parameters(thing) + action, args = self._normalize_parameters(thing, additional_args=additional_args) # module: <stuff> is the more new-style invocation # walk the input dictionary to see we recognize a module name for (item, value) in iteritems(self._task_ds): - if item in module_finder: + if item in module_loader or item == 'meta': # finding more than one module name is a problem if action is not None: raise AnsibleParserError("conflicting action statements", obj=self._task_ds) action = item thing = value - action, args = self._normalize_parameters(value, action=action) + action, args = self._normalize_parameters(value, action=action, additional_args=additional_args) # if we didn't see any module in the task at all, it's not a task really if action is None: raise AnsibleParserError("no action detected in task", obj=self._task_ds) + # FIXME: disabled for now, as there are other places besides the shell/script modules where + # having variables as the sole param for the module is valid (include_vars, add_host, and group_by?) + #elif args.get('_raw_params', '') != '' and action not in ('command', 'shell', 'script', 'include_vars'): + # raise AnsibleParserError("this task has extra params, which is only allowed in the command, shell or script module.", obj=self._task_ds) # shell modules require special handling (action, args) = self._handle_shell_weirdness(action, args) diff --git a/v2/ansible/parsing/splitter.py b/v2/ansible/parsing/splitter.py index 470ab90c3c..9705baf169 100644 --- a/v2/ansible/parsing/splitter.py +++ b/v2/ansible/parsing/splitter.py @@ -40,7 +40,20 @@ def parse_kv(args, check_raw=False): raw_params = [] for x in vargs: if "=" in x: - k, v = x.split("=", 1) + pos = 0 + try: + while True: + pos = x.index('=', pos + 1) + if pos > 0 and x[pos - 1] != '\\': + break + except ValueError: + # ran out of string, but we must have some escaped equals, + # so replace those and append this to the list of raw params + raw_params.append(x.replace('\\=', '=')) + continue + + k = x[:pos] + v = x[pos + 1:] # only internal variables can start with an underscore, so # we don't allow users to set them directy in arguments diff --git a/v2/test/parsing/yaml/__init__.py b/v2/ansible/parsing/utils/__init__.py index 785fc45992..785fc45992 100644 --- a/v2/test/parsing/yaml/__init__.py +++ b/v2/ansible/parsing/utils/__init__.py diff --git a/v2/ansible/parsing/utils/jsonify.py b/v2/ansible/parsing/utils/jsonify.py new file mode 100644 index 0000000000..37c97d0195 --- /dev/null +++ b/v2/ansible/parsing/utils/jsonify.py @@ -0,0 +1,26 @@ +# FIXME: header + +try: + import json +except ImportError: + import simplejson as json + +def jsonify(result, format=False): + ''' format JSON output (uncompressed or uncompressed) ''' + + if result is None: + return "{}" + result2 = result.copy() + for key, value in result2.items(): + if type(value) is str: + result2[key] = value.decode('utf-8', 'ignore') + + indent = None + if format: + indent = 4 + + try: + return json.dumps(result2, sort_keys=True, indent=indent, ensure_ascii=False) + except UnicodeDecodeError: + return json.dumps(result2, sort_keys=True, indent=indent) + diff --git a/v2/ansible/parsing/yaml/__init__.py b/v2/ansible/parsing/yaml/__init__.py index 3f5ebb7c99..785fc45992 100644 --- a/v2/ansible/parsing/yaml/__init__.py +++ b/v2/ansible/parsing/yaml/__init__.py @@ -19,156 +19,3 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import json -import os - -from yaml import load, YAMLError - -from ansible.errors import AnsibleParserError - -from ansible.parsing.vault import VaultLib -from ansible.parsing.splitter import unquote -from ansible.parsing.yaml.loader import AnsibleLoader -from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject -from ansible.parsing.yaml.strings import YAML_SYNTAX_ERROR - -class DataLoader(): - - ''' - The DataLoader class is used to load and parse YAML or JSON content, - either from a given file name or from a string that was previously - read in through other means. A Vault password can be specified, and - any vault-encrypted files will be decrypted. - - Data read from files will also be cached, so the file will never be - read from disk more than once. - - Usage: - - dl = DataLoader() - (or) - dl = DataLoader(vault_password='foo') - - ds = dl.load('...') - ds = dl.load_from_file('/path/to/file') - ''' - - _FILE_CACHE = dict() - - def __init__(self, vault_password=None): - self._basedir = '.' - self._vault = VaultLib(password=vault_password) - - def load(self, data, file_name='<string>', show_content=True): - ''' - Creates a python datastructure from the given data, which can be either - a JSON or YAML string. - ''' - - try: - # we first try to load this data as JSON - return json.loads(data) - except: - try: - # if loading JSON failed for any reason, we go ahead - # and try to parse it as YAML instead - return self._safe_load(data, file_name=file_name) - except YAMLError as yaml_exc: - self._handle_error(yaml_exc, file_name, show_content) - - def load_from_file(self, file_name): - ''' Loads data from a file, which can contain either JSON or YAML. ''' - - file_name = self.path_dwim(file_name) - - # if the file has already been read in and cached, we'll - # return those results to avoid more file/vault operations - if file_name in self._FILE_CACHE: - return self._FILE_CACHE[file_name] - - # read the file contents and load the data structure from them - (file_data, show_content) = self._get_file_contents(file_name) - parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content) - - # cache the file contents for next time - self._FILE_CACHE[file_name] = parsed_data - - return parsed_data - - def path_exists(self, path): - return os.path.exists(path) - - def is_directory(self, path): - return os.path.isdir(path) - - def is_file(self, path): - return os.path.isfile(path) - - def _safe_load(self, stream, file_name=None): - ''' Implements yaml.safe_load(), except using our custom loader class. ''' - - loader = AnsibleLoader(stream, file_name) - try: - return loader.get_single_data() - finally: - loader.dispose() - - def _get_file_contents(self, file_name): - ''' - Reads the file contents from the given file name, and will decrypt them - if they are found to be vault-encrypted. - ''' - if not self.path_exists(file_name) or not self.is_file(file_name): - raise AnsibleParserError("the file_name '%s' does not exist, or is not readable" % file_name) - - show_content = True - try: - with open(file_name, 'r') as f: - data = f.read() - if self._vault.is_encrypted(data): - data = self._vault.decrypt(data) - show_content = False - return (data, show_content) - except (IOError, OSError) as e: - raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e))) - - def _handle_error(self, yaml_exc, file_name, show_content): - ''' - Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the - file name/position where a YAML exception occurred, and raises an AnsibleParserError - to display the syntax exception information. - ''' - - # if the YAML exception contains a problem mark, use it to construct - # an object the error class can use to display the faulty line - err_obj = None - if hasattr(yaml_exc, 'problem_mark'): - err_obj = AnsibleBaseYAMLObject() - err_obj.set_position_info(file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) - - raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content) - - def get_basedir(self): - ''' returns the current basedir ''' - return self._basedir - - def set_basedir(self, basedir): - ''' sets the base directory, used to find files when a relative path is given ''' - - if basedir is not None: - self._basedir = basedir - - def path_dwim(self, given): - ''' - make relative paths work like folks expect. - ''' - - given = unquote(given) - - if given.startswith("/"): - return os.path.abspath(given) - elif given.startswith("~"): - return os.path.abspath(os.path.expanduser(given)) - else: - return os.path.abspath(os.path.join(self._basedir, given)) - diff --git a/v2/ansible/playbook/__init__.py b/v2/ansible/playbook/__init__.py index 2d594c4802..1c03355907 100644 --- a/v2/ansible/playbook/__init__.py +++ b/v2/ansible/playbook/__init__.py @@ -22,9 +22,10 @@ __metaclass__ = type import os from ansible.errors import AnsibleError, AnsibleParserError -from ansible.parsing.yaml import DataLoader +from ansible.parsing import DataLoader from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.play import Play +from ansible.playbook.playbook_include import PlaybookInclude from ansible.plugins import push_basedir @@ -33,34 +34,33 @@ __all__ = ['Playbook'] class Playbook: - def __init__(self, loader=None): + def __init__(self, loader): # Entries in the datastructure of a playbook may # be either a play or an include statement self._entries = [] - self._basedir = '.' - - if loader: - self._loader = loader - else: - self._loader = DataLoader() + self._basedir = os.getcwd() + self._loader = loader @staticmethod - def load(file_name, loader=None): + def load(file_name, variable_manager=None, loader=None): pb = Playbook(loader=loader) - pb._load_playbook_data(file_name) + pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager) return pb - def _load_playbook_data(self, file_name): + def _load_playbook_data(self, file_name, variable_manager): + + if os.path.isabs(file_name): + self._basedir = os.path.dirname(file_name) + else: + self._basedir = os.path.normpath(os.path.join(self._basedir, os.path.dirname(file_name))) - # add the base directory of the file to the data loader, - # so that it knows where to find relatively pathed files - basedir = os.path.dirname(file_name) - self._loader.set_basedir(basedir) + # set the loaders basedir + self._loader.set_basedir(self._basedir) # also add the basedir to the list of module directories - push_basedir(basedir) + push_basedir(self._basedir) - ds = self._loader.load_from_file(file_name) + ds = self._loader.load_from_file(os.path.basename(file_name)) if not isinstance(ds, list): raise AnsibleParserError("playbooks must be a list of plays", obj=ds) @@ -72,11 +72,14 @@ class Playbook: raise AnsibleParserError("playbook entries must be either a valid play or an include statement", obj=entry) if 'include' in entry: - entry_obj = PlaybookInclude.load(entry, loader=self._loader) + pb = PlaybookInclude.load(entry, basedir=self._basedir, variable_manager=variable_manager, loader=self._loader) + self._entries.extend(pb._entries) else: - entry_obj = Play.load(entry, loader=self._loader) + entry_obj = Play.load(entry, variable_manager=variable_manager, loader=self._loader) + self._entries.append(entry_obj) - self._entries.append(entry_obj) + def get_loader(self): + return self._loader def get_entries(self): return self._entries[:] diff --git a/v2/ansible/playbook/attribute.py b/v2/ansible/playbook/attribute.py index 1e7e404181..8a727a0193 100644 --- a/v2/ansible/playbook/attribute.py +++ b/v2/ansible/playbook/attribute.py @@ -21,11 +21,12 @@ __metaclass__ = type class Attribute: - def __init__(self, isa=None, private=False, default=None): + def __init__(self, isa=None, private=False, default=None, required=False): self.isa = isa self.private = private self.default = default + self.required = required class FieldAttribute(Attribute): pass diff --git a/v2/ansible/playbook/base.py b/v2/ansible/playbook/base.py index c7748095a5..dffdabd4af 100644 --- a/v2/ansible/playbook/base.py +++ b/v2/ansible/playbook/base.py @@ -19,22 +19,36 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import uuid + from inspect import getmembers from io import FileIO from six import iteritems, string_types +from jinja2.exceptions import UndefinedError + from ansible.errors import AnsibleParserError +from ansible.parsing import DataLoader from ansible.playbook.attribute import Attribute, FieldAttribute -from ansible.parsing.yaml import DataLoader +from ansible.template import Templar +from ansible.utils.boolean import boolean + +from ansible.utils.debug import debug + +from ansible.template import template class Base: def __init__(self): - # initialize the data loader, this will be provided later - # when the object is actually loaded + # initialize the data loader and variable manager, which will be provided + # later when the object is actually loaded self._loader = None + self._variable_manager = None + + # every object gets a random uuid: + self._uuid = uuid.uuid4() # each class knows attributes set upon it, see Task.py for example self._attributes = dict() @@ -60,11 +74,15 @@ class Base: return ds - def load_data(self, ds, loader=None): + def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None + # the variable manager class is used to manage and merge variables + # down to a single dictionary for reference in templating, etc. + self._variable_manager = variable_manager + # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader @@ -94,13 +112,27 @@ class Base: else: self._attributes[name] = ds[name] - # return the constructed object + # run early, non-critical validation self.validate() + + # cache the datastructure internally + self._ds = ds + + # return the constructed object return self + def get_ds(self): + try: + return getattr(self, '_ds') + except AttributeError: + return None + def get_loader(self): return self._loader + def get_variable_manager(self): + return self._variable_manager + def _validate_attributes(self, ds): ''' Ensures that there are no keys in the datastructure which do @@ -112,7 +144,7 @@ class Base: if key not in valid_attrs: raise AnsibleParserError("'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds) - def validate(self): + def validate(self, all_vars=dict()): ''' validation that is done at parse time, not load time ''' # walk all fields in the object @@ -121,16 +153,110 @@ class Base: # run validator only if present method = getattr(self, '_validate_%s' % name, None) if method: - method(self, attribute) + method(attribute, name, getattr(self, name)) + + def copy(self): + ''' + Create a copy of this object and return it. + ''' + + new_me = self.__class__() - def post_validate(self, runner_context): + for (name, attribute) in iteritems(self._get_base_attributes()): + setattr(new_me, name, getattr(self, name)) + + new_me._loader = self._loader + new_me._variable_manager = self._variable_manager + + return new_me + + def post_validate(self, all_vars=dict(), fail_on_undefined=True): ''' we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. ''' - raise exception.NotImplementedError + basedir = None + if self._loader is not None: + basedir = self._loader.get_basedir() + + templar = Templar(loader=self._loader, variables=all_vars, fail_on_undefined=fail_on_undefined) + + for (name, attribute) in iteritems(self._get_base_attributes()): + + if getattr(self, name) is None: + if not attribute.required: + continue + else: + raise AnsibleParserError("the field '%s' is required but was not set" % name) + + try: + # if the attribute contains a variable, template it now + value = templar.template(getattr(self, name)) + + # run the post-validator if present + method = getattr(self, '_post_validate_%s' % name, None) + if method: + value = method(attribute, value, all_vars, fail_on_undefined) + else: + # otherwise, just make sure the attribute is of the type it should be + if attribute.isa == 'string': + value = unicode(value) + elif attribute.isa == 'int': + value = int(value) + elif attribute.isa == 'bool': + value = boolean(value) + elif attribute.isa == 'list': + if not isinstance(value, list): + value = [ value ] + elif attribute.isa == 'dict' and not isinstance(value, dict): + raise TypeError() + + # and assign the massaged value back to the attribute field + setattr(self, name, value) + + except (TypeError, ValueError), e: + raise AnsibleParserError("the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds()) + except UndefinedError, e: + if fail_on_undefined: + raise AnsibleParserError("the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name,e), obj=self.get_ds()) + + def serialize(self): + ''' + Serializes the object derived from the base object into + a dictionary of values. This only serializes the field + attributes for the object, so this may need to be overridden + for any classes which wish to add additional items not stored + as field attributes. + ''' + + #debug("starting serialization of %s" % self.__class__.__name__) + repr = dict() + + for (name, attribute) in iteritems(self._get_base_attributes()): + repr[name] = getattr(self, name) + + #debug("done serializing %s" % self.__class__.__name__) + return repr + + def deserialize(self, data): + ''' + Given a dictionary of values, load up the field attributes for + this object. As with serialize(), if there are any non-field + attribute data members, this method will need to be overridden + and extended. + ''' + + #debug("starting deserialization of %s" % self.__class__.__name__) + assert isinstance(data, dict) + + for (name, attribute) in iteritems(self._get_base_attributes()): + if name in data: + setattr(self, name, data[name]) + else: + setattr(self, name, attribute.default) + #debug("done deserializing %s" % self.__class__.__name__) def __getattr__(self, needle): @@ -146,3 +272,11 @@ class Base: return self._attributes[needle] raise AttributeError("attribute not found: %s" % needle) + + def __getstate__(self): + return self.serialize() + + def __setstate__(self, data): + self.__init__() + self.deserialize(data) + diff --git a/v2/ansible/playbook/block.py b/v2/ansible/playbook/block.py index 0fc19113f0..1e62aa0c98 100644 --- a/v2/ansible/playbook/block.py +++ b/v2/ansible/playbook/block.py @@ -21,36 +21,49 @@ __metaclass__ = type from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base +from ansible.playbook.conditional import Conditional from ansible.playbook.helpers import load_list_of_tasks +from ansible.playbook.role import Role +from ansible.playbook.taggable import Taggable from ansible.playbook.task_include import TaskInclude -class Block(Base): +class Block(Base, Conditional, Taggable): - _block = FieldAttribute(isa='list') - _rescue = FieldAttribute(isa='list') - _always = FieldAttribute(isa='list') - _tags = FieldAttribute(isa='list', default=[]) - _when = FieldAttribute(isa='list', default=[]) + _block = FieldAttribute(isa='list') + _rescue = FieldAttribute(isa='list') + _always = FieldAttribute(isa='list') # for future consideration? this would be functionally # similar to the 'else' clause for exceptions #_otherwise = FieldAttribute(isa='list') - def __init__(self, parent_block=None, role=None, task_include=None): + def __init__(self, parent_block=None, role=None, task_include=None, use_handlers=False): self._parent_block = parent_block - self._role = role + self._role = role self._task_include = task_include + self._use_handlers = use_handlers + super(Block, self).__init__() - def get_variables(self): - # blocks do not (currently) store any variables directly, - # so we just return an empty dict here - return dict() + def get_vars(self): + ''' + Blocks do not store variables directly, however they may be a member + of a role or task include which does, so return those if present. + ''' + + all_vars = dict() + + if self._role: + all_vars.update(self._role.get_vars()) + if self._task_include: + all_vars.update(self._task_include.get_vars()) + + return all_vars @staticmethod - def load(data, parent_block=None, role=None, task_include=None, loader=None): - b = Block(parent_block=parent_block, role=role, task_include=task_include) - return b.load_data(data, loader=loader) + def load(data, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None): + b = Block(parent_block=parent_block, role=role, task_include=task_include, use_handlers=use_handlers) + return b.load_data(data, variable_manager=variable_manager, loader=loader) def munge(self, ds): ''' @@ -70,17 +83,49 @@ class Block(Base): return ds def _load_block(self, attr, ds): - return load_list_of_tasks(ds, block=self, loader=self._loader) + return load_list_of_tasks( + ds, + block=self, + role=self._role, + task_include=self._task_include, + variable_manager=self._variable_manager, + loader=self._loader, + use_handlers=self._use_handlers, + ) def _load_rescue(self, attr, ds): - return load_list_of_tasks(ds, block=self, loader=self._loader) + return load_list_of_tasks( + ds, + block=self, + role=self._role, + task_include=self._task_include, + variable_manager=self._variable_manager, + loader=self._loader, + use_handlers=self._use_handlers, + ) def _load_always(self, attr, ds): - return load_list_of_tasks(ds, block=self, loader=self._loader) + return load_list_of_tasks( + ds, + block=self, + role=self._role, + task_include=self._task_include, + variable_manager=self._variable_manager, + loader=self._loader, + use_handlers=self._use_handlers, + ) # not currently used #def _load_otherwise(self, attr, ds): - # return self._load_list_of_tasks(ds, block=self, loader=self._loader) + # return load_list_of_tasks( + # ds, + # block=self, + # role=self._role, + # task_include=self._task_include, + # variable_manager=self._variable_manager, + # loader=self._loader, + # use_handlers=self._use_handlers, + # ) def compile(self): ''' @@ -93,3 +138,92 @@ class Block(Base): task_list.extend(task.compile()) return task_list + + def copy(self): + new_me = super(Block, self).copy() + new_me._use_handlers = self._use_handlers + + new_me._parent_block = None + if self._parent_block: + new_me._parent_block = self._parent_block.copy() + + new_me._role = None + if self._role: + new_me._role = self._role + + new_me._task_include = None + if self._task_include: + new_me._task_include = self._task_include.copy() + + return new_me + + def serialize(self): + ''' + Override of the default serialize method, since when we're serializing + a task we don't want to include the attribute list of tasks. + ''' + + data = dict(when=self.when) + + if self._role is not None: + data['role'] = self._role.serialize() + if self._task_include is not None: + data['task_include'] = self._task_include.serialize() + + return data + + def deserialize(self, data): + ''' + Override of the default deserialize method, to match the above overridden + serialize method + ''' + + from ansible.playbook.task_include import TaskInclude + + # unpack the when attribute, which is the only one we want + self.when = data.get('when') + + # if there was a serialized role, unpack it too + role_data = data.get('role') + if role_data: + r = Role() + r.deserialize(role_data) + self._role = r + + # if there was a serialized task include, unpack it too + ti_data = data.get('task_include') + if ti_data: + ti = TaskInclude() + ti.deserialize(ti_data) + self._task_include = ti + + def evaluate_conditional(self, all_vars): + if self._task_include is not None: + if not self._task_include.evaluate_conditional(all_vars): + return False + if self._parent_block is not None: + if not self._parent_block.evaluate_conditional(all_vars): + return False + elif self._role is not None: + if not self._role.evaluate_conditional(all_vars): + return False + return super(Block, self).evaluate_conditional(all_vars) + + def evaluate_tags(self, only_tags, skip_tags, all_vars): + result = False + if self._parent_block is not None: + result |= self._parent_block.evaluate_tags(only_tags=only_tags, skip_tags=skip_tags, all_vars=all_vars) + elif self._role is not None: + result |= self._role.evaluate_tags(only_tags=only_tags, skip_tags=skip_tags, all_vars=all_vars) + return result | super(Block, self).evaluate_tags(only_tags=only_tags, skip_tags=skip_tags, all_vars=all_vars) + + def set_loader(self, loader): + self._loader = loader + if self._parent_block: + self._parent_block.set_loader(loader) + elif self._role: + self._role.set_loader(loader) + + if self._task_include: + self._task_include.set_loader(loader) + diff --git a/v2/ansible/playbook/conditional.py b/v2/ansible/playbook/conditional.py index b921d4191a..2d8db78bba 100644 --- a/v2/ansible/playbook/conditional.py +++ b/v2/ansible/playbook/conditional.py @@ -19,16 +19,85 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible.errors import * +from ansible.playbook.attribute import FieldAttribute +from ansible.template import Templar + class Conditional: - def __init__(self, task): - self._task = task - self._conditionals = [] + ''' + This is a mix-in class, to be used with Base to allow the object + to be run conditionally when a condition is met or skipped. + ''' + + _when = FieldAttribute(isa='list', default=[]) + + def __init__(self, loader=None): + # when used directly, this class needs a loader, but we want to + # make sure we don't trample on the existing one if this class + # is used as a mix-in with a playbook base class + if not hasattr(self, '_loader'): + if loader is None: + raise AnsibleError("a loader must be specified when using Conditional() directly") + else: + self._loader = loader + super(Conditional, self).__init__() + + def _validate_when(self, attr, name, value): + if not isinstance(value, list): + setattr(self, name, [ value ]) + + def evaluate_conditional(self, all_vars): + ''' + Loops through the conditionals set on this object, returning + False if any of them evaluate as such. + ''' + + templar = Templar(loader=self._loader, variables=all_vars, fail_on_undefined=False) + for conditional in self.when: + if not self._check_conditional(conditional, templar, all_vars): + return False + return True + + def _check_conditional(self, conditional, templar, all_vars): + ''' + This method does the low-level evaluation of each conditional + set on this object, using jinja2 to wrap the conditionals for + evaluation. + ''' + + if conditional is None or conditional == '': + return True + + # FIXME: this should be removable now, leaving it here just in case + # allow variable names + #if conditional in all_vars and '-' not in str(all_vars[conditional]): + # conditional = all_vars[conditional] + + conditional = templar.template(conditional, convert_bare=True) + if not isinstance(conditional, basestring) or conditional == "": + return conditional + + # a Jinja2 evaluation that results in something Python can eval! + presented = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % conditional + conditional = templar.template(presented) - def evaluate(self, context): - pass + val = conditional.strip() + if val == presented: + # the templating failed, meaning most likely a + # variable was undefined. If we happened to be + # looking for an undefined variable, return True, + # otherwise fail + if "is undefined" in conditional: + return True + elif "is defined" in conditional: + return False + else: + raise AnsibleError("error while evaluating conditional: %s" % original) + elif val == "True": + return True + elif val == "False": + return False + else: + raise AnsibleError("unable to evaluate conditional: %s" % original) - def push(self, conditionals): - if not isinstance(conditionals, list): - conditionals = [ conditionals ] - self._conditionals.extend(conditionals) diff --git a/v2/ansible/playbook/handler.py b/v2/ansible/playbook/handler.py index 1f7aa957a5..c8c1572e48 100644 --- a/v2/ansible/playbook/handler.py +++ b/v2/ansible/playbook/handler.py @@ -19,22 +19,35 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from v2.errors import AnsibleError -from v2.inventory import Host -from v2.playbook import Task +from ansible.errors import AnsibleError +#from ansible.inventory.host import Host +from ansible.playbook.task import Task class Handler(Task): - def __init__(self): - pass + def __init__(self, block=None, role=None, task_include=None): + self._flagged_hosts = [] + + super(Handler, self).__init__(block=block, role=role, task_include=task_include) + + def __repr__(self): + ''' returns a human readable representation of the handler ''' + return "HANDLER: %s" % self.get_name() + + @staticmethod + def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None): + t = Handler(block=block, role=role, task_include=task_include) + return t.load_data(data, variable_manager=variable_manager, loader=loader) def flag_for_host(self, host): - assert instanceof(host, Host) - pass + #assert instanceof(host, Host) + if host not in self._flagged_hosts: + self._flagged_hosts.append(host) - def has_triggered(self): - return self._triggered + def has_triggered(self, host): + return host in self._flagged_hosts - def set_triggered(self, triggered): - assert instanceof(triggered, bool) - self._triggered = triggered + def serialize(self): + result = super(Handler, self).serialize() + result['is_handler'] = True + return result diff --git a/v2/ansible/playbook/helpers.py b/v2/ansible/playbook/helpers.py index 1d79721dce..3b6d59d019 100644 --- a/v2/ansible/playbook/helpers.py +++ b/v2/ansible/playbook/helpers.py @@ -24,13 +24,13 @@ from ansible.errors import AnsibleParserError from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject -def load_list_of_blocks(ds, parent_block=None, role=None, task_include=None, loader=None): +def load_list_of_blocks(ds, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None): ''' Given a list of mixed task/block data (parsed from YAML), return a list of Block() objects, where implicit blocks are created for each bare Task. ''' - + # we import here to prevent a circular dependency with imports from ansible.playbook.block import Block @@ -39,19 +39,28 @@ def load_list_of_blocks(ds, parent_block=None, role=None, task_include=None, loa block_list = [] if ds: for block in ds: - b = Block.load(block, parent_block=parent_block, role=role, task_include=task_include, loader=loader) + b = Block.load( + block, + parent_block=parent_block, + role=role, + task_include=task_include, + use_handlers=use_handlers, + variable_manager=variable_manager, + loader=loader + ) block_list.append(b) return block_list -def load_list_of_tasks(ds, block=None, role=None, task_include=None, loader=None): +def load_list_of_tasks(ds, block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None): ''' Given a list of task datastructures (parsed from YAML), return a list of Task() or TaskInclude() objects. ''' # we import here to prevent a circular dependency with imports + from ansible.playbook.handler import Handler from ansible.playbook.task import Task from ansible.playbook.task_include import TaskInclude @@ -70,19 +79,29 @@ def load_list_of_tasks(ds, block=None, role=None, task_include=None, loader=None cur_basedir = loader.get_basedir() loader.set_basedir(new_basedir) - t = TaskInclude.load(task, block=block, role=role, task_include=task_include, loader=loader) + t = TaskInclude.load( + task, + block=block, + role=role, + task_include=task_include, + use_handlers=use_handlers, + loader=loader + ) if cur_basedir and loader: loader.set_basedir(cur_basedir) else: - t = Task.load(task, block=block, role=role, task_include=task_include, loader=loader) + if use_handlers: + t = Handler.load(task, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader) + else: + t = Task.load(task, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader) task_list.append(t) return task_list -def load_list_of_roles(ds, loader=None): +def load_list_of_roles(ds, current_role_path=None, variable_manager=None, loader=None): ''' Loads and returns a list of RoleInclude objects from the datastructure list of role definitions @@ -95,7 +114,7 @@ def load_list_of_roles(ds, loader=None): roles = [] for role_def in ds: - i = RoleInclude.load(role_def, loader=loader) + i = RoleInclude.load(role_def, current_role_path=current_role_path, variable_manager=variable_manager, loader=loader) roles.append(i) return roles diff --git a/v2/ansible/playbook/play.py b/v2/ansible/playbook/play.py index 6dd92ffba0..3de550b23c 100644 --- a/v2/ansible/playbook/play.py +++ b/v2/ansible/playbook/play.py @@ -21,18 +21,19 @@ __metaclass__ = type from ansible.errors import AnsibleError, AnsibleParserError -from ansible.parsing.yaml import DataLoader - from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base from ansible.playbook.helpers import load_list_of_blocks, load_list_of_roles, compile_block_list from ansible.playbook.role import Role +from ansible.playbook.taggable import Taggable + +from ansible.utils.vars import combine_vars __all__ = ['Play'] -class Play(Base): +class Play(Base, Taggable): """ A play is a language feature that represents a list of roles and/or @@ -51,15 +52,16 @@ class Play(Base): _accelerate_port = FieldAttribute(isa='int', default=5099) _connection = FieldAttribute(isa='string', default='smart') _gather_facts = FieldAttribute(isa='string', default='smart') - _hosts = FieldAttribute(isa='list', default=[]) + _hosts = FieldAttribute(isa='list', default=[], required=True) _name = FieldAttribute(isa='string', default='<no name specified>') _port = FieldAttribute(isa='int', default=22) _remote_user = FieldAttribute(isa='string', default='root') _su = FieldAttribute(isa='bool', default=False) _su_user = FieldAttribute(isa='string', default='root') + _su_pass = FieldAttribute(isa='string') _sudo = FieldAttribute(isa='bool', default=False) _sudo_user = FieldAttribute(isa='string', default='root') - _tags = FieldAttribute(isa='list', default=[]) + _sudo_pass = FieldAttribute(isa='string') # Variable Attributes _vars = FieldAttribute(isa='dict', default=dict()) @@ -78,9 +80,11 @@ class Play(Base): # Flag/Setting Attributes _any_errors_fatal = FieldAttribute(isa='bool', default=False) + _environment = FieldAttribute(isa='dict', default=dict()) _max_fail_percentage = FieldAttribute(isa='string', default='0') _no_log = FieldAttribute(isa='bool', default=False) _serial = FieldAttribute(isa='int', default=0) + _strategy = FieldAttribute(isa='string', default='linear') # ================================================================================= @@ -95,9 +99,9 @@ class Play(Base): return "PLAY: %s" % self._attributes.get('name') @staticmethod - def load(data, loader=None): + def load(data, variable_manager=None, loader=None): p = Play() - return p.load_data(data, loader=loader) + return p.load_data(data, variable_manager=variable_manager, loader=loader) def munge(self, ds): ''' @@ -120,40 +124,68 @@ class Play(Base): return ds + def _load_vars(self, attr, ds): + ''' + Vars in a play can be specified either as a dictionary directly, or + as a list of dictionaries. If the later, this method will turn the + list into a single dictionary. + ''' + + try: + if isinstance(ds, dict): + return ds + elif isinstance(ds, list): + all_vars = dict() + for item in ds: + if not isinstance(item, dict): + raise ValueError + all_vars = combine_vars(all_vars, item) + return all_vars + else: + raise ValueError + except ValueError: + raise AnsibleParsingError("Vars in a playbook must be specified as a dictionary, or a list of dictionaries", obj=ds) + def _load_tasks(self, attr, ds): ''' Loads a list of blocks from a list which may be mixed tasks/blocks. Bare tasks outside of a block are given an implicit block. ''' - return load_list_of_blocks(ds, loader=self._loader) + return load_list_of_blocks(ds, variable_manager=self._variable_manager, loader=self._loader) def _load_pre_tasks(self, attr, ds): ''' Loads a list of blocks from a list which may be mixed tasks/blocks. Bare tasks outside of a block are given an implicit block. ''' - return load_list_of_blocks(ds, loader=self._loader) + return load_list_of_blocks(ds, variable_manager=self._variable_manager, loader=self._loader) def _load_post_tasks(self, attr, ds): ''' Loads a list of blocks from a list which may be mixed tasks/blocks. Bare tasks outside of a block are given an implicit block. ''' - return load_list_of_blocks(ds, loader=self._loader) + return load_list_of_blocks(ds, variable_manager=self._variable_manager, loader=self._loader) def _load_handlers(self, attr, ds): ''' Loads a list of blocks from a list which may be mixed handlers/blocks. Bare handlers outside of a block are given an implicit block. ''' - return load_list_of_blocks(ds, loader=self._loader) + return load_list_of_blocks(ds, use_handlers=True, variable_manager=self._variable_manager, loader=self._loader) def _load_roles(self, attr, ds): ''' Loads and returns a list of RoleInclude objects from the datastructure - list of role definitions + list of role definitions and creates the Role from those objects ''' - return load_list_of_roles(ds, loader=self._loader) + + role_includes = load_list_of_roles(ds, variable_manager=self._variable_manager, loader=self._loader) + + roles = [] + for ri in role_includes: + roles.append(Role.load(ri)) + return roles # FIXME: post_validation needs to ensure that su/sudo are not both set @@ -169,13 +201,8 @@ class Play(Base): task_list = [] if len(self.roles) > 0: - for ri in self.roles: - # The internal list of roles are actually RoleInclude objects, - # so we load the role from that now - role = Role.load(ri) - - # FIXME: evauluate conditional of roles here? - task_list.extend(role.compile()) + for r in self.roles: + task_list.extend(r.compile()) return task_list @@ -194,3 +221,40 @@ class Play(Base): task_list.extend(compile_block_list(self.post_tasks)) return task_list + + def get_vars(self): + return self.vars.copy() + + def get_vars_files(self): + return self.vars_files + + def get_handlers(self): + return self.handlers[:] + + def get_roles(self): + return self.roles[:] + + def serialize(self): + data = super(Play, self).serialize() + + roles = [] + for role in self.get_roles(): + roles.append(role.serialize()) + data['roles'] = roles + + return data + + def deserialize(self, data): + super(Play, self).deserialize(data) + + if 'roles' in data: + role_data = data.get('roles', []) + roles = [] + for role in role_data: + r = Role() + r.deserialize(role) + roles.append(r) + + setattr(self, 'roles', roles) + del data['roles'] + diff --git a/v2/ansible/playbook/playbook_include.py b/v2/ansible/playbook/playbook_include.py index ae8ccff595..159c3d25da 100644 --- a/v2/ansible/playbook/playbook_include.py +++ b/v2/ansible/playbook/playbook_include.py @@ -18,3 +18,108 @@ # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type + +import os + +from ansible.parsing.splitter import split_args, parse_kv +from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping +from ansible.playbook.attribute import FieldAttribute +from ansible.playbook.base import Base +from ansible.playbook.conditional import Conditional +from ansible.playbook.taggable import Taggable + +class PlaybookInclude(Base): + + _name = FieldAttribute(isa='string') + _include = FieldAttribute(isa='string') + _vars = FieldAttribute(isa='dict', default=dict()) + + @staticmethod + def load(data, basedir, variable_manager=None, loader=None): + return PlaybookInclude().load_data(ds=data, basedir=basedir, variable_manager=variable_manager, loader=loader) + + def load_data(self, ds, basedir, variable_manager=None, loader=None): + ''' + Overrides the base load_data(), as we're actually going to return a new + Playbook() object rather than a PlaybookInclude object + ''' + + # import here to avoid a dependency loop + from ansible.playbook import Playbook + + # first, we use the original parent method to correctly load the object + # via the munge/load_data system we normally use for other playbook objects + new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader) + + # then we use the object to load a Playbook + pb = Playbook(loader=loader) + + file_name = new_obj.include + if not os.path.isabs(file_name): + file_name = os.path.join(basedir, file_name) + + pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager) + + # finally, playbook includes can specify a list of variables, which are simply + # used to update the vars of each play in the playbook + for entry in pb._entries: + entry.vars.update(new_obj.vars) + + return pb + + def munge(self, ds): + ''' + Regorganizes the data for a PlaybookInclude datastructure to line + up with what we expect the proper attributes to be + ''' + + assert isinstance(ds, dict) + + # the new, cleaned datastructure, which will have legacy + # items reduced to a standard structure + new_ds = AnsibleMapping() + if isinstance(ds, AnsibleBaseYAMLObject): + new_ds.copy_position_info(ds) + + for (k,v) in ds.iteritems(): + if k == 'include': + self._munge_include(ds, new_ds, k, v) + elif k.replace("with_", "") in lookup_loader: + self._munge_loop(ds, new_ds, k, v) + else: + # some basic error checking, to make sure vars are properly + # formatted and do not conflict with k=v parameters + # FIXME: we could merge these instead, but controlling the order + # in which they're encountered could be difficult + if k == 'vars': + if 'vars' in new_ds: + raise AnsibleParserError("include parameters cannot be mixed with 'vars' entries for include statements", obj=ds) + elif not isinstance(v, dict): + raise AnsibleParserError("vars for include statements must be specified as a dictionary", obj=ds) + new_ds[k] = v + + return new_ds + + def _munge_include(self, ds, new_ds, k, v): + ''' + Splits the include line up into filename and parameters + ''' + + # The include line must include at least one item, which is the filename + # to include. Anything after that should be regarded as a parameter to the include + items = split_args(v) + if len(items) == 0: + raise AnsibleParserError("include statements must specify the file name to include", obj=ds) + else: + # FIXME/TODO: validate that items[0] is a file, which also + # exists and is readable + new_ds['include'] = items[0] + if len(items) > 1: + # rejoin the parameter portion of the arguments and + # then use parse_kv() to get a dict of params back + params = parse_kv(" ".join(items[1:])) + if 'vars' in new_ds: + # FIXME: see fixme above regarding merging vars + raise AnsibleParserError("include parameters cannot be mixed with 'vars' entries for include statements", obj=ds) + new_ds['vars'] = params + diff --git a/v2/ansible/playbook/role/__init__.py b/v2/ansible/playbook/role/__init__.py index ab8a779fde..0bdab79946 100644 --- a/v2/ansible/playbook/role/__init__.py +++ b/v2/ansible/playbook/role/__init__.py @@ -27,16 +27,40 @@ from hashlib import sha1 from types import NoneType from ansible.errors import AnsibleError, AnsibleParserError -from ansible.parsing.yaml import DataLoader +from ansible.parsing import DataLoader from ansible.playbook.attribute import FieldAttribute from ansible.playbook.base import Base +from ansible.playbook.conditional import Conditional from ansible.playbook.helpers import load_list_of_blocks, compile_block_list from ansible.playbook.role.include import RoleInclude from ansible.playbook.role.metadata import RoleMetadata - - -__all__ = ['Role', 'ROLE_CACHE'] - +from ansible.playbook.taggable import Taggable +from ansible.plugins import module_loader +from ansible.utils.vars import combine_vars + + +__all__ = ['Role', 'ROLE_CACHE', 'hash_params'] + +# FIXME: this should be a utility function, but can't be a member of +# the role due to the fact that it would require the use of self +# in a static method. This is also used in the base class for +# strategies (ansible/plugins/strategies/__init__.py) +def hash_params(params): + if not isinstance(params, dict): + return params + else: + s = set() + for k,v in params.iteritems(): + if isinstance(v, dict): + s.update((k, hash_params(v))) + elif isinstance(v, list): + things = [] + for item in v: + things.append(hash_params(item)) + s.update((k, tuple(things))) + else: + s.update((k, v)) + return frozenset(s) # The role cache is used to prevent re-loading roles, which # may already exist. Keys into this cache are the SHA1 hash @@ -45,7 +69,7 @@ __all__ = ['Role', 'ROLE_CACHE'] ROLE_CACHE = dict() -class Role: +class Role(Base, Conditional, Taggable): def __init__(self): self._role_name = None @@ -60,6 +84,10 @@ class Role: self._handler_blocks = [] self._default_vars = dict() self._role_vars = dict() + self._had_task_run = False + self._completed = False + + super(Role, self).__init__() def __repr__(self): return self.get_name() @@ -71,27 +99,60 @@ class Role: def load(role_include, parent_role=None): # FIXME: add back in the role caching support try: + # The ROLE_CACHE is a dictionary of role names, with each entry + # containing another dictionary corresponding to a set of parameters + # specified for a role as the key and the Role() object itself. + # We use frozenset to make the dictionary hashable. + + #hashed_params = frozenset(role_include.get_role_params().iteritems()) + hashed_params = hash_params(role_include.get_role_params()) + if role_include.role in ROLE_CACHE: + for (entry, role_obj) in ROLE_CACHE[role_include.role].iteritems(): + if hashed_params == entry: + if parent_role: + role_obj.add_parent(parent_role) + return role_obj + r = Role() r._load_role_data(role_include, parent_role=parent_role) + + if role_include.role not in ROLE_CACHE: + ROLE_CACHE[role_include.role] = dict() + + ROLE_CACHE[role_include.role][hashed_params] = r + return r + except RuntimeError: # FIXME: needs a better way to access the ds in the role include raise AnsibleError("A recursion loop was detected with the roles specified. Make sure child roles do not have dependencies on parent roles", obj=role_include._ds) - return r def _load_role_data(self, role_include, parent_role=None): - self._role_name = role_include.role - self._role_path = role_include.get_role_path() - self._role_params = role_include.get_role_params() - self._loader = role_include.get_loader() + self._role_name = role_include.role + self._role_path = role_include.get_role_path() + self._role_params = role_include.get_role_params() + self._variable_manager = role_include.get_variable_manager() + self._loader = role_include.get_loader() if parent_role: self.add_parent(parent_role) + current_when = getattr(self, 'when')[:] + current_when.extend(role_include.when) + setattr(self, 'when', current_when) + + current_tags = getattr(self, 'tags')[:] + current_tags.extend(role_include.tags) + setattr(self, 'tags', current_tags) + # save the current base directory for the loader and set it to the current role path - cur_basedir = self._loader.get_basedir() - self._loader.set_basedir(self._role_path) + #cur_basedir = self._loader.get_basedir() + #self._loader.set_basedir(self._role_path) # load the role's files, if they exist + library = os.path.join(self._role_path, 'library') + if os.path.isdir(library): + module_loader.add_directory(library) + metadata = self._load_role_yaml('meta') if metadata: self._metadata = RoleMetadata.load(metadata, owner=self, loader=self._loader) @@ -106,16 +167,20 @@ class Role: self._handler_blocks = load_list_of_blocks(handler_data, role=self, loader=self._loader) # vars and default vars are regular dictionaries - self._role_vars = self._load_role_yaml('vars') + self._role_vars = self._load_role_yaml('vars') if not isinstance(self._role_vars, (dict, NoneType)): raise AnsibleParserError("The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) + elif self._role_vars is None: + self._role_vars = dict() self._default_vars = self._load_role_yaml('defaults') if not isinstance(self._default_vars, (dict, NoneType)): raise AnsibleParserError("The default/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) + elif self._default_vars is None: + self._default_vars = dict() # and finally restore the previous base directory - self._loader.set_basedir(cur_basedir) + #self._loader.set_basedir(cur_basedir) def _load_role_yaml(self, subdir): file_path = os.path.join(self._role_path, subdir) @@ -169,29 +234,50 @@ class Role: def get_parents(self): return self._parents - # FIXME: not yet used - #def get_variables(self): - # # returns the merged variables for this role, including - # # recursively merging those of all child roles - # return dict() + def get_default_vars(self): + # FIXME: get these from dependent roles too + default_vars = dict() + for dep in self.get_all_dependencies(): + default_vars = combine_vars(default_vars, dep.get_default_vars()) + default_vars = combine_vars(default_vars, self._default_vars) + return default_vars + + def get_inherited_vars(self): + inherited_vars = dict() + for parent in self._parents: + inherited_vars = combine_vars(inherited_vars, parent.get_inherited_vars()) + inherited_vars = combine_vars(inherited_vars, parent._role_vars) + inherited_vars = combine_vars(inherited_vars, parent._role_params) + return inherited_vars + + def get_vars(self): + all_vars = self.get_inherited_vars() + + for dep in self.get_all_dependencies(): + all_vars = combine_vars(all_vars, dep.get_vars()) + + all_vars = combine_vars(all_vars, self._role_vars) + all_vars = combine_vars(all_vars, self._role_params) + + return all_vars def get_direct_dependencies(self): return self._dependencies[:] def get_all_dependencies(self): - # returns a list built recursively, of all deps from - # all child dependencies + ''' + Returns a list of all deps, built recursively from all child dependencies, + in the proper order in which they should be executed or evaluated. + ''' child_deps = [] - direct_deps = self.get_direct_dependencies() - for dep in direct_deps: - dep_deps = dep.get_all_dependencies() - for dep_dep in dep_deps: - if dep_dep not in child_deps: - child_deps.append(dep_dep) + for dep in self.get_direct_dependencies(): + for child_dep in dep.get_all_dependencies(): + child_deps.append(child_dep) + child_deps.append(dep) - return direct_deps + child_deps + return child_deps def get_task_blocks(self): return self._task_blocks[:] @@ -199,20 +285,109 @@ class Role: def get_handler_blocks(self): return self._handler_blocks[:] - def compile(self): + def has_run(self): + ''' + Returns true if this role has been iterated over completely and + at least one task was run + ''' + + return self._had_task_run and self._completed + + def compile(self, dep_chain=[]): ''' Returns the task list for this role, which is created by first recursively compiling the tasks for all direct dependencies, and then adding on the tasks for this role. + + The role compile() also remembers and saves the dependency chain + with each task, so tasks know by which route they were found, and + can correctly take their parent's tags/conditionals into account. ''' task_list = [] + # update the dependency chain here + new_dep_chain = dep_chain + [self] + deps = self.get_direct_dependencies() for dep in deps: - task_list.extend(dep.compile()) + dep_tasks = dep.compile(dep_chain=new_dep_chain) + for dep_task in dep_tasks: + # since we're modifying the task, and need it to be unique, + # we make a copy of it here and assign the dependency chain + # to the copy, then append the copy to the task list. + new_dep_task = dep_task.copy() + new_dep_task._dep_chain = new_dep_chain + task_list.append(new_dep_task) task_list.extend(compile_block_list(self._task_blocks)) return task_list + def serialize(self, include_deps=True): + res = super(Role, self).serialize() + + res['_role_name'] = self._role_name + res['_role_path'] = self._role_path + res['_role_vars'] = self._role_vars + res['_role_params'] = self._role_params + res['_default_vars'] = self._default_vars + res['_had_task_run'] = self._had_task_run + res['_completed'] = self._completed + + if self._metadata: + res['_metadata'] = self._metadata.serialize() + + if include_deps: + deps = [] + for role in self.get_direct_dependencies(): + deps.append(role.serialize()) + res['_dependencies'] = deps + + parents = [] + for parent in self._parents: + parents.append(parent.serialize(include_deps=False)) + res['_parents'] = parents + + return res + + def deserialize(self, data, include_deps=True): + self._role_name = data.get('_role_name', '') + self._role_path = data.get('_role_path', '') + self._role_vars = data.get('_role_vars', dict()) + self._role_params = data.get('_role_params', dict()) + self._default_vars = data.get('_default_vars', dict()) + self._had_task_run = data.get('_had_task_run', False) + self._completed = data.get('_completed', False) + + if include_deps: + deps = [] + for dep in data.get('_dependencies', []): + r = Role() + r.deserialize(dep) + deps.append(r) + setattr(self, '_dependencies', deps) + + parent_data = data.get('_parents', []) + parents = [] + for parent in parent_data: + r = Role() + r.deserialize(parent, include_deps=False) + parents.append(r) + setattr(self, '_parents', parents) + + metadata_data = data.get('_metadata') + if metadata_data: + m = RoleMetadata() + m.deserialize(metadata_data) + self._metadata = m + + super(Role, self).deserialize(data) + + def set_loader(self, loader): + self._loader = loader + for parent in self._parents: + parent.set_loader(loader) + for dep in self.get_direct_dependencies(): + dep.set_loader(loader) + diff --git a/v2/ansible/playbook/role/definition.py b/v2/ansible/playbook/role/definition.py index 34b0248820..c9ec4259c1 100644 --- a/v2/ansible/playbook/role/definition.py +++ b/v2/ansible/playbook/role/definition.py @@ -23,29 +23,34 @@ from six import iteritems, string_types import os +from ansible import constants as C from ansible.errors import AnsibleError from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base +from ansible.playbook.conditional import Conditional +from ansible.playbook.taggable import Taggable +from ansible.utils.path import unfrackpath __all__ = ['RoleDefinition'] -class RoleDefinition(Base): +class RoleDefinition(Base, Conditional, Taggable): _role = FieldAttribute(isa='string') - def __init__(self): - self._role_path = None - self._role_params = dict() + def __init__(self, role_basedir=None): + self._role_path = None + self._role_basedir = role_basedir + self._role_params = dict() super(RoleDefinition, self).__init__() - def __repr__(self): - return 'ROLEDEF: ' + self._attributes.get('role', '<no name set>') + #def __repr__(self): + # return 'ROLEDEF: ' + self._attributes.get('role', '<no name set>') @staticmethod - def load(data, loader=None): + def load(data, variable_manager=None, loader=None): raise AnsibleError("not implemented") def munge(self, ds): @@ -109,21 +114,33 @@ class RoleDefinition(Base): append it to the default role path ''' - # FIXME: this should use unfrackpath once the utils code has been sorted out - role_path = os.path.normpath(role_name) + role_path = unfrackpath(role_name) + if self._loader.path_exists(role_path): role_name = os.path.basename(role_name) return (role_name, role_path) else: - # FIXME: this should search in the configured roles path - for path in ('./roles', '/etc/ansible/roles'): - role_path = os.path.join(path, role_name) + # we always start the search for roles in the base directory of the playbook + role_search_paths = [os.path.join(self._loader.get_basedir(), 'roles'), './roles', './'] + + # also search in the configured roles path + if C.DEFAULT_ROLES_PATH: + configured_paths = C.DEFAULT_ROLES_PATH.split(os.pathsep) + role_search_paths.extend(configured_paths) + + # finally, append the roles basedir, if it was set, so we can + # search relative to that directory for dependent roles + if self._role_basedir: + role_search_paths.append(self._role_basedir) + + # now iterate through the possible paths and return the first one we find + for path in role_search_paths: + role_path = unfrackpath(os.path.join(path, role_name)) if self._loader.path_exists(role_path): return (role_name, role_path) - # FIXME: make the parser smart about list/string entries - # in the yaml so the error line/file can be reported - # here + # FIXME: make the parser smart about list/string entries in + # the yaml so the error line/file can be reported here raise AnsibleError("the role '%s' was not found" % role_name) diff --git a/v2/ansible/playbook/role/include.py b/v2/ansible/playbook/role/include.py index d36b0a9397..b063aecc35 100644 --- a/v2/ansible/playbook/role/include.py +++ b/v2/ansible/playbook/role/include.py @@ -37,16 +37,13 @@ class RoleInclude(RoleDefinition): FIXME: docstring """ - _tags = FieldAttribute(isa='list', default=[]) - _when = FieldAttribute(isa='list', default=[]) - - def __init__(self): - super(RoleInclude, self).__init__() + def __init__(self, role_basedir=None): + super(RoleInclude, self).__init__(role_basedir=role_basedir) @staticmethod - def load(data, parent_role=None, loader=None): + def load(data, current_role_path=None, parent_role=None, variable_manager=None, loader=None): assert isinstance(data, string_types) or isinstance(data, dict) - ri = RoleInclude() - return ri.load_data(data, loader=loader) + ri = RoleInclude(role_basedir=current_role_path) + return ri.load_data(data, variable_manager=variable_manager, loader=loader) diff --git a/v2/ansible/playbook/role/metadata.py b/v2/ansible/playbook/role/metadata.py index 19b0f01f62..05ed2f3585 100644 --- a/v2/ansible/playbook/role/metadata.py +++ b/v2/ansible/playbook/role/metadata.py @@ -19,6 +19,8 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import os + from six import iteritems, string_types from ansible.errors import AnsibleParserError @@ -41,12 +43,12 @@ class RoleMetadata(Base): _dependencies = FieldAttribute(isa='list', default=[]) _galaxy_info = FieldAttribute(isa='GalaxyInfo') - def __init__(self): - self._owner = None + def __init__(self, owner=None): + self._owner = owner super(RoleMetadata, self).__init__() @staticmethod - def load(data, owner, loader=None): + def load(data, owner, variable_manager=None, loader=None): ''' Returns a new RoleMetadata object based on the datastructure passed in. ''' @@ -54,7 +56,7 @@ class RoleMetadata(Base): if not isinstance(data, dict): raise AnsibleParserError("the 'meta/main.yml' for role %s is not a dictionary" % owner.get_name()) - m = RoleMetadata().load_data(data, loader=loader) + m = RoleMetadata(owner=owner).load_data(data, variable_manager=variable_manager, loader=loader) return m def _load_dependencies(self, attr, ds): @@ -62,7 +64,12 @@ class RoleMetadata(Base): This is a helper loading function for the dependencies list, which returns a list of RoleInclude objects ''' - return load_list_of_roles(ds, loader=self._loader) + + current_role_path = None + if self._owner: + current_role_path = os.path.dirname(self._owner._role_path) + + return load_list_of_roles(ds, current_role_path=current_role_path, variable_manager=self._variable_manager, loader=self._loader) def _load_galaxy_info(self, attr, ds): ''' @@ -72,3 +79,13 @@ class RoleMetadata(Base): ''' return ds + + def serialize(self): + return dict( + allow_duplicates = self.allow_duplicates, + dependencies = self.dependencies, + ) + + def deserialize(self, data): + setattr(self, 'allow_duplicates', data.get('allow_duplicates', False)) + setattr(self, 'dependencies', data.get('dependencies', [])) diff --git a/v2/ansible/playbook/tag.py b/v2/ansible/playbook/tag.py deleted file mode 100644 index cb3e3f9291..0000000000 --- a/v2/ansible/playbook/tag.py +++ /dev/null @@ -1,55 +0,0 @@ -# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see <http://www.gnu.org/licenses/>. - -# Make coding more python3-ish -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -from errors import AnsibleError -from ansible.utils import list_union - -class Tag: - def __init__(self, tags=[]): - assert isinstance(tags, list) - self._tags = tags - - def push(self, tags): - if not isinstance(tags, list): - tags = [ tags ] - for tag in tags: - if not isinstance(tag, basestring): - tag = str(tag) - if tag not in self._tags: - self._tags.append(tag) - - def get_tags(self): - return self._tags - - def merge(self, tags): - # returns a union of the tags, which can be a string, - # a list of strings, or another Tag() class - if isinstance(tags, basestring): - tags = Tag([tags]) - elif isinstance(tags, list): - tags = Tag(tags) - elif not isinstance(tags, Tag): - raise AnsibleError('expected a Tag() instance, instead got %s' % type(tags)) - return utils.list_union(self._tags, tags.get_tags()) - - def matches(self, tag): - return tag in self._tags - diff --git a/v2/ansible/playbook/taggable.py b/v2/ansible/playbook/taggable.py new file mode 100644 index 0000000000..e83f1d7ae5 --- /dev/null +++ b/v2/ansible/playbook/taggable.py @@ -0,0 +1,59 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleError +from ansible.playbook.attribute import FieldAttribute +from ansible.template import Templar + +class Taggable: + _tags = FieldAttribute(isa='list', default=[]) + + def __init__(self): + super(Taggable, self).__init__() + + def _load_tags(self, attr, ds): + if isinstance(ds, list): + return ds + elif isinstance(ds, basestring): + return [ ds ] + else: + raise AnsibleError('tags must be specified as a list', obj=ds) + + def evaluate_tags(self, only_tags, skip_tags, all_vars): + templar = Templar(loader=self._loader, variables=all_vars) + tags = templar.template(self.tags) + if not isinstance(tags, list): + tags = set([tags]) + else: + tags = set(tags) + + #print("%s tags are: %s, only_tags=%s, skip_tags=%s" % (self, my_tags, only_tags, skip_tags)) + if skip_tags: + skipped_tags = tags.intersection(skip_tags) + if len(skipped_tags) > 0: + return False + matched_tags = tags.intersection(only_tags) + #print("matched tags are: %s" % matched_tags) + if len(matched_tags) > 0 or 'all' in only_tags: + return True + else: + return False + diff --git a/v2/ansible/playbook/task.py b/v2/ansible/playbook/task.py index c4c22025ed..df91961555 100644 --- a/v2/ansible/playbook/task.py +++ b/v2/ansible/playbook/task.py @@ -19,18 +19,23 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.playbook.base import Base -from ansible.playbook.attribute import Attribute, FieldAttribute - from ansible.errors import AnsibleError -from ansible.parsing.splitter import parse_kv from ansible.parsing.mod_args import ModuleArgsParser -from ansible.parsing.yaml import DataLoader +from ansible.parsing.splitter import parse_kv from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping -from ansible.plugins import module_finder, lookup_finder -class Task(Base): +from ansible.plugins import module_loader, lookup_loader + +from ansible.playbook.attribute import Attribute, FieldAttribute +from ansible.playbook.base import Base +from ansible.playbook.block import Block +from ansible.playbook.conditional import Conditional +from ansible.playbook.role import Role +from ansible.playbook.taggable import Taggable +from ansible.playbook.task_include import TaskInclude + +class Task(Base, Conditional, Taggable): """ A task is a language feature that represents a call to a module, with given arguments and other parameters. @@ -49,15 +54,15 @@ class Task(Base): # will be used if defined # might be possible to define others - _args = FieldAttribute(isa='dict') + _args = FieldAttribute(isa='dict', default=dict()) _action = FieldAttribute(isa='string') _always_run = FieldAttribute(isa='bool') _any_errors_fatal = FieldAttribute(isa='bool') - _async = FieldAttribute(isa='int') + _async = FieldAttribute(isa='int', default=0) _changed_when = FieldAttribute(isa='string') _connection = FieldAttribute(isa='string') - _delay = FieldAttribute(isa='int') + _delay = FieldAttribute(isa='int', default=5) _delegate_to = FieldAttribute(isa='string') _environment = FieldAttribute(isa='dict') _failed_when = FieldAttribute(isa='string') @@ -75,10 +80,10 @@ class Task(Base): _no_log = FieldAttribute(isa='bool') _notify = FieldAttribute(isa='list') - _poll = FieldAttribute(isa='integer') + _poll = FieldAttribute(isa='int') _register = FieldAttribute(isa='string') _remote_user = FieldAttribute(isa='string') - _retries = FieldAttribute(isa='integer') + _retries = FieldAttribute(isa='int', default=1) _run_once = FieldAttribute(isa='bool') _su = FieldAttribute(isa='bool') _su_pass = FieldAttribute(isa='string') @@ -86,10 +91,8 @@ class Task(Base): _sudo = FieldAttribute(isa='bool') _sudo_user = FieldAttribute(isa='string') _sudo_pass = FieldAttribute(isa='string') - _tags = FieldAttribute(isa='list', default=[]) _transport = FieldAttribute(isa='string') _until = FieldAttribute(isa='list') # ? - _when = FieldAttribute(isa='list', default=[]) def __init__(self, block=None, role=None, task_include=None): ''' constructors a task, without the Task.load classmethod, it will be pretty blank ''' @@ -97,6 +100,7 @@ class Task(Base): self._block = block self._role = role self._task_include = task_include + self._dep_chain = [] super(Task, self).__init__() @@ -104,12 +108,15 @@ class Task(Base): ''' return the name of the task ''' if self._role and self.name: - return "%s : %s" % (self._role.name, self.name) + return "%s : %s" % (self._role.get_name(), self.name) elif self.name: return self.name else: flattened_args = self._merge_kv(self.args) - return "%s %s" % (self.action, flattened_args) + if self._role: + return "%s : %s %s" % (self._role.get_name(), self.action, flattened_args) + else: + return "%s %s" % (self.action, flattened_args) def _merge_kv(self, ds): if ds is None: @@ -126,9 +133,9 @@ class Task(Base): return buf @staticmethod - def load(data, block=None, role=None, task_include=None, loader=None): + def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None): t = Task(block=block, role=role, task_include=task_include) - return t.load_data(data, loader=loader) + return t.load_data(data, variable_manager=variable_manager, loader=loader) def __repr__(self): ''' returns a human readable representation of the task ''' @@ -173,13 +180,39 @@ class Task(Base): # we don't want to re-assign these values, which were # determined by the ModuleArgsParser() above continue - elif k.replace("with_", "") in lookup_finder: + elif k.replace("with_", "") in lookup_loader: self._munge_loop(ds, new_ds, k, v) else: new_ds[k] = v return new_ds + def post_validate(self, all_vars=dict(), fail_on_undefined=True): + ''' + Override of base class post_validate, to also do final validation on + the block and task include (if any) to which this task belongs. + ''' + + if self._block: + self._block.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) + if self._task_include: + self._task_include.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) + + super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) + + def get_vars(self): + all_vars = dict() + if self._task_include: + all_vars.update(self._task_include.get_vars()) + + all_vars.update(self.serialize()) + + if 'tags' in all_vars: + del all_vars['tags'] + if 'when' in all_vars: + del all_vars['when'] + return all_vars + def compile(self): ''' For tasks, this is just a dummy method returning an array @@ -188,3 +221,101 @@ class Task(Base): ''' return [self] + + def copy(self): + new_me = super(Task, self).copy() + new_me._dep_chain = self._dep_chain[:] + + new_me._block = None + if self._block: + new_me._block = self._block.copy() + + new_me._role = None + if self._role: + new_me._role = self._role + + new_me._task_include = None + if self._task_include: + new_me._task_include = self._task_include.copy() + + return new_me + + def serialize(self): + data = super(Task, self).serialize() + data['dep_chain'] = self._dep_chain + + if self._block: + data['block'] = self._block.serialize() + + if self._role: + data['role'] = self._role.serialize() + + if self._task_include: + data['task_include'] = self._task_include.serialize() + + return data + + def deserialize(self, data): + block_data = data.get('block') + self._dep_chain = data.get('dep_chain', []) + + if block_data: + b = Block() + b.deserialize(block_data) + self._block = b + del data['block'] + + role_data = data.get('role') + if role_data: + r = Role() + r.deserialize(role_data) + self._role = r + del data['role'] + + ti_data = data.get('task_include') + if ti_data: + ti = TaskInclude() + ti.deserialize(ti_data) + self._task_include = ti + del data['task_include'] + + super(Task, self).deserialize(data) + + def evaluate_conditional(self, all_vars): + if len(self._dep_chain): + for dep in self._dep_chain: + if not dep.evaluate_conditional(all_vars): + return False + if self._block is not None: + if not self._block.evaluate_conditional(all_vars): + return False + if self._task_include is not None: + if not self._task_include.evaluate_conditional(all_vars): + return False + return super(Task, self).evaluate_conditional(all_vars) + + def evaluate_tags(self, only_tags, skip_tags, all_vars): + result = False + if len(self._dep_chain): + for dep in self._dep_chain: + result |= dep.evaluate_tags(only_tags=only_tags, skip_tags=skip_tags, all_vars=all_vars) + if self._block is not None: + result |= self._block.evaluate_tags(only_tags=only_tags, skip_tags=skip_tags, all_vars=all_vars) + return result | super(Task, self).evaluate_tags(only_tags=only_tags, skip_tags=skip_tags, all_vars=all_vars) + + def set_loader(self, loader): + ''' + Sets the loader on this object and recursively on parent, child objects. + This is used primarily after the Task has been serialized/deserialized, which + does not preserve the loader. + ''' + + self._loader = loader + + if self._block: + self._block.set_loader(loader) + if self._task_include: + self._task_include.set_loader(loader) + + for dep in self._dep_chain: + dep.set_loader(loader) diff --git a/v2/ansible/playbook/task_include.py b/v2/ansible/playbook/task_include.py index dbbc388f68..d7aba9e815 100644 --- a/v2/ansible/playbook/task_include.py +++ b/v2/ansible/playbook/task_include.py @@ -24,14 +24,16 @@ from ansible.parsing.splitter import split_args, parse_kv from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base +from ansible.playbook.conditional import Conditional from ansible.playbook.helpers import load_list_of_blocks, compile_block_list -from ansible.plugins import lookup_finder +from ansible.playbook.taggable import Taggable +from ansible.plugins import lookup_loader __all__ = ['TaskInclude'] -class TaskInclude(Base): +class TaskInclude(Base, Conditional, Taggable): ''' A class used to wrap the use of `include: /some/other/file.yml` @@ -49,26 +51,26 @@ class TaskInclude(Base): #----------------------------------------------------------------- # Attributes + _name = FieldAttribute(isa='string') _include = FieldAttribute(isa='string') _loop = FieldAttribute(isa='string', private=True) _loop_args = FieldAttribute(isa='list', private=True) - _tags = FieldAttribute(isa='list', default=[]) _vars = FieldAttribute(isa='dict', default=dict()) - _when = FieldAttribute(isa='list', default=[]) - def __init__(self, block=None, role=None, task_include=None): + def __init__(self, block=None, role=None, task_include=None, use_handlers=False): self._block = block self._role = role self._task_include = task_include + self._use_handlers = use_handlers self._task_blocks = [] super(TaskInclude, self).__init__() @staticmethod - def load(data, block=None, role=None, task_include=None, loader=None): - ti = TaskInclude(block=block, role=role, task_include=None) - return ti.load_data(data, loader=loader) + def load(data, block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None): + ti = TaskInclude(block=block, role=role, task_include=None, use_handlers=use_handlers) + return ti.load_data(data, variable_manager=variable_manager, loader=loader) def munge(self, ds): ''' @@ -87,7 +89,7 @@ class TaskInclude(Base): for (k,v) in ds.iteritems(): if k == 'include': self._munge_include(ds, new_ds, k, v) - elif k.replace("with_", "") in lookup_finder: + elif k.replace("with_", "") in lookup_loader: self._munge_loop(ds, new_ds, k, v) else: # some basic error checking, to make sure vars are properly @@ -144,12 +146,13 @@ class TaskInclude(Base): raise AnsibleParsingError("included task files must contain a list of tasks", obj=ds) self._task_blocks = load_list_of_blocks( - data, - parent_block=self._block, - task_include=self, - role=self._role, - loader=self._loader - ) + data, + parent_block=self._block, + task_include=self, + role=self._role, + use_handlers=self._use_handlers, + loader=self._loader + ) return ds def compile(self): @@ -161,3 +164,79 @@ class TaskInclude(Base): task_list.extend(compile_block_list(self._task_blocks)) return task_list + def get_vars(self): + ''' + Returns the vars for this task include, but also first merges in + those from any parent task include which may exist. + ''' + + all_vars = dict() + if self._task_include: + all_vars.update(self._task_include.get_vars()) + if self._block: + all_vars.update(self._block.get_vars()) + all_vars.update(self.vars) + return all_vars + + def serialize(self): + + data = super(TaskInclude, self).serialize() + + if self._block: + data['block'] = self._block.serialize() + + if self._role: + data['role'] = self._role.serialize() + + if self._task_include: + data['task_include'] = self._task_include.serialize() + + return data + + def deserialize(self, data): + + # import here to prevent circular importing issues + from ansible.playbook.block import Block + from ansible.playbook.role import Role + + block_data = data.get('block') + if block_data: + b = Block() + b.deserialize(block_data) + self._block = b + del data['block'] + + role_data = data.get('role') + if role_data: + r = Role() + r.deserialize(role_data) + self._role = r + del data['role'] + + ti_data = data.get('task_include') + if ti_data: + ti = TaskInclude() + ti.deserialize(ti_data) + self._task_include = ti + del data['task_include'] + + super(TaskInclude, self).deserialize(data) + + def evaluate_conditional(self, all_vars): + if self._task_include is not None: + if not self._task_include.evaluate_conditional(all_vars): + return False + if self._block is not None: + if not self._block.evaluate_conditional(all_vars): + return False + elif self._role is not None: + if not self._role.evaluate_conditional(all_vars): + return False + return super(TaskInclude, self).evaluate_conditional(all_vars) + + def set_loader(self, loader): + self._loader = loader + if self._block: + self._block.set_loader(loader) + elif self._task_include: + self._task_include.set_loader(loader) diff --git a/v2/ansible/plugins/__init__.py b/v2/ansible/plugins/__init__.py index 5ab704b8a1..2125cff7b8 100644 --- a/v2/ansible/plugins/__init__.py +++ b/v2/ansible/plugins/__init__.py @@ -240,7 +240,7 @@ callback_loader = PluginLoader( connection_loader = PluginLoader( 'Connection', - 'ansible.plugins.connection', + 'ansible.plugins.connections', C.DEFAULT_CONNECTION_PLUGIN_PATH, 'connection_plugins', aliases={'paramiko': 'paramiko_ssh'} @@ -253,37 +253,44 @@ shell_loader = PluginLoader( 'shell_plugins', ) -module_finder = PluginLoader( +module_loader = PluginLoader( '', 'ansible.modules', C.DEFAULT_MODULE_PATH, 'library' ) -lookup_finder = PluginLoader( +lookup_loader = PluginLoader( 'LookupModule', 'ansible.plugins.lookup', C.DEFAULT_LOOKUP_PLUGIN_PATH, 'lookup_plugins' ) -vars_finder = PluginLoader( +vars_loader = PluginLoader( 'VarsModule', 'ansible.plugins.vars', C.DEFAULT_VARS_PLUGIN_PATH, 'vars_plugins' ) -filter_finder = PluginLoader( +filter_loader = PluginLoader( 'FilterModule', 'ansible.plugins.filter', C.DEFAULT_FILTER_PLUGIN_PATH, 'filter_plugins' ) -fragment_finder = PluginLoader( +fragment_loader = PluginLoader( 'ModuleDocFragment', 'ansible.utils.module_docs_fragments', os.path.join(os.path.dirname(__file__), 'module_docs_fragments'), '', ) + +strategy_loader = PluginLoader( + 'StrategyModule', + 'ansible.plugins.strategies', + None, + 'strategy_plugins', +) diff --git a/v2/ansible/plugins/action/__init__.py b/v2/ansible/plugins/action/__init__.py index 785fc45992..6eb69e45d6 100644 --- a/v2/ansible/plugins/action/__init__.py +++ b/v2/ansible/plugins/action/__init__.py @@ -19,3 +19,449 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import StringIO +import json +import os +import random +import sys # FIXME: probably not needed +import tempfile +import time + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.executor.module_common import ModuleReplacer +from ansible.parsing.utils.jsonify import jsonify +from ansible.plugins import shell_loader + +from ansible.utils.debug import debug + +class ActionBase: + + ''' + This class is the base class for all action plugins, and defines + code common to all actions. The base class handles the connection + by putting/getting files and executing commands based on the current + action in use. + ''' + + def __init__(self, task, connection, connection_info, loader, module_loader): + self._task = task + self._connection = connection + self._connection_info = connection_info + self._loader = loader + self._module_loader = module_loader + self._shell = self.get_shell() + + self._supports_check_mode = True + + def get_shell(self): + + # FIXME: no more inject, get this from the host variables? + #default_shell = getattr(self._connection, 'default_shell', '') + #shell_type = inject.get('ansible_shell_type') + #if not shell_type: + # if default_shell: + # shell_type = default_shell + # else: + # shell_type = os.path.basename(C.DEFAULT_EXECUTABLE) + + shell_type = getattr(self._connection, 'default_shell', '') + if not shell_type: + shell_type = os.path.basename(C.DEFAULT_EXECUTABLE) + + shell_plugin = shell_loader.get(shell_type) + if shell_plugin is None: + shell_plugin = shell_loader.get('sh') + + return shell_plugin + + def _configure_module(self, module_name, module_args): + ''' + Handles the loading and templating of the module code through the + ModuleReplacer class. + ''' + + # Search module path(s) for named module. + module_suffixes = getattr(self._connection, 'default_suffixes', None) + module_path = self._module_loader.find_plugin(module_name, module_suffixes, transport=self._connection.get_transport()) + if module_path is None: + module_path2 = self._module_loader.find_plugin('ping', module_suffixes) + if module_path2 is not None: + raise AnsibleError("The module %s was not found in configured module paths" % (module_name)) + else: + raise AnsibleError("The module %s was not found in configured module paths. " \ + "Additionally, core modules are missing. If this is a checkout, " \ + "run 'git submodule update --init --recursive' to correct this problem." % (module_name)) + + # insert shared code and arguments into the module + (module_data, module_style, module_shebang) = ModuleReplacer().modify_module(module_path, module_args) + + return (module_style, module_shebang, module_data) + + def _compute_environment_string(self): + ''' + Builds the environment string to be used when executing the remote task. + ''' + + enviro = {} + + # FIXME: not sure where this comes from, probably task but maybe also the play? + #if self.environment: + # enviro = template.template(self.basedir, self.environment, inject, convert_bare=True) + # enviro = utils.safe_eval(enviro) + # if type(enviro) != dict: + # raise errors.AnsibleError("environment must be a dictionary, received %s" % enviro) + + return self._shell.env_prefix(**enviro) + + def _early_needs_tmp_path(self): + ''' + Determines if a temp path should be created before the action is executed. + ''' + + # FIXME: modified from original, needs testing? Since this is now inside + # the action plugin, it should make it just this simple + return getattr(self, 'TRANSFERS_FILES', False) + + def _late_needs_tmp_path(self, tmp, module_style): + ''' + Determines if a temp path is required after some early actions have already taken place. + ''' + if tmp and "tmp" in tmp: + # tmp has already been created + return False + if not self._connection._has_pipelining or not C.ANSIBLE_SSH_PIPELINING or C.DEFAULT_KEEP_REMOTE_FILES or self._connection_info.su: + # tmp is necessary to store module source code + return True + if not self._connection._has_pipelining: + # tmp is necessary to store the module source code + # or we want to keep the files on the target system + return True + if module_style != "new": + # even when conn has pipelining, old style modules need tmp to store arguments + return True + return False + + # FIXME: return a datastructure in this function instead of raising errors - + # the new executor pipeline handles it much better that way + def _make_tmp_path(self): + ''' + Create and return a temporary path on a remote box. + ''' + + basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48)) + use_system_tmp = False + + if (self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root'): + use_system_tmp = True + + tmp_mode = None + if self._connection_info.remote_user != 'root' or \ + ((self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root')): + tmp_mode = 'a+rx' + + cmd = self._shell.mkdtemp(basefile, use_system_tmp, tmp_mode) + result = self._low_level_execute_command(cmd, None, sudoable=False) + + # error handling on this seems a little aggressive? + if result['rc'] != 0: + if result['rc'] == 5: + output = 'Authentication failure.' + elif result['rc'] == 255 and self._connection.get_transport() in ['ssh']: + # FIXME: more utils.VERBOSITY + #if utils.VERBOSITY > 3: + # output = 'SSH encountered an unknown error. The output was:\n%s' % (result['stdout']+result['stderr']) + #else: + # output = 'SSH encountered an unknown error during the connection. We recommend you re-run the command using -vvvv, which will enable SSH debugging output to help diagnose the issue' + output = 'SSH encountered an unknown error. The output was:\n%s' % (result['stdout']+result['stderr']) + elif 'No space left on device' in result['stderr']: + output = result['stderr'] + else: + output = 'Authentication or permission failure. In some cases, you may have been able to authenticate and did not have permissions on the remote directory. Consider changing the remote temp path in ansible.cfg to a path rooted in "/tmp". Failed command was: %s, exited with result %d' % (cmd, result['rc']) + if 'stdout' in result and result['stdout'] != '': + output = output + ": %s" % result['stdout'] + raise AnsibleError(output) + + # FIXME: do we still need to do this? + #rc = self._shell.join_path(utils.last_non_blank_line(result['stdout']).strip(), '') + rc = self._shell.join_path(result['stdout'].strip(), '').splitlines()[-1] + + # Catch failure conditions, files should never be + # written to locations in /. + if rc == '/': + raise AnsibleError('failed to resolve remote temporary directory from %s: `%s` returned empty string' % (basetmp, cmd)) + + return rc + + def _remove_tmp_path(self, tmp_path): + '''Remove a temporary path we created. ''' + + if "-tmp-" in tmp_path: + cmd = self._shell.remove(tmp_path, recurse=True) + # If we have gotten here we have a working ssh configuration. + # If ssh breaks we could leave tmp directories out on the remote system. + self._low_level_execute_command(cmd, None, sudoable=False) + + def _transfer_data(self, remote_path, data): + ''' + Copies the module data out to the temporary module path. + ''' + + if type(data) == dict: + data = jsonify(data) + + afd, afile = tempfile.mkstemp() + afo = os.fdopen(afd, 'w') + try: + if not isinstance(data, unicode): + #ensure the data is valid UTF-8 + data = data.decode('utf-8') + else: + data = data.encode('utf-8') + afo.write(data) + except Exception, e: + raise AnsibleError("failure encoding into utf-8: %s" % str(e)) + + afo.flush() + afo.close() + + try: + self._connection.put_file(afile, remote_path) + finally: + os.unlink(afile) + + return remote_path + + def _remote_chmod(self, tmp, mode, path, sudoable=False): + ''' + Issue a remote chmod command + ''' + + cmd = self._shell.chmod(mode, path) + return self._low_level_execute_command(cmd, tmp, sudoable=sudoable) + + def _remote_checksum(self, tmp, path): + ''' + Takes a remote checksum and returns 1 if no file + ''' + + # FIXME: figure out how this will work, probably pulled from the + # variable manager data + #python_interp = inject['hostvars'][inject['inventory_hostname']].get('ansible_python_interpreter', 'python') + python_interp = 'python' + cmd = self._shell.checksum(path, python_interp) + data = self._low_level_execute_command(cmd, tmp, sudoable=True) + # FIXME: implement this function? + #data2 = utils.last_non_blank_line(data['stdout']) + try: + data2 = data['stdout'].strip().splitlines()[-1] + if data2 == '': + # this may happen if the connection to the remote server + # failed, so just return "INVALIDCHECKSUM" to avoid errors + return "INVALIDCHECKSUM" + else: + return data2.split()[0] + except IndexError: + # FIXME: this should probably not print to sys.stderr, but should instead + # fail in a more normal way? + sys.stderr.write("warning: Calculating checksum failed unusually, please report this to the list so it can be fixed\n") + sys.stderr.write("command: %s\n" % cmd) + sys.stderr.write("----\n") + sys.stderr.write("output: %s\n" % data) + sys.stderr.write("----\n") + # this will signal that it changed and allow things to keep going + return "INVALIDCHECKSUM" + + def _remote_expand_user(self, path, tmp): + ''' takes a remote path and performs tilde expansion on the remote host ''' + if not path.startswith('~'): + return path + + split_path = path.split(os.path.sep, 1) + expand_path = split_path[0] + if expand_path == '~': + if self._connection_info.sudo and self._connection_info.sudo_user: + expand_path = '~%s' % self._connection_info.sudo_user + elif self._connection_info.su and self._connection_info.su_user: + expand_path = '~%s' % self._connection_info.su_user + + cmd = self._shell.expand_user(expand_path) + data = self._low_level_execute_command(cmd, tmp, sudoable=False) + #initial_fragment = utils.last_non_blank_line(data['stdout']) + initial_fragment = data['stdout'].strip().splitlines()[-1] + + if not initial_fragment: + # Something went wrong trying to expand the path remotely. Return + # the original string + return path + + if len(split_path) > 1: + return self._shell.join_path(initial_fragment, *split_path[1:]) + else: + return initial_fragment + + def _filter_leading_non_json_lines(self, data): + ''' + Used to avoid random output from SSH at the top of JSON output, like messages from + tcagetattr, or where dropbear spews MOTD on every single command (which is nuts). + + need to filter anything which starts not with '{', '[', ', '=' or is an empty line. + filter only leading lines since multiline JSON is valid. + ''' + + filtered_lines = StringIO.StringIO() + stop_filtering = False + for line in data.splitlines(): + if stop_filtering or line.startswith('{') or line.startswith('['): + stop_filtering = True + filtered_lines.write(line + '\n') + return filtered_lines.getvalue() + + def _execute_module(self, module_name=None, module_args=None, tmp=None, persist_files=False, delete_remote_tmp=True): + ''' + Transfer and run a module along with its arguments. + ''' + + # if a module name was not specified for this execution, use + # the action from the task + if module_name is None: + module_name = self._task.action + if module_args is None: + module_args = self._task.args + + # set check mode in the module arguments, if required + if self._connection_info.check_mode and not self._task.always_run: + if not self._supports_check_mode: + raise AnsibleError("check mode is not supported for this operation") + module_args['_ansible_check_mode'] = True + + # set no log in the module arguments, if required + if self._connection_info.no_log: + module_args['_ansible_no_log'] = True + + debug("in _execute_module (%s, %s)" % (module_name, module_args)) + + (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args) + if not shebang: + raise AnsibleError("module is missing interpreter line") + + # a remote tmp path may be necessary and not already created + remote_module_path = None + if not tmp and self._late_needs_tmp_path(tmp, module_style): + tmp = self._make_tmp_path() + remote_module_path = self._shell.join_path(tmp, module_name) + + # FIXME: async stuff here? + #if (module_style != 'new' or async_jid is not None or not self._connection._has_pipelining or not C.ANSIBLE_SSH_PIPELINING or C.DEFAULT_KEEP_REMOTE_FILES): + if remote_module_path: + self._transfer_data(remote_module_path, module_data) + + environment_string = self._compute_environment_string() + + if tmp and "tmp" in tmp and ((self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root')): + # deal with possible umask issues once sudo'ed to other user + self._remote_chmod(tmp, 'a+r', remote_module_path) + + cmd = "" + in_data = None + + # FIXME: all of the old-module style and async stuff has been removed from here, and + # might need to be re-added (unless we decide to drop support for old-style modules + # at this point and rework things to support non-python modules specifically) + if self._connection._has_pipelining and C.ANSIBLE_SSH_PIPELINING and not C.DEFAULT_KEEP_REMOTE_FILES: + in_data = module_data + else: + if remote_module_path: + cmd = remote_module_path + + rm_tmp = None + if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp: + if not self._connection_info.sudo or self._connection_info.su or self._connection_info.sudo_user == 'root' or self._connection_info.su_user == 'root': + # not sudoing or sudoing to root, so can cleanup files in the same step + rm_tmp = tmp + + cmd = self._shell.build_module_command(environment_string, shebang, cmd, rm_tmp) + cmd = cmd.strip() + + sudoable = True + if module_name == "accelerate": + # always run the accelerate module as the user + # specified in the play, not the sudo_user + sudoable = False + + res = self._low_level_execute_command(cmd, tmp, sudoable=sudoable, in_data=in_data) + + if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp: + if (self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root'): + # not sudoing to root, so maybe can't delete files as that other user + # have to clean up temp files as original user in a second step + cmd2 = self._shell.remove(tmp, recurse=True) + self._low_level_execute_command(cmd2, tmp, sudoable=False) + + # FIXME: in error situations, the stdout may not contain valid data, so we + # should check for bad rc codes better to catch this here + if 'stdout' in res and res['stdout'].strip(): + data = json.loads(self._filter_leading_non_json_lines(res['stdout'])) + if 'parsed' in data and data['parsed'] == False: + data['msg'] += res['stderr'] + # pre-split stdout into lines, if stdout is in the data and there + # isn't already a stdout_lines value there + if 'stdout' in data and 'stdout_lines' not in data: + data['stdout_lines'] = data.get('stdout', '').splitlines() + else: + data = dict() + + # store the module invocation details back into the result + data['invocation'] = dict( + module_args = module_args, + module_name = module_name, + ) + + debug("done with _execute_module (%s, %s)" % (module_name, module_args)) + return data + + def _low_level_execute_command(self, cmd, tmp, executable=None, sudoable=True, in_data=None): + ''' + This is the function which executes the low level shell command, which + may be commands to create/remove directories for temporary files, or to + run the module code or python directly when pipelining. + ''' + + debug("in _low_level_execute_command() (%s)" % (cmd,)) + if not cmd: + # this can happen with powershell modules when there is no analog to a Windows command (like chmod) + debug("no command, exiting _low_level_execute_command()") + return dict(stdout='', stderr='') + + if executable is None: + executable = C.DEFAULT_EXECUTABLE + + prompt = None + success_key = None + + if sudoable: + if self._connection_info.su and self._connection_info.su_user: + cmd, prompt, success_key = self._connection_info.make_su_cmd(executable, cmd) + elif self._connection_info.sudo and self._connection_info.sudo_user: + # FIXME: hard-coded sudo_exe here + cmd, prompt, success_key = self._connection_info.make_sudo_cmd('/usr/bin/sudo', executable, cmd) + + debug("executing the command through the connection") + rc, stdin, stdout, stderr = self._connection.exec_command(cmd, tmp, executable=executable, in_data=in_data) + debug("command execution done") + + if not isinstance(stdout, basestring): + out = ''.join(stdout.readlines()) + else: + out = stdout + + if not isinstance(stderr, basestring): + err = ''.join(stderr.readlines()) + else: + err = stderr + + debug("done with _low_level_execute_command() (%s)" % (cmd,)) + if rc is not None: + return dict(rc=rc, stdout=out, stderr=err) + else: + return dict(stdout=out, stderr=err) diff --git a/v2/ansible/plugins/action/add_host.py b/v2/ansible/plugins/action/add_host.py new file mode 100644 index 0000000000..e28361b714 --- /dev/null +++ b/v2/ansible/plugins/action/add_host.py @@ -0,0 +1,62 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright 2012, Seth Vidal <skvidal@fedoraproject.org> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + ''' Create inventory hosts and groups in the memory inventory''' + + ### We need to be able to modify the inventory + BYPASS_HOST_LOOP = True + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + # FIXME: is this necessary in v2? + #if self.runner.noop_on_check(inject): + # return ReturnData(conn=conn, comm_ok=True, result=dict(skipped=True, msg='check mode not supported for this module')) + + # Parse out any hostname:port patterns + new_name = self._task.args.get('name', self._task.args.get('hostname', None)) + #vv("creating host via 'add_host': hostname=%s" % new_name) + + if ":" in new_name: + new_name, new_port = new_name.split(":") + self._task.args['ansible_ssh_port'] = new_port + + groups = self._task.args.get('groupname', self._task.args.get('groups', self._task.args.get('group', ''))) + # add it to the group if that was specified + new_groups = [] + if groups: + for group_name in groups.split(","): + if group_name not in new_groups: + new_groups.append(group_name.strip()) + + # Add any variables to the new_host + host_vars = dict() + for k in self._task.args.keys(): + if not k in [ 'name', 'hostname', 'groupname', 'groups' ]: + host_vars[k] = self._task.args[k] + + return dict(changed=True, add_host=dict(host_name=new_name, groups=new_groups, host_vars=host_vars)) + + diff --git a/v2/ansible/plugins/action/assemble.py b/v2/ansible/plugins/action/assemble.py new file mode 100644 index 0000000000..1ae8be0203 --- /dev/null +++ b/v2/ansible/plugins/action/assemble.py @@ -0,0 +1,154 @@ +# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com> +# Stephen Fromm <sfromm@gmail.com> +# Brian Coca <briancoca+dev@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +import os +import os.path +import pipes +import shutil +import tempfile +import base64 +import re + +from ansible.plugins.action import ActionBase +from ansible.utils.boolean import boolean +from ansible.utils.hashing import checksum_s + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def _assemble_from_fragments(self, src_path, delimiter=None, compiled_regexp=None): + ''' assemble a file from a directory of fragments ''' + + tmpfd, temp_path = tempfile.mkstemp() + tmp = os.fdopen(tmpfd,'w') + delimit_me = False + add_newline = False + + for f in sorted(os.listdir(src_path)): + if compiled_regexp and not compiled_regexp.search(f): + continue + fragment = "%s/%s" % (src_path, f) + if not os.path.isfile(fragment): + continue + fragment_content = file(fragment).read() + + # always put a newline between fragments if the previous fragment didn't end with a newline. + if add_newline: + tmp.write('\n') + + # delimiters should only appear between fragments + if delimit_me: + if delimiter: + # un-escape anything like newlines + delimiter = delimiter.decode('unicode-escape') + tmp.write(delimiter) + # always make sure there's a newline after the + # delimiter, so lines don't run together + if delimiter[-1] != '\n': + tmp.write('\n') + + tmp.write(fragment_content) + delimit_me = True + if fragment_content.endswith('\n'): + add_newline = False + else: + add_newline = True + + tmp.close() + return temp_path + + def run(self, tmp=None, task_vars=dict()): + + src = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + delimiter = self._task.args.get('delimiter', None) + remote_src = self._task.args.get('remote_src', 'yes') + regexp = self._task.args.get('regexp', None) + + if src is None or dest is None: + return dict(failed=True, msg="src and dest are required") + + if boolean(remote_src): + return self._execute_module(tmp=tmp) + elif self._task._role is not None: + src = self._loader.path_dwim_relative(self._task._role._role_path, 'files', src) + else: + # the source is local, so expand it here + src = os.path.expanduser(src) + + _re = None + if regexp is not None: + _re = re.compile(regexp) + + # Does all work assembling the file + path = self._assemble_from_fragments(src, delimiter, _re) + + path_checksum = checksum_s(path) + dest = self._remote_expand_user(dest, tmp) + remote_checksum = self._remote_checksum(tmp, dest) + + if path_checksum != remote_checksum: + resultant = file(path).read() + # FIXME: diff needs to be moved somewhere else + #if self.runner.diff: + # dest_result = self._execute_module(module_name='slurp', module_args=dict(path=dest), tmp=tmp, persist_files=True) + # if 'content' in dest_result: + # dest_contents = dest_result['content'] + # if dest_result['encoding'] == 'base64': + # dest_contents = base64.b64decode(dest_contents) + # else: + # raise Exception("unknown encoding, failed: %s" % dest_result) + xfered = self._transfer_data('src', resultant) + + # fix file permissions when the copy is done as a different user + if self._connection_info.sudo and self._connection_info.sudo_user != 'root' or self._connection_info.su and self._connection_info.su_user != 'root': + self._remote_chmod('a+r', xfered, tmp) + + # run the copy module + + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=xfered, + dest=dest, + original_basename=os.path.basename(src), + ) + ) + + # FIXME: checkmode stuff + #if self.runner.noop_on_check(inject): + # return ReturnData(conn=conn, comm_ok=True, result=dict(changed=True), diff=dict(before_header=dest, after_header=src, after=resultant)) + #else: + # res = self.runner._execute_module(conn, tmp, 'copy', module_args_tmp, inject=inject) + # res.diff = dict(after=resultant) + # return res + res = self._execute_module(module_name='copy', module_args=new_module_args, tmp=tmp) + #res.diff = dict(after=resultant) + return res + else: + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=xfered, + dest=dest, + original_basename=os.path.basename(src), + ) + ) + + return self._execute_module(module_name='file', module_args=new_module_args, tmp=tmp) diff --git a/v2/ansible/plugins/action/assert.py b/v2/ansible/plugins/action/assert.py new file mode 100644 index 0000000000..7204d93875 --- /dev/null +++ b/v2/ansible/plugins/action/assert.py @@ -0,0 +1,63 @@ +# Copyright 2012, Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.errors import AnsibleError +from ansible.playbook.conditional import Conditional +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + ''' Fail with custom message ''' + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + if not 'that' in self._task.args: + raise AnsibleError('conditional required in "that" string') + + msg = None + if 'msg' in self._task.args: + msg = self._task.args['msg'] + + # make sure the 'that' items are a list + thats = self._task.args['that'] + if not isinstance(thats, list): + thats = [ thats ] + + # Now we iterate over the that items, temporarily assigning them + # to the task's when value so we can evaluate the conditional using + # the built in evaluate function. The when has already been evaluated + # by this point, and is not used again, so we don't care about mangling + # that value now + cond = Conditional(loader=self._loader) + for that in thats: + cond.when = [ that ] + test_result = cond.evaluate_conditional(all_vars=task_vars) + if not test_result: + result = dict( + failed = True, + evaluated_to = test_result, + assertion = that, + ) + + if msg: + result['msg'] = msg + + return result + + return dict(changed=False, msg='all assertions passed') + diff --git a/v2/ansible/plugins/action/async.py b/v2/ansible/plugins/action/async.py new file mode 100644 index 0000000000..6fbf93d61f --- /dev/null +++ b/v2/ansible/plugins/action/async.py @@ -0,0 +1,68 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import json +import random + +from ansible import constants as C +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=dict()): + ''' transfer the given module name, plus the async module, then run it ''' + + # FIXME: noop stuff needs to be sorted ut + #if self.runner.noop_on_check(inject): + # return ReturnData(conn=conn, comm_ok=True, result=dict(skipped=True, msg='check mode not supported for this module')) + + if not tmp: + tmp = self._make_tmp_path() + + module_name = self._task.action + async_module_path = self._shell.join_path(tmp, 'async_wrapper') + remote_module_path = self._shell.join_path(tmp, module_name) + + env_string = self._compute_environment_string() + + # configure, upload, and chmod the target module + (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=self._task.args) + self._transfer_data(remote_module_path, module_data) + self._remote_chmod(tmp, 'a+rx', remote_module_path) + + # configure, upload, and chmod the async_wrapper module + (async_module_style, shebang, async_module_data) = self._configure_module(module_name='async_wrapper', module_args=dict()) + self._transfer_data(async_module_path, async_module_data) + self._remote_chmod(tmp, 'a+rx', async_module_path) + + argsfile = self._transfer_data(self._shell.join_path(tmp, 'arguments'), json.dumps(self._task.args)) + + async_limit = self._task.async + async_jid = str(random.randint(0, 999999999999)) + + async_cmd = " ".join([str(x) for x in [async_module_path, async_jid, async_limit, remote_module_path, argsfile]]) + result = self._low_level_execute_command(cmd=async_cmd, tmp=None) + + # clean up after + if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES: + self._remove_tmp_path(tmp) + + result['changed'] = True + + return result + + diff --git a/v2/ansible/plugins/action/copy.py b/v2/ansible/plugins/action/copy.py new file mode 100644 index 0000000000..6975bff1bf --- /dev/null +++ b/v2/ansible/plugins/action/copy.py @@ -0,0 +1,378 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import base64 +import json +import os +import pipes +import stat +import tempfile + +from ansible import constants as C +from ansible.plugins.action import ActionBase +from ansible.utils.boolean import boolean +from ansible.utils.hashing import checksum + +## fixes https://github.com/ansible/ansible/issues/3518 +# http://mypy.pythonblogs.com/12_mypy/archive/1253_workaround_for_python_bug_ascii_codec_cant_encode_character_uxa0_in_position_111_ordinal_not_in_range128.html + +import sys +reload(sys) +sys.setdefaultencoding("utf8") + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=dict()): + ''' handler for file transfer operations ''' + + source = self._task.args.get('src', None) + content = self._task.args.get('content', None) + dest = self._task.args.get('dest', None) + raw = boolean(self._task.args.get('raw', 'no')) + force = boolean(self._task.args.get('force', 'yes')) + + # content with newlines is going to be escaped to safely load in yaml + # now we need to unescape it so that the newlines are evaluated properly + # when writing the file to disk + if content: + if isinstance(content, unicode): + try: + content = content.decode('unicode-escape') + except UnicodeDecodeError: + pass + + # FIXME: first available file needs to be reworked somehow... + #if (source is None and content is None and not 'first_available_file' in inject) or dest is None: + # result=dict(failed=True, msg="src (or content) and dest are required") + # return ReturnData(conn=conn, result=result) + #elif (source is not None or 'first_available_file' in inject) and content is not None: + # result=dict(failed=True, msg="src and content are mutually exclusive") + # return ReturnData(conn=conn, result=result) + + # Check if the source ends with a "/" + source_trailing_slash = False + if source: + source_trailing_slash = source.endswith("/") + + # Define content_tempfile in case we set it after finding content populated. + content_tempfile = None + + # If content is defined make a temp file and write the content into it. + if content is not None: + try: + # If content comes to us as a dict it should be decoded json. + # We need to encode it back into a string to write it out. + if type(content) is dict: + content_tempfile = self._create_content_tempfile(json.dumps(content)) + else: + content_tempfile = self._create_content_tempfile(content) + source = content_tempfile + except Exception, err: + return dict(failed=True, msg="could not write content temp file: %s" % err) + + ############################################################################################### + # FIXME: first_available_file needs to be reworked? + ############################################################################################### + # if we have first_available_file in our vars + # look up the files and use the first one we find as src + #elif 'first_available_file' in inject: + # found = False + # for fn in inject.get('first_available_file'): + # fn_orig = fn + # fnt = template.template(self.runner.basedir, fn, inject) + # fnd = utils.path_dwim(self.runner.basedir, fnt) + # if not os.path.exists(fnd) and '_original_file' in inject: + # fnd = utils.path_dwim_relative(inject['_original_file'], 'files', fnt, self.runner.basedir, check=False) + # if os.path.exists(fnd): + # source = fnd + # found = True + # break + # if not found: + # results = dict(failed=True, msg="could not find src in first_available_file list") + # return ReturnData(conn=conn, result=results) + ############################################################################################### + else: + if self._task._role is not None: + source = self._loader.path_dwim_relative(self._task._role._role_path, 'files', source) + else: + source = self._loader.path_dwim(source) + + # A list of source file tuples (full_path, relative_path) which will try to copy to the destination + source_files = [] + + # If source is a directory populate our list else source is a file and translate it to a tuple. + if os.path.isdir(source): + # Get the amount of spaces to remove to get the relative path. + if source_trailing_slash: + sz = len(source) + else: + sz = len(source.rsplit('/', 1)[0]) + 1 + + # Walk the directory and append the file tuples to source_files. + for base_path, sub_folders, files in os.walk(source): + for file in files: + full_path = os.path.join(base_path, file) + rel_path = full_path[sz:] + source_files.append((full_path, rel_path)) + + # If it's recursive copy, destination is always a dir, + # explicitly mark it so (note - copy module relies on this). + if not self._shell.path_has_trailing_slash(dest): + dest = self._shell.join_path(dest, '') + else: + source_files.append((source, os.path.basename(source))) + + changed = False + diffs = [] + module_result = {"changed": False} + + # A register for if we executed a module. + # Used to cut down on command calls when not recursive. + module_executed = False + + # Tell _execute_module to delete the file if there is one file. + delete_remote_tmp = (len(source_files) == 1) + + # If this is a recursive action create a tmp path that we can share as the _exec_module create is too late. + if not delete_remote_tmp: + if tmp is None or "-tmp-" not in tmp: + tmp = self._make_tmp_path() + + # expand any user home dir specifier + dest = self._remote_expand_user(dest, tmp) + + for source_full, source_rel in source_files: + # Generate a hash of the local file. + local_checksum = checksum(source_full) + + # If local_checksum is not defined we can't find the file so we should fail out. + if local_checksum is None: + return dict(failed=True, msg="could not find src=%s" % source_full) + + # This is kind of optimization - if user told us destination is + # dir, do path manipulation right away, otherwise we still check + # for dest being a dir via remote call below. + if self._shell.path_has_trailing_slash(dest): + dest_file = self._shell.join_path(dest, source_rel) + else: + dest_file = self._shell.join_path(dest) + + # Attempt to get the remote checksum + remote_checksum = self._remote_checksum(tmp, dest_file) + + if remote_checksum == '3': + # The remote_checksum was executed on a directory. + if content is not None: + # If source was defined as content remove the temporary file and fail out. + self._remove_tempfile_if_content_defined(content, content_tempfile) + return dict(failed=True, msg="can not use content with a dir as dest") + else: + # Append the relative source location to the destination and retry remote_checksum + dest_file = self._shell.join_path(dest, source_rel) + remote_checksum = self._remote_checksum(tmp, dest_file) + + if remote_checksum != '1' and not force: + # remote_file does not exist so continue to next iteration. + continue + + if local_checksum != remote_checksum: + # The checksums don't match and we will change or error out. + changed = True + + # Create a tmp path if missing only if this is not recursive. + # If this is recursive we already have a tmp path. + if delete_remote_tmp: + if tmp is None or "-tmp-" not in tmp: + tmp = self._make_tmp_path() + + # FIXME: runner shouldn't have the diff option there + #if self.runner.diff and not raw: + # diff = self._get_diff_data(tmp, dest_file, source_full) + #else: + # diff = {} + diff = {} + + # FIXME: noop stuff + #if self.runner.noop_on_check(inject): + # self._remove_tempfile_if_content_defined(content, content_tempfile) + # diffs.append(diff) + # changed = True + # module_result = dict(changed=True) + # continue + + # Define a remote directory that we will copy the file to. + tmp_src = tmp + 'source' + + if not raw: + self._connection.put_file(source_full, tmp_src) + else: + self._connection.put_file(source_full, dest_file) + + # We have copied the file remotely and no longer require our content_tempfile + self._remove_tempfile_if_content_defined(content, content_tempfile) + + # fix file permissions when the copy is done as a different user + if (self._connection_info.sudo and self._connection_info.sudo_user != 'root' or self._connection_info.su and self._connection_info.su_user != 'root') and not raw: + self._remote_chmod('a+r', tmp_src, tmp) + + if raw: + # Continue to next iteration if raw is defined. + continue + + # Run the copy module + + # src and dest here come after original and override them + # we pass dest only to make sure it includes trailing slash in case of recursive copy + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=tmp_src, + dest=dest, + original_basename=source_rel, + ) + ) + + # FIXME: checkmode and no_log stuff + #if self.runner.noop_on_check(inject): + # new_module_args['CHECKMODE'] = True + #if self.runner.no_log: + # new_module_args['NO_LOG'] = True + + module_return = self._execute_module(module_name='copy', module_args=new_module_args, delete_remote_tmp=delete_remote_tmp) + module_executed = True + + else: + # no need to transfer the file, already correct hash, but still need to call + # the file module in case we want to change attributes + self._remove_tempfile_if_content_defined(content, content_tempfile) + + if raw: + # Continue to next iteration if raw is defined. + # self._remove_tmp_path(tmp) + continue + + # Build temporary module_args. + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=source_rel, + dest=dest, + original_basename=source_rel + ) + ) + # FIXME: checkmode and no_log stuff + #if self.runner.noop_on_check(inject): + # new_module_args['CHECKMODE'] = True + #if self.runner.no_log: + # new_module_args['NO_LOG'] = True + + # Execute the file module. + module_return = self._execute_module(module_name='file', module_args=new_module_args, delete_remote_tmp=delete_remote_tmp) + module_executed = True + + if not module_return.get('checksum'): + module_return['checksum'] = local_checksum + if module_return.get('failed') == True: + return module_return + if module_return.get('changed') == True: + changed = True + + # Delete tmp path if we were recursive or if we did not execute a module. + if (not C.DEFAULT_KEEP_REMOTE_FILES and not delete_remote_tmp) \ + or (not C.DEFAULT_KEEP_REMOTE_FILES and delete_remote_tmp and not module_executed): + self._remove_tmp_path(tmp) + + # the file module returns the file path as 'path', but + # the copy module uses 'dest', so add it if it's not there + if 'path' in module_return and 'dest' not in module_return: + module_return['dest'] = module_return['path'] + + # TODO: Support detailed status/diff for multiple files + if len(source_files) == 1: + result = module_return + else: + result = dict(dest=dest, src=source, changed=changed) + + # FIXME: move diffs into the result? + #if len(diffs) == 1: + # return ReturnData(conn=conn, result=result, diff=diffs[0]) + #else: + # return ReturnData(conn=conn, result=result) + + return result + + def _create_content_tempfile(self, content): + ''' Create a tempfile containing defined content ''' + fd, content_tempfile = tempfile.mkstemp() + f = os.fdopen(fd, 'w') + try: + f.write(content) + except Exception, err: + os.remove(content_tempfile) + raise Exception(err) + finally: + f.close() + return content_tempfile + + def _get_diff_data(self, tmp, destination, source): + peek_result = self._execute_module(module_name='file', module_args=dict(path=destination, diff_peek=True), persist_files=True) + if 'failed' in peek_result and peek_result['failed'] or peek_result.get('rc', 0) != 0: + return {} + + diff = {} + if peek_result['state'] == 'absent': + diff['before'] = '' + elif peek_result['appears_binary']: + diff['dst_binary'] = 1 + # FIXME: this should not be in utils.. + #elif peek_result['size'] > utils.MAX_FILE_SIZE_FOR_DIFF: + # diff['dst_larger'] = utils.MAX_FILE_SIZE_FOR_DIFF + else: + dest_result = self._execute_module(module_name='slurp', module_args=dict(path=destination), tmp=tmp, persist_files=True) + if 'content' in dest_result: + dest_contents = dest_result['content'] + if dest_result['encoding'] == 'base64': + dest_contents = base64.b64decode(dest_contents) + else: + raise Exception("unknown encoding, failed: %s" % dest_result) + diff['before_header'] = destination + diff['before'] = dest_contents + + src = open(source) + src_contents = src.read(8192) + st = os.stat(source) + if "\x00" in src_contents: + diff['src_binary'] = 1 + # FIXME: this should not be in utils + #elif st[stat.ST_SIZE] > utils.MAX_FILE_SIZE_FOR_DIFF: + # diff['src_larger'] = utils.MAX_FILE_SIZE_FOR_DIFF + else: + src.seek(0) + diff['after_header'] = source + diff['after'] = src.read() + + return diff + + def _remove_tempfile_if_content_defined(self, content, content_tempfile): + if content is not None: + os.remove(content_tempfile) + diff --git a/v2/ansible/plugins/action/debug.py b/v2/ansible/plugins/action/debug.py new file mode 100644 index 0000000000..dcee3e6347 --- /dev/null +++ b/v2/ansible/plugins/action/debug.py @@ -0,0 +1,46 @@ +# Copyright 2012, Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.plugins.action import ActionBase +from ansible.utils.boolean import boolean +from ansible.template import Templar + +class ActionModule(ActionBase): + ''' Print statements during execution ''' + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + if 'msg' in self._task.args: + if 'fail' in self._task.args and boolean(self._task.args['fail']): + result = dict(failed=True, msg=self._task.args['msg']) + else: + result = dict(msg=self._task.args['msg']) + # FIXME: move the LOOKUP_REGEX somewhere else + elif 'var' in self._task.args: # and not utils.LOOKUP_REGEX.search(self._task.args['var']): + templar = Templar(loader=self._loader, variables=task_vars) + results = templar.template(self._task.args['var'], convert_bare=True) + result = dict() + result[self._task.args['var']] = results + else: + result = dict(msg='here we are') + + # force flag to make debug output module always verbose + result['verbose_always'] = True + + return result diff --git a/v2/ansible/plugins/action/fail.py b/v2/ansible/plugins/action/fail.py new file mode 100644 index 0000000000..a95ccb32f7 --- /dev/null +++ b/v2/ansible/plugins/action/fail.py @@ -0,0 +1,33 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2012, Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + ''' Fail with custom message ''' + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + msg = 'Failed as requested from task' + if self._task.args and 'msg' in self._task.args: + msg = self._task.args.get('msg') + + return dict(failed=True, msg=msg) + diff --git a/v2/ansible/plugins/action/fetch.py b/v2/ansible/plugins/action/fetch.py new file mode 100644 index 0000000000..0ce33c650f --- /dev/null +++ b/v2/ansible/plugins/action/fetch.py @@ -0,0 +1,152 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os +import pwd +import random +import traceback +import tempfile +import base64 + +from ansible import constants as C +from ansible.errors import * +from ansible.plugins.action import ActionBase +from ansible.utils.boolean import boolean +from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=dict()): + ''' handler for fetch operations ''' + + # FIXME: is this even required anymore? + #if self.runner.noop_on_check(inject): + # return ReturnData(conn=conn, comm_ok=True, result=dict(skipped=True, msg='check mode not (yet) supported for this module')) + + source = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + flat = boolean(self._task.args.get('flat')) + fail_on_missing = boolean(self._task.args.get('fail_on_missing')) + validate_checksum = boolean(self._task.args.get('validate_checksum', self._task.args.get('validate_md5'))) + + if 'validate_md5' in self._task.args and 'validate_checksum' in self._task.args: + return dict(failed=True, msg="validate_checksum and validate_md5 cannot both be specified") + + if source is None or dest is None: + return dict(failed=True, msg="src and dest are required") + + source = self._shell.join_path(source) + source = self._remote_expand_user(source, tmp) + + # calculate checksum for the remote file + remote_checksum = self._remote_checksum(tmp, source) + + # use slurp if sudo and permissions are lacking + remote_data = None + if remote_checksum in ('1', '2') or self._connection_info.sudo: + slurpres = self._execute_module(module_name='slurp', module_args=dict(src=source), tmp=tmp) + if slurpres.get('rc') == 0: + if slurpres['encoding'] == 'base64': + remote_data = base64.b64decode(slurpres['content']) + if remote_data is not None: + remote_checksum = checksum_s(remote_data) + # the source path may have been expanded on the + # target system, so we compare it here and use the + # expanded version if it's different + remote_source = slurpres.get('source') + if remote_source and remote_source != source: + source = remote_source + else: + # FIXME: should raise an error here? the old code did nothing + pass + + # calculate the destination name + if os.path.sep not in self._shell.join_path('a', ''): + source_local = source.replace('\\', '/') + else: + source_local = source + + dest = os.path.expanduser(dest) + if flat: + if dest.endswith("/"): + # if the path ends with "/", we'll use the source filename as the + # destination filename + base = os.path.basename(source_local) + dest = os.path.join(dest, base) + if not dest.startswith("/"): + # if dest does not start with "/", we'll assume a relative path + dest = self._loader.path_dwim(dest) + else: + # files are saved in dest dir, with a subdir for each host, then the filename + dest = "%s/%s/%s" % (self._loader.path_dwim(dest), self._connection._host, source_local) + + dest = dest.replace("//","/") + + if remote_checksum in ('0', '1', '2', '3', '4'): + # these don't fail because you may want to transfer a log file that possibly MAY exist + # but keep going to fetch other log files + if remote_checksum == '0': + result = dict(msg="unable to calculate the checksum of the remote file", file=source, changed=False) + elif remote_checksum == '1': + if fail_on_missing: + result = dict(failed=True, msg="the remote file does not exist", file=source) + else: + result = dict(msg="the remote file does not exist, not transferring, ignored", file=source, changed=False) + elif remote_checksum == '2': + result = dict(msg="no read permission on remote file, not transferring, ignored", file=source, changed=False) + elif remote_checksum == '3': + result = dict(msg="remote file is a directory, fetch cannot work on directories", file=source, changed=False) + elif remote_checksum == '4': + result = dict(msg="python isn't present on the system. Unable to compute checksum", file=source, changed=False) + return result + + # calculate checksum for the local file + local_checksum = checksum(dest) + + if remote_checksum != local_checksum: + # create the containing directories, if needed + if not os.path.isdir(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + + # fetch the file and check for changes + if remote_data is None: + self._connection.fetch_file(source, dest) + else: + f = open(dest, 'w') + f.write(remote_data) + f.close() + new_checksum = secure_hash(dest) + # For backwards compatibility. We'll return None on FIPS enabled + # systems + try: + new_md5 = md5(dest) + except ValueError: + new_md5 = None + + if validate_checksum and new_checksum != remote_checksum: + return dict(failed=True, md5sum=new_md5, msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum) + return dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum) + else: + # For backwards compatibility. We'll return None on FIPS enabled + # systems + try: + local_md5 = md5(dest) + except ValueError: + local_md5 = None + + return dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum) + diff --git a/v2/ansible/plugins/action/group_by.py b/v2/ansible/plugins/action/group_by.py new file mode 100644 index 0000000000..50e0cc09c4 --- /dev/null +++ b/v2/ansible/plugins/action/group_by.py @@ -0,0 +1,37 @@ +# Copyright 2012, Jeroen Hoekx <jeroen@hoekx.be> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.errors import * +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + ''' Create inventory groups based on variables ''' + + ### We need to be able to modify the inventory + BYPASS_HOST_LOOP = True + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + if not 'key' in self._task.args: + return dict(failed=True, msg="the 'key' param is required when using group_by") + + group_name = self._task.args.get('key') + group_name = group_name.replace(' ','-') + + return dict(changed=True, add_group=group_name) + diff --git a/v2/ansible/plugins/action/include_vars.py b/v2/ansible/plugins/action/include_vars.py new file mode 100644 index 0000000000..345e0edc0e --- /dev/null +++ b/v2/ansible/plugins/action/include_vars.py @@ -0,0 +1,48 @@ +# (c) 2013-2014, Benno Joy <benno@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os + +from types import NoneType + +from ansible.errors import AnsibleError +from ansible.parsing import DataLoader +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + source = self._task.args.get('_raw_params') + + if self._task._role: + source = self._loader.path_dwim_relative(self._task._role._role_path, 'vars', source) + else: + source = self._loader.path_dwim(source) + + if os.path.exists(source): + data = self._loader.load_from_file(source) + if data is None: + data = {} + if not isinstance(data, dict): + raise AnsibleError("%s must be stored as a dictionary/hash" % source) + return dict(ansible_facts=data) + else: + return dict(failed=True, msg="Source file not found.", file=source) + diff --git a/v2/ansible/plugins/action/normal.py b/v2/ansible/plugins/action/normal.py new file mode 100644 index 0000000000..66721b4eb2 --- /dev/null +++ b/v2/ansible/plugins/action/normal.py @@ -0,0 +1,27 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=dict()): + + #vv("REMOTE_MODULE %s %s" % (module_name, module_args), host=conn.host) + return self._execute_module(tmp) + + diff --git a/v2/ansible/plugins/action/pause.py b/v2/ansible/plugins/action/pause.py new file mode 100644 index 0000000000..9c6075e101 --- /dev/null +++ b/v2/ansible/plugins/action/pause.py @@ -0,0 +1,134 @@ +# Copyright 2012, Tim Bielawa <tbielawa@redhat.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import datetime +import sys +import time + +from termios import tcflush, TCIFLUSH + +from ansible.errors import * +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + ''' pauses execution for a length or time, or until input is received ''' + + PAUSE_TYPES = ['seconds', 'minutes', 'prompt', ''] + BYPASS_HOST_LOOP = True + + def run(self, tmp=None, task_vars=dict()): + ''' run the pause action module ''' + + duration_unit = 'minutes' + prompt = None + seconds = None + result = dict( + changed = False, + rc = 0, + stderr = '', + stdout = '', + start = None, + stop = None, + delta = None, + ) + + # FIXME: not sure if we can get this info directly like this anymore? + #hosts = ', '.join(self.runner.host_set) + + # Is 'args' empty, then this is the default prompted pause + if self._task.args is None or len(self._task.args.keys()) == 0: + pause_type = 'prompt' + #prompt = "[%s]\nPress enter to continue:\n" % hosts + prompt = "[%s]\nPress enter to continue:\n" % self._task.get_name().strip() + + # Are 'minutes' or 'seconds' keys that exist in 'args'? + elif 'minutes' in self._task.args or 'seconds' in self._task.args: + try: + if 'minutes' in self._task.args: + pause_type = 'minutes' + # The time() command operates in seconds so we need to + # recalculate for minutes=X values. + seconds = int(self._task.args['minutes']) * 60 + else: + pause_type = 'seconds' + seconds = int(self._task.args['seconds']) + duration_unit = 'seconds' + + except ValueError, e: + return dict(failed=True, msg="non-integer value given for prompt duration:\n%s" % str(e)) + + # Is 'prompt' a key in 'args'? + elif 'prompt' in self._task.args: + pause_type = 'prompt' + #prompt = "[%s]\n%s:\n" % (hosts, self._task.args['prompt']) + prompt = "[%s]\n%s:\n" % (self._task.get_name().strip(), self._task.args['prompt']) + + # I have no idea what you're trying to do. But it's so wrong. + else: + return dict(failed=True, msg="invalid pause type given. must be one of: %s" % ", ".join(self.PAUSE_TYPES)) + + #vv("created 'pause' ActionModule: pause_type=%s, duration_unit=%s, calculated_seconds=%s, prompt=%s" % \ + # (self.pause_type, self.duration_unit, self.seconds, self.prompt)) + + ######################################################################## + # Begin the hard work! + + start = time.time() + result['start'] = str(datetime.datetime.now()) + + + # FIXME: this is all very broken right now, as prompting from the worker side + # is not really going to be supported, and actions marked as BYPASS_HOST_LOOP + # probably should not be run through the executor engine at all. Also, ctrl+c + # is now captured on the parent thread, so it can't be caught here via the + # KeyboardInterrupt exception. + + try: + if not pause_type == 'prompt': + print "(^C-c = continue early, ^C-a = abort)" + #print("[%s]\nPausing for %s seconds" % (hosts, seconds)) + print("[%s]\nPausing for %s seconds" % (self._task.get_name().strip(), seconds)) + time.sleep(seconds) + else: + # Clear out any unflushed buffered input which would + # otherwise be consumed by raw_input() prematurely. + #tcflush(sys.stdin, TCIFLUSH) + result['user_input'] = raw_input(prompt.encode(sys.stdout.encoding)) + except KeyboardInterrupt: + while True: + print '\nAction? (a)bort/(c)ontinue: ' + c = getch() + if c == 'c': + # continue playbook evaluation + break + elif c == 'a': + # abort further playbook evaluation + raise ae('user requested abort!') + finally: + duration = time.time() - start + result['stop'] = str(datetime.datetime.now()) + result['delta'] = int(duration) + + if duration_unit == 'minutes': + duration = round(duration / 60.0, 2) + else: + duration = round(duration, 2) + + result['stdout'] = "Paused for %s %s" % (duration, duration_unit) + + return result + diff --git a/v2/ansible/plugins/action/raw.py b/v2/ansible/plugins/action/raw.py new file mode 100644 index 0000000000..d1d1b28056 --- /dev/null +++ b/v2/ansible/plugins/action/raw.py @@ -0,0 +1,39 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + + # FIXME: need to rework the noop stuff still + #if self.runner.noop_on_check(inject): + # # in --check mode, always skip this module execution + # return ReturnData(conn=conn, comm_ok=True, result=dict(skipped=True)) + + executable = self._task.args.get('executable') + result = self._low_level_execute_command(self._task.args.get('_raw_params'), tmp=tmp, executable=executable) + + # for some modules (script, raw), the sudo success key + # may leak into the stdout due to the way the sudo/su + # command is constructed, so we filter that out here + if result.get('stdout','').strip().startswith('SUDO-SUCCESS-'): + result['stdout'] = re.sub(r'^((\r)?\n)?SUDO-SUCCESS.*(\r)?\n', '', result['stdout']) + + return result diff --git a/v2/ansible/plugins/action/script.py b/v2/ansible/plugins/action/script.py new file mode 100644 index 0000000000..6e8c1e1b9a --- /dev/null +++ b/v2/ansible/plugins/action/script.py @@ -0,0 +1,97 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os + +from ansible import constants as C +from ansible.plugins.action import ActionBase + +class ActionModule(ActionBase): + TRANSFERS_FILES = True + + def run(self, tmp=None, task_vars=None): + ''' handler for file transfer operations ''' + + # FIXME: noop stuff still needs to be sorted out + #if self.runner.noop_on_check(inject): + # # in check mode, always skip this module + # return ReturnData(conn=conn, comm_ok=True, + # result=dict(skipped=True, msg='check mode not supported for this module')) + + if not tmp: + tmp = self._make_tmp_path() + + creates = self._task.args.get('creates') + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + result = self._execute_module(module_name='stat', module_args=dict(path=creates), tmp=tmp, persist_files=True) + stat = result.get('stat', None) + if stat and stat.get('exists', False): + return dict(skipped=True, msg=("skipped, since %s exists" % creates)) + + removes = self._task.args.get('removes') + if removes: + # do not run the command if the line contains removes=filename + # and the filename does not exist. This allows idempotence + # of command executions. + result = self._execute_module(module_name='stat', module_args=dict(path=removes), tmp=tmp, persist_files=True) + stat = result.get('stat', None) + if stat and not stat.get('exists', False): + return dict(skipped=True, msg=("skipped, since %s does not exist" % removes)) + + # the script name is the first item in the raw params, so we split it + # out now so we know the file name we need to transfer to the remote, + # and everything else is an argument to the script which we need later + # to append to the remote command + parts = self._task.args.get('_raw_params', '').strip().split() + source = parts[0] + args = ' '.join(parts[1:]) + + if self._task._role is not None: + source = self._loader.path_dwim_relative(self._task._role._role_path, 'files', source) + else: + source = self._loader.path_dwim(source) + + # transfer the file to a remote tmp location + tmp_src = self._shell.join_path(tmp, os.path.basename(source)) + self._connection.put_file(source, tmp_src) + + sudoable = True + # set file permissions, more permissive when the copy is done as a different user + if ((self._connection_info.sudo and self._connection_info.sudo_user != 'root') or + (self._connection_info.su and self._connection_info.su_user != 'root')): + chmod_mode = 'a+rx' + sudoable = False + else: + chmod_mode = '+rx' + self._remote_chmod(tmp, chmod_mode, tmp_src, sudoable=sudoable) + + # add preparation steps to one ssh roundtrip executing the script + env_string = self._compute_environment_string() + script_cmd = ' '.join([env_string, tmp_src, args]) + + result = self._low_level_execute_command(cmd=script_cmd, tmp=None, sudoable=sudoable) + + # clean up after + if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES: + self._remove_tmp_path(tmp) + + result['changed'] = True + + return result diff --git a/v2/ansible/plugins/action/set_fact.py b/v2/ansible/plugins/action/set_fact.py new file mode 100644 index 0000000000..bf89e7ec51 --- /dev/null +++ b/v2/ansible/plugins/action/set_fact.py @@ -0,0 +1,36 @@ +# Copyright 2013 Dag Wieers <dag@wieers.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from ansible.template import Templar +from ansible.utils.boolean import boolean + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=dict()): + templar = Templar(loader=self._loader, variables=task_vars) + facts = dict() + if self._task.args: + for (k, v) in self._task.args.iteritems(): + k = templar.template(k) + if isinstance(v, basestring) and v.lower() in ('true', 'false', 'yes', 'no'): + v = boolean(v) + facts[k] = v + return dict(changed=True, ansible_facts=facts) diff --git a/v2/ansible/plugins/action/synchronize.py b/v2/ansible/plugins/action/synchronize.py new file mode 100644 index 0000000000..298d6a1959 --- /dev/null +++ b/v2/ansible/plugins/action/synchronize.py @@ -0,0 +1,176 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2012-2013, Timothy Appnel <tim@appnel.com> +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os.path + +from ansible.plugins.action import ActionBase +from ansible.utils.boolean import boolean + +class ActionModule(ActionBase): + + def _get_absolute_path(self, path, task_vars): + if 'vars' in task_vars: + if '_original_file' in task_vars['vars']: + # roles + original_path = path + path = self._loader.path_dwim_relative(task_vars['_original_file'], 'files', path, self.runner.basedir) + if original_path and original_path[-1] == '/' and path[-1] != '/': + # make sure the dwim'd path ends in a trailing "/" + # if the original path did + path += '/' + + return path + + def _process_origin(self, host, path, user, task_vars): + + if not host in ['127.0.0.1', 'localhost']: + if user: + return '%s@%s:%s' % (user, host, path) + else: + return '%s:%s' % (host, path) + else: + if not ':' in path: + if not path.startswith('/'): + path = self._get_absolute_path(path=path, task_vars=task_vars) + return path + + def _process_remote(self, host, path, user, task_vars): + transport = self._connection_info.connection + return_data = None + if not host in ['127.0.0.1', 'localhost'] or transport != "local": + if user: + return_data = '%s@%s:%s' % (user, host, path) + else: + return_data = '%s:%s' % (host, path) + else: + return_data = path + + if not ':' in return_data: + if not return_data.startswith('/'): + return_data = self._get_absolute_path(path=return_data, task_vars=task_vars) + + return return_data + + def run(self, tmp=None, task_vars=dict()): + ''' generates params and passes them on to the rsync module ''' + + original_transport = task_vars.get('ansible_connection', self._connection_info.connection) + transport_overridden = False + if task_vars.get('delegate_to') is None: + task_vars['delegate_to'] = '127.0.0.1' + # IF original transport is not local, override transport and disable sudo. + if original_transport != 'local': + task_vars['ansible_connection'] = 'local' + self.transport_overridden = True + self.runner.sudo = False + + src = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + + # FIXME: this doesn't appear to be used anywhere? + local_rsync_path = task_vars.get('ansible_rsync_path') + + # from the perspective of the rsync call the delegate is the localhost + src_host = '127.0.0.1' + dest_host = task_vars.get('ansible_ssh_host', task_vars.get('inventory_hostname')) + + # allow ansible_ssh_host to be templated + # FIXME: does this still need to be templated? + #dest_host = template.template(self.runner.basedir, dest_host, task_vars, fail_on_undefined=True) + dest_is_local = dest_host in ['127.0.0.1', 'localhost'] + + # CHECK FOR NON-DEFAULT SSH PORT + dest_port = self._task.args.get('dest_port') + inv_port = task_vars.get('ansible_ssh_port', task_vars.get('inventory_hostname')) + if inv_port != dest_port and inv_port != task_vars.get('inventory_hostname'): + dest_port = inv_port + + # edge case: explicit delegate and dest_host are the same + if dest_host == task_vars.get('delegate_to'): + dest_host = '127.0.0.1' + + # SWITCH SRC AND DEST PER MODE + if self._task.args.get('mode', 'push') == 'pull': + (dest_host, src_host) = (src_host, dest_host) + + # CHECK DELEGATE HOST INFO + use_delegate = False + # FIXME: not sure if this is in connection info yet or not... + #if conn.delegate != conn.host: + # if 'hostvars' in task_vars: + # if conn.delegate in task_vars['hostvars'] and self.original_transport != 'local': + # # use a delegate host instead of localhost + # use_delegate = True + + # COMPARE DELEGATE, HOST AND TRANSPORT + process_args = False + if not dest_host is src_host and self.original_transport != 'local': + # interpret and task_vars remote host info into src or dest + process_args = True + + # MUNGE SRC AND DEST PER REMOTE_HOST INFO + if process_args or use_delegate: + + user = None + if boolean(options.get('set_remote_user', 'yes')): + if use_delegate: + user = task_vars['hostvars'][conn.delegate].get('ansible_ssh_user') + + if not use_delegate or not user: + user = task_vars.get('ansible_ssh_user', self.runner.remote_user) + + if use_delegate: + # FIXME + private_key = task_vars.get('ansible_ssh_private_key_file', self.runner.private_key_file) + else: + private_key = task_vars.get('ansible_ssh_private_key_file', self.runner.private_key_file) + + if private_key is not None: + private_key = os.path.expanduser(private_key) + + # use the mode to define src and dest's url + if self._task.args.get('mode', 'push') == 'pull': + # src is a remote path: <user>@<host>, dest is a local path + src = self._process_remote(src_host, src, user, task_vars) + dest = self._process_origin(dest_host, dest, user, task_vars) + else: + # src is a local path, dest is a remote path: <user>@<host> + src = self._process_origin(src_host, src, user, task_vars) + dest = self._process_remote(dest_host, dest, user, task_vars) + + # Allow custom rsync path argument. + rsync_path = self._task.args.get('rsync_path', None) + + # If no rsync_path is set, sudo was originally set, and dest is remote then add 'sudo rsync' argument. + if not rsync_path and self.transport_overridden and self._connection_info.sudo and not dest_is_local: + self._task.args['rsync_path'] = 'sudo rsync' + + # make sure rsync path is quoted. + if rsync_path: + rsync_path = '"%s"' % rsync_path + + # FIXME: noop stuff still needs to be figured out + #module_args = "" + #if self.runner.noop_on_check(task_vars): + # module_args = "CHECKMODE=True" + + # run the module and store the result + result = self.runner._execute_module('synchronize', module_args=, complex_args=options, task_vars=task_vars) + + return result + diff --git a/v2/ansible/plugins/action/template.py b/v2/ansible/plugins/action/template.py new file mode 100644 index 0000000000..56cd5bbcd0 --- /dev/null +++ b/v2/ansible/plugins/action/template.py @@ -0,0 +1,164 @@ +# (c) 2015, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import base64 +import os + +from ansible.plugins.action import ActionBase +from ansible.template import Templar +from ansible.utils.hashing import checksum_s + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def run(self, tmp=None, task_vars=dict()): + ''' handler for template operations ''' + + source = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + + if (source is None and 'first_available_file' not in task_vars) or dest is None: + return dict(failed=True, msg="src and dest are required") + + if tmp is None: + tmp = self._make_tmp_path() + + ################################################################################################## + # FIXME: this all needs to be sorted out + ################################################################################################## + # if we have first_available_file in our vars + # look up the files and use the first one we find as src + #if 'first_available_file' in task_vars: + # found = False + # for fn in task_vars.get('first_available_file'): + # fn_orig = fn + # fnt = template.template(self.runner.basedir, fn, task_vars) + # fnd = utils.path_dwim(self.runner.basedir, fnt) + # if not os.path.exists(fnd) and '_original_file' in task_vars: + # fnd = utils.path_dwim_relative(task_vars['_original_file'], 'templates', fnt, self.runner.basedir, check=False) + # if os.path.exists(fnd): + # source = fnd + # found = True + # break + # if not found: + # result = dict(failed=True, msg="could not find src in first_available_file list") + # return ReturnData(conn=conn, comm_ok=False, result=result) + #else: + if 1: + if self._task._role is not None: + source = self._loader.path_dwim_relative(self._task._role._role_path, 'templates', source) + else: + source = self._loader.path_dwim(source) + ################################################################################################## + # END FIXME + ################################################################################################## + + # Expand any user home dir specification + dest = self._remote_expand_user(dest, tmp) + + if dest.endswith("/"): # CCTODO: Fix path for Windows hosts. + base = os.path.basename(source) + dest = os.path.join(dest, base) + + # template the source data locally & get ready to transfer + templar = Templar(loader=self._loader, variables=task_vars) + try: + with open(source, 'r') as f: + template_data = f.read() + resultant = templar.template(template_data, preserve_trailing_newlines=True) + except Exception, e: + return dict(failed=True, msg=type(e).__name__ + ": " + str(e)) + + local_checksum = checksum_s(resultant) + remote_checksum = self._remote_checksum(tmp, dest) + + if remote_checksum in ('0', '2', '3', '4'): + # Note: 1 means the file is not present which is fine; template will create it + return dict(failed=True, msg="failed to checksum remote file. Checksum error code: %s" % remote_checksum) + + if local_checksum != remote_checksum: + # if showing diffs, we need to get the remote value + dest_contents = '' + + # FIXME: still need to implement diff mechanism + #if self.runner.diff: + # # using persist_files to keep the temp directory around to avoid needing to grab another + # dest_result = self.runner._execute_module(conn, tmp, 'slurp', "path=%s" % dest, task_vars=task_vars, persist_files=True) + # if 'content' in dest_result.result: + # dest_contents = dest_result.result['content'] + # if dest_result.result['encoding'] == 'base64': + # dest_contents = base64.b64decode(dest_contents) + # else: + # raise Exception("unknown encoding, failed: %s" % dest_result.result) + + xfered = self._transfer_data(self._shell.join_path(tmp, 'source'), resultant) + + # fix file permissions when the copy is done as a different user + if self._connection_info.sudo and self._connection_info.sudo_user != 'root' or self._connection_info.su and self._connection_info.su_user != 'root': + self._remote_chmod('a+r', xfered, tmp) + + # run the copy module + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=xfered, + dest=dest, + original_basename=os.path.basename(source), + follow=True, + ), + ) + + # FIXME: noop stuff needs to be sorted out + #if self.runner.noop_on_check(task_vars): + # return ReturnData(conn=conn, comm_ok=True, result=dict(changed=True), diff=dict(before_header=dest, after_header=source, before=dest_contents, after=resultant)) + #else: + # res = self.runner._execute_module(conn, tmp, 'copy', module_args_tmp, task_vars=task_vars, complex_args=complex_args) + # if res.result.get('changed', False): + # res.diff = dict(before=dest_contents, after=resultant) + # return res + + result = self._execute_module(module_name='copy', module_args=new_module_args) + if result.get('changed', False): + result['diff'] = dict(before=dest_contents, after=resultant) + return result + + else: + # when running the file module based on the template data, we do + # not want the source filename (the name of the template) to be used, + # since this would mess up links, so we clear the src param and tell + # the module to follow links. When doing that, we have to set + # original_basename to the template just in case the dest is + # a directory. + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=None, + original_basename=os.path.basename(source), + follow=True, + ), + ) + + # FIXME: this may not be required anymore, as the checkmod params + # should be in the regular module args? + # be sure to task_vars the check mode param into the module args and + # rely on the file module to report its changed status + #if self.runner.noop_on_check(task_vars): + # new_module_args['CHECKMODE'] = True + + return self._execute_module(module_name='file', module_args=new_module_args) + diff --git a/v2/ansible/plugins/action/unarchive.py b/v2/ansible/plugins/action/unarchive.py new file mode 100644 index 0000000000..fab0843e9f --- /dev/null +++ b/v2/ansible/plugins/action/unarchive.py @@ -0,0 +1,118 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2013, Dylan Martin <dmartin@seattlecentral.edu> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os + +from ansible.plugins.action import ActionBase + +## fixes https://github.com/ansible/ansible/issues/3518 +# http://mypy.pythonblogs.com/12_mypy/archive/1253_workaround_for_python_bug_ascii_codec_cant_encode_character_uxa0_in_position_111_ordinal_not_in_range128.html +import sys +reload(sys) +sys.setdefaultencoding("utf8") +import pipes + + +class ActionModule(ActionBase): + + TRANSFERS_FILES = True + + def run(self, tmp=None, task_vars=dict()): + ''' handler for unarchive operations ''' + + source = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + copy = self._task.args.get('copy', True) + creates = self._task.args.get('creates', None) + + if source is None or dest is None: + return dict(failed=True, msg="src (or content) and dest are required") + + if not tmp: + tmp = self._make_tmp_path() + + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + module_args_tmp = "path=%s" % creates + result = self._execute_module(module_name='stat', module_args=dict(path=creates)) + stat = result.get('stat', None) + if stat and stat.get('exists', False): + return dict(skipped=True, msg=("skipped, since %s exists" % creates)) + + dest = self._remote_expand_user(dest, tmp) # CCTODO: Fix path for Windows hosts. + source = os.path.expanduser(source) + + if copy: + # FIXME: the original file stuff needs to be reworked + if '_original_file' in task_vars: + source = self._loader.path_dwim_relative(task_vars['_original_file'], 'files', source) + else: + source = self._loader.path_dwim(source) + + remote_checksum = self._remote_checksum(tmp, dest) + if remote_checksum != '3': + return dict(failed=True, msg="dest '%s' must be an existing dir" % dest) + elif remote_checksum == '4': + return dict(failed=True, msg="python isn't present on the system. Unable to compute checksum") + + if copy: + # transfer the file to a remote tmp location + tmp_src = tmp + 'source' + self._connection.put_file(source, tmp_src) + + # handle diff mode client side + # handle check mode client side + # fix file permissions when the copy is done as a different user + if copy: + if self._connection_info.sudo and self._connection_info.sudo_user != 'root' or self._connection_info.su and self._connection_info.su_user != 'root': + # FIXME: noop stuff needs to be reworked + #if not self.runner.noop_on_check(task_vars): + # self.runner._remote_chmod(conn, 'a+r', tmp_src, tmp) + self._remote_chmod(tmp, 'a+r', tmp_src) + + # Build temporary module_args. + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + src=tmp_src, + original_basename=os.path.basename(source), + ), + ) + + # make sure checkmod is passed on correctly + # FIXME: noop again, probably doesn't need to be done here anymore? + #if self.runner.noop_on_check(task_vars): + # new_module_args['CHECKMODE'] = True + + else: + new_module_args = self._task.args.copy() + new_module_args.update( + dict( + original_basename=os.path.basename(source), + ), + ) + # make sure checkmod is passed on correctly + # FIXME: noop again, probably doesn't need to be done here anymore? + #if self.runner.noop_on_check(task_vars): + # module_args += " CHECKMODE=True" + + # execute the unarchive module now, with the updated args + return self._execute_module(module_args=new_module_args) + diff --git a/v2/ansible/plugins/callback/__init__.py b/v2/ansible/plugins/callback/__init__.py index 785fc45992..c6905229f9 100644 --- a/v2/ansible/plugins/callback/__init__.py +++ b/v2/ansible/plugins/callback/__init__.py @@ -19,3 +19,86 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible.utils.display import Display + +__all__ = ["CallbackBase"] + +class CallbackBase: + + ''' + This is a base ansible callback class that does nothing. New callbacks should + use this class as a base and override any callback methods they wish to execute + custom actions. + ''' + + # FIXME: the list of functions here needs to be updated once we have + # finalized the list of callback methods used in the default callback + + def __init__(self): + self._display = Display() + + def set_connection_info(self, conn_info): + # FIXME: this is a temporary hack, as the connection info object + # should be created early and passed down through objects + self._display._verbosity = conn_info.verbosity + + def on_any(self, *args, **kwargs): + pass + + def runner_on_failed(self, host, res, ignore_errors=False): + pass + + def runner_on_ok(self, host, res): + pass + + def runner_on_skipped(self, host, item=None): + pass + + def runner_on_unreachable(self, host, res): + pass + + def runner_on_no_hosts(self): + pass + + def runner_on_async_poll(self, host, res, jid, clock): + pass + + def runner_on_async_ok(self, host, res, jid): + pass + + def runner_on_async_failed(self, host, res, jid): + pass + + def playbook_on_start(self): + pass + + def playbook_on_notify(self, host, handler): + pass + + def playbook_on_no_hosts_matched(self): + pass + + def playbook_on_no_hosts_remaining(self): + pass + + def playbook_on_task_start(self, name, is_conditional): + pass + + def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): + pass + + def playbook_on_setup(self): + pass + + def playbook_on_import_for_host(self, host, imported_file): + pass + + def playbook_on_not_import_for_host(self, host, missing_file): + pass + + def playbook_on_play_start(self, name): + pass + + def playbook_on_stats(self, stats): + pass + diff --git a/v2/ansible/plugins/callback/default.py b/v2/ansible/plugins/callback/default.py new file mode 100644 index 0000000000..091def9427 --- /dev/null +++ b/v2/ansible/plugins/callback/default.py @@ -0,0 +1,131 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +from ansible.plugins.callback import CallbackBase + +class CallbackModule(CallbackBase): + + ''' + This is the default callback interface, which simply prints messages + to stdout when new callback events are received. + ''' + + def _print_banner(self, msg, color=None): + ''' + Prints a header-looking line with stars taking up to 80 columns + of width (3 columns, minimum) + ''' + msg = msg.strip() + star_len = (80 - len(msg)) + if star_len < 0: + star_len = 3 + stars = "*" * star_len + self._display.display("\n%s %s" % (msg, stars), color=color) + + def on_any(self, *args, **kwargs): + pass + + def runner_on_failed(self, task, result, ignore_errors=False): + self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), json.dumps(result._result, ensure_ascii=False)), color='red') + + def runner_on_ok(self, task, result): + + if result._result.get('changed', False): + msg = "changed: [%s]" % result._host.get_name() + color = 'yellow' + else: + msg = "ok: [%s]" % result._host.get_name() + color = 'green' + + if (self._display._verbosity > 0 or 'verbose_always' in result._result) and result._task.action != 'setup': + indent = None + if 'verbose_always' in result._result: + indent = 4 + del result._result['verbose_always'] + msg += " => %s" % json.dumps(result._result, indent=indent, ensure_ascii=False) + self._display.display(msg, color=color) + + def runner_on_skipped(self, task, result): + msg = "skipping: [%s]" % result._host.get_name() + if self._display._verbosity > 0 or 'verbose_always' in result._result: + indent = None + if 'verbose_always' in result._result: + indent = 4 + del result._result['verbose_always'] + msg += " => %s" % json.dumps(result._result, indent=indent, ensure_ascii=False) + self._display.display(msg, color='cyan') + + def runner_on_unreachable(self, task, result): + self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), result._result), color='red') + + def runner_on_no_hosts(self, task): + pass + + def runner_on_async_poll(self, host, res, jid, clock): + pass + + def runner_on_async_ok(self, host, res, jid): + pass + + def runner_on_async_failed(self, host, res, jid): + pass + + def playbook_on_start(self): + pass + + def playbook_on_notify(self, host, handler): + pass + + def playbook_on_no_hosts_matched(self): + self._display.display("skipping: no hosts matched", color='cyan') + + def playbook_on_no_hosts_remaining(self): + self._print_banner("NO MORE HOSTS LEFT") + + def playbook_on_task_start(self, name, is_conditional): + self._print_banner("TASK [%s]" % name.strip()) + + def playbook_on_cleanup_task_start(self, name): + self._print_banner("CLEANUP TASK [%s]" % name.strip()) + + def playbook_on_handler_task_start(self, name): + self._print_banner("RUNNING HANDLER [%s]" % name.strip()) + + def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): + pass + + def playbook_on_setup(self): + pass + + def playbook_on_import_for_host(self, host, imported_file): + pass + + def playbook_on_not_import_for_host(self, host, missing_file): + pass + + def playbook_on_play_start(self, name): + self._print_banner("PLAY [%s]" % name.strip()) + + def playbook_on_stats(self, stats): + pass + diff --git a/v2/ansible/plugins/callback/minimal.py b/v2/ansible/plugins/callback/minimal.py new file mode 100644 index 0000000000..0b20eee64d --- /dev/null +++ b/v2/ansible/plugins/callback/minimal.py @@ -0,0 +1,111 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +from ansible.plugins.callback import CallbackBase + + +class CallbackModule(CallbackBase): + + ''' + This is the default callback interface, which simply prints messages + to stdout when new callback events are received. + ''' + + def _print_banner(self, msg): + ''' + Prints a header-looking line with stars taking up to 80 columns + of width (3 columns, minimum) + ''' + msg = msg.strip() + star_len = (80 - len(msg)) + if star_len < 0: + star_len = 3 + stars = "*" * star_len + self._display.display("\n%s %s\n" % (msg, stars)) + + def on_any(self, *args, **kwargs): + pass + + def runner_on_failed(self, task, result, ignore_errors=False): + self._display.display("%s | FAILED! => %s" % (result._host.get_name(), result._result), color='red') + + def runner_on_ok(self, task, result): + self._display.display("%s | SUCCESS => %s" % (result._host.get_name(), json.dumps(result._result, indent=4)), color='green') + + def runner_on_skipped(self, task, result): + pass + + def runner_on_unreachable(self, task, result): + self._display.display("%s | UNREACHABLE!" % result._host.get_name(), color='yellow') + + def runner_on_no_hosts(self, task): + pass + + def runner_on_async_poll(self, host, res, jid, clock): + pass + + def runner_on_async_ok(self, host, res, jid): + pass + + def runner_on_async_failed(self, host, res, jid): + pass + + def playbook_on_start(self): + pass + + def playbook_on_notify(self, host, handler): + pass + + def playbook_on_no_hosts_matched(self): + pass + + def playbook_on_no_hosts_remaining(self): + pass + + def playbook_on_task_start(self, name, is_conditional): + pass + + def playbook_on_cleanup_task_start(self, name): + pass + + def playbook_on_handler_task_start(self, name): + pass + + def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): + pass + + def playbook_on_setup(self): + pass + + def playbook_on_import_for_host(self, host, imported_file): + pass + + def playbook_on_not_import_for_host(self, host, missing_file): + pass + + def playbook_on_play_start(self, name): + pass + + def playbook_on_stats(self, stats): + pass + diff --git a/v2/ansible/plugins/connections/__init__.py b/v2/ansible/plugins/connections/__init__.py index 785fc45992..8dbd808191 100644 --- a/v2/ansible/plugins/connections/__init__.py +++ b/v2/ansible/plugins/connections/__init__.py @@ -19,3 +19,24 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible import constants as C + +# FIXME: this object should be created upfront and passed through +# the entire chain of calls to here, as there are other things +# which may want to output display/logs too +from ansible.utils.display import Display + +__all__ = ['ConnectionBase'] + + +class ConnectionBase: + ''' + A base class for connections to contain common code. + ''' + + def __init__(self, host, connection_info, *args, **kwargs): + self._host = host + self._connection_info = connection_info + self._has_pipelining = False + self._display = Display(connection_info) + diff --git a/v2/ansible/plugins/connections/accelerate.py b/v2/ansible/plugins/connections/accelerate.py new file mode 100644 index 0000000000..a31124e119 --- /dev/null +++ b/v2/ansible/plugins/connections/accelerate.py @@ -0,0 +1,371 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import json +import os +import base64 +import socket +import struct +import time +from ansible.callbacks import vvv, vvvv +from ansible.errors import AnsibleError, AnsibleFileNotFound +from ansible.runner.connection_plugins.ssh import Connection as SSHConnection +from ansible.runner.connection_plugins.paramiko_ssh import Connection as ParamikoConnection +from ansible import utils +from ansible import constants + +# the chunk size to read and send, assuming mtu 1500 and +# leaving room for base64 (+33%) encoding and header (8 bytes) +# ((1400-8)/4)*3) = 1044 +# which leaves room for the TCP/IP header. We set this to a +# multiple of the value to speed up file reads. +CHUNK_SIZE=1044*20 + +class Connection(object): + ''' raw socket accelerated connection ''' + + def __init__(self, runner, host, port, user, password, private_key_file, *args, **kwargs): + + self.runner = runner + self.host = host + self.context = None + self.conn = None + self.user = user + self.key = utils.key_for_hostname(host) + self.port = port[0] + self.accport = port[1] + self.is_connected = False + self.has_pipelining = False + + if not self.port: + self.port = constants.DEFAULT_REMOTE_PORT + elif not isinstance(self.port, int): + self.port = int(self.port) + + if not self.accport: + self.accport = constants.ACCELERATE_PORT + elif not isinstance(self.accport, int): + self.accport = int(self.accport) + + if self.runner.original_transport == "paramiko": + self.ssh = ParamikoConnection( + runner=self.runner, + host=self.host, + port=self.port, + user=self.user, + password=password, + private_key_file=private_key_file + ) + else: + self.ssh = SSHConnection( + runner=self.runner, + host=self.host, + port=self.port, + user=self.user, + password=password, + private_key_file=private_key_file + ) + + if not getattr(self.ssh, 'shell', None): + self.ssh.shell = utils.plugins.shell_loader.get('sh') + + # attempt to work around shared-memory funness + if getattr(self.runner, 'aes_keys', None): + utils.AES_KEYS = self.runner.aes_keys + + def _execute_accelerate_module(self): + args = "password=%s port=%s minutes=%d debug=%d ipv6=%s" % ( + base64.b64encode(self.key.__str__()), + str(self.accport), + constants.ACCELERATE_DAEMON_TIMEOUT, + int(utils.VERBOSITY), + self.runner.accelerate_ipv6, + ) + if constants.ACCELERATE_MULTI_KEY: + args += " multi_key=yes" + inject = dict(password=self.key) + if getattr(self.runner, 'accelerate_inventory_host', False): + inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.runner.accelerate_inventory_host)) + else: + inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.host)) + vvvv("attempting to start up the accelerate daemon...") + self.ssh.connect() + tmp_path = self.runner._make_tmp_path(self.ssh) + return self.runner._execute_module(self.ssh, tmp_path, 'accelerate', args, inject=inject) + + def connect(self, allow_ssh=True): + ''' activates the connection object ''' + + try: + if not self.is_connected: + wrong_user = False + tries = 3 + self.conn = socket.socket() + self.conn.settimeout(constants.ACCELERATE_CONNECT_TIMEOUT) + vvvv("attempting connection to %s via the accelerated port %d" % (self.host,self.accport)) + while tries > 0: + try: + self.conn.connect((self.host,self.accport)) + break + except socket.error: + vvvv("connection to %s failed, retrying..." % self.host) + time.sleep(0.1) + tries -= 1 + if tries == 0: + vvv("Could not connect via the accelerated connection, exceeded # of tries") + raise AnsibleError("FAILED") + elif wrong_user: + vvv("Restarting daemon with a different remote_user") + raise AnsibleError("WRONG_USER") + + self.conn.settimeout(constants.ACCELERATE_TIMEOUT) + if not self.validate_user(): + # the accelerated daemon was started with a + # different remote_user. The above command + # should have caused the accelerate daemon to + # shutdown, so we'll reconnect. + wrong_user = True + + except AnsibleError, e: + if allow_ssh: + if "WRONG_USER" in e: + vvv("Switching users, waiting for the daemon on %s to shutdown completely..." % self.host) + time.sleep(5) + vvv("Falling back to ssh to startup accelerated mode") + res = self._execute_accelerate_module() + if not res.is_successful(): + raise AnsibleError("Failed to launch the accelerated daemon on %s (reason: %s)" % (self.host,res.result.get('msg'))) + return self.connect(allow_ssh=False) + else: + raise AnsibleError("Failed to connect to %s:%s" % (self.host,self.accport)) + self.is_connected = True + return self + + def send_data(self, data): + packed_len = struct.pack('!Q',len(data)) + return self.conn.sendall(packed_len + data) + + def recv_data(self): + header_len = 8 # size of a packed unsigned long long + data = b"" + try: + vvvv("%s: in recv_data(), waiting for the header" % self.host) + while len(data) < header_len: + d = self.conn.recv(header_len - len(data)) + if not d: + vvvv("%s: received nothing, bailing out" % self.host) + return None + data += d + vvvv("%s: got the header, unpacking" % self.host) + data_len = struct.unpack('!Q',data[:header_len])[0] + data = data[header_len:] + vvvv("%s: data received so far (expecting %d): %d" % (self.host,data_len,len(data))) + while len(data) < data_len: + d = self.conn.recv(data_len - len(data)) + if not d: + vvvv("%s: received nothing, bailing out" % self.host) + return None + vvvv("%s: received %d bytes" % (self.host, len(d))) + data += d + vvvv("%s: received all of the data, returning" % self.host) + return data + except socket.timeout: + raise AnsibleError("timed out while waiting to receive data") + + def validate_user(self): + ''' + Checks the remote uid of the accelerated daemon vs. the + one specified for this play and will cause the accel + daemon to exit if they don't match + ''' + + vvvv("%s: sending request for validate_user" % self.host) + data = dict( + mode='validate_user', + username=self.user, + ) + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + if self.send_data(data): + raise AnsibleError("Failed to send command to %s" % self.host) + + vvvv("%s: waiting for validate_user response" % self.host) + while True: + # we loop here while waiting for the response, because a + # long running command may cause us to receive keepalive packets + # ({"pong":"true"}) rather than the response we want. + response = self.recv_data() + if not response: + raise AnsibleError("Failed to get a response from %s" % self.host) + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + if "pong" in response: + # it's a keepalive, go back to waiting + vvvv("%s: received a keepalive packet" % self.host) + continue + else: + vvvv("%s: received the validate_user response: %s" % (self.host, response)) + break + + if response.get('failed'): + return False + else: + return response.get('rc') == 0 + + def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None): + ''' run a command on the remote host ''' + + if su or su_user: + raise AnsibleError("Internal Error: this module does not support running commands via su") + + if in_data: + raise AnsibleError("Internal Error: this module does not support optimized module pipelining") + + if executable == "": + executable = constants.DEFAULT_EXECUTABLE + + if self.runner.sudo and sudoable and sudo_user: + cmd, prompt, success_key = utils.make_sudo_cmd(self.runner.sudo_exe, sudo_user, executable, cmd) + + vvv("EXEC COMMAND %s" % cmd) + + data = dict( + mode='command', + cmd=cmd, + tmp_path=tmp_path, + executable=executable, + ) + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + if self.send_data(data): + raise AnsibleError("Failed to send command to %s" % self.host) + + while True: + # we loop here while waiting for the response, because a + # long running command may cause us to receive keepalive packets + # ({"pong":"true"}) rather than the response we want. + response = self.recv_data() + if not response: + raise AnsibleError("Failed to get a response from %s" % self.host) + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + if "pong" in response: + # it's a keepalive, go back to waiting + vvvv("%s: received a keepalive packet" % self.host) + continue + else: + vvvv("%s: received the response" % self.host) + break + + return (response.get('rc',None), '', response.get('stdout',''), response.get('stderr','')) + + def put_file(self, in_path, out_path): + + ''' transfer a file from local to remote ''' + vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + + if not os.path.exists(in_path): + raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) + + fd = file(in_path, 'rb') + fstat = os.stat(in_path) + try: + vvv("PUT file is %d bytes" % fstat.st_size) + last = False + while fd.tell() <= fstat.st_size and not last: + vvvv("file position currently %ld, file size is %ld" % (fd.tell(), fstat.st_size)) + data = fd.read(CHUNK_SIZE) + if fd.tell() >= fstat.st_size: + last = True + data = dict(mode='put', data=base64.b64encode(data), out_path=out_path, last=last) + if self.runner.sudo: + data['user'] = self.runner.sudo_user + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + + if self.send_data(data): + raise AnsibleError("failed to send the file to %s" % self.host) + + response = self.recv_data() + if not response: + raise AnsibleError("Failed to get a response from %s" % self.host) + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + + if response.get('failed',False): + raise AnsibleError("failed to put the file in the requested location") + finally: + fd.close() + vvvv("waiting for final response after PUT") + response = self.recv_data() + if not response: + raise AnsibleError("Failed to get a response from %s" % self.host) + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + + if response.get('failed',False): + raise AnsibleError("failed to put the file in the requested location") + + def fetch_file(self, in_path, out_path): + ''' save a remote file to the specified path ''' + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + + data = dict(mode='fetch', in_path=in_path) + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + if self.send_data(data): + raise AnsibleError("failed to initiate the file fetch with %s" % self.host) + + fh = open(out_path, "w") + try: + bytes = 0 + while True: + response = self.recv_data() + if not response: + raise AnsibleError("Failed to get a response from %s" % self.host) + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + if response.get('failed', False): + raise AnsibleError("Error during file fetch, aborting") + out = base64.b64decode(response['data']) + fh.write(out) + bytes += len(out) + # send an empty response back to signify we + # received the last chunk without errors + data = utils.jsonify(dict()) + data = utils.encrypt(self.key, data) + if self.send_data(data): + raise AnsibleError("failed to send ack during file fetch") + if response.get('last', False): + break + finally: + # we don't currently care about this final response, + # we just receive it and drop it. It may be used at some + # point in the future or we may just have the put/fetch + # operations not send back a final response at all + response = self.recv_data() + vvv("FETCH wrote %d bytes to %s" % (bytes, out_path)) + fh.close() + + def close(self): + ''' terminate the connection ''' + # Be a good citizen + try: + self.conn.close() + except: + pass + diff --git a/v2/ansible/plugins/connections/chroot.py b/v2/ansible/plugins/connections/chroot.py new file mode 100644 index 0000000000..38c8af7a69 --- /dev/null +++ b/v2/ansible/plugins/connections/chroot.py @@ -0,0 +1,130 @@ +# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# (c) 2013, Maykel Moya <mmoya@speedyrails.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import distutils.spawn +import traceback +import os +import shutil +import subprocess +from ansible import errors +from ansible import utils +from ansible.callbacks import vvv + +class Connection(object): + ''' Local chroot based connections ''' + + def __init__(self, runner, host, port, *args, **kwargs): + self.chroot = host + self.has_pipelining = False + + if os.geteuid() != 0: + raise errors.AnsibleError("chroot connection requires running as root") + + # we're running as root on the local system so do some + # trivial checks for ensuring 'host' is actually a chroot'able dir + if not os.path.isdir(self.chroot): + raise errors.AnsibleError("%s is not a directory" % self.chroot) + + chrootsh = os.path.join(self.chroot, 'bin/sh') + if not utils.is_executable(chrootsh): + raise errors.AnsibleError("%s does not look like a chrootable dir (/bin/sh missing)" % self.chroot) + + self.chroot_cmd = distutils.spawn.find_executable('chroot') + if not self.chroot_cmd: + raise errors.AnsibleError("chroot command not found in PATH") + + self.runner = runner + self.host = host + # port is unused, since this is local + self.port = port + + def connect(self, port=None): + ''' connect to the chroot; nothing to do here ''' + + vvv("THIS IS A LOCAL CHROOT DIR", host=self.chroot) + + return self + + def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None): + ''' run a command on the chroot ''' + + if su or su_user: + raise errors.AnsibleError("Internal Error: this module does not support running commands via su") + + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + + # We enter chroot as root so sudo stuff can be ignored + + if executable: + local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd] + else: + local_cmd = '%s "%s" %s' % (self.chroot_cmd, self.chroot, cmd) + + vvv("EXEC %s" % (local_cmd), host=self.chroot) + p = subprocess.Popen(local_cmd, shell=isinstance(local_cmd, basestring), + cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, stderr = p.communicate() + return (p.returncode, '', stdout, stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to chroot ''' + + if not out_path.startswith(os.path.sep): + out_path = os.path.join(os.path.sep, out_path) + normpath = os.path.normpath(out_path) + out_path = os.path.join(self.chroot, normpath[1:]) + + vvv("PUT %s TO %s" % (in_path, out_path), host=self.chroot) + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + try: + shutil.copyfile(in_path, out_path) + except shutil.Error: + traceback.print_exc() + raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path)) + except IOError: + traceback.print_exc() + raise errors.AnsibleError("failed to transfer file to %s" % out_path) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from chroot to local ''' + + if not in_path.startswith(os.path.sep): + in_path = os.path.join(os.path.sep, in_path) + normpath = os.path.normpath(in_path) + in_path = os.path.join(self.chroot, normpath[1:]) + + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.chroot) + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + try: + shutil.copyfile(in_path, out_path) + except shutil.Error: + traceback.print_exc() + raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path)) + except IOError: + traceback.print_exc() + raise errors.AnsibleError("failed to transfer file to %s" % out_path) + + def close(self): + ''' terminate the connection; nothing to do here ''' + pass diff --git a/v2/ansible/plugins/connections/fireball.py b/v2/ansible/plugins/connections/fireball.py new file mode 100644 index 0000000000..dd9e09bacd --- /dev/null +++ b/v2/ansible/plugins/connections/fireball.py @@ -0,0 +1,151 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import json +import os +import base64 +from ansible.callbacks import vvv +from ansible import utils +from ansible import errors +from ansible import constants + +HAVE_ZMQ=False + +try: + import zmq + HAVE_ZMQ=True +except ImportError: + pass + +class Connection(object): + ''' ZeroMQ accelerated connection ''' + + def __init__(self, runner, host, port, *args, **kwargs): + + self.runner = runner + self.has_pipelining = False + + # attempt to work around shared-memory funness + if getattr(self.runner, 'aes_keys', None): + utils.AES_KEYS = self.runner.aes_keys + + self.host = host + self.key = utils.key_for_hostname(host) + self.context = None + self.socket = None + + if port is None: + self.port = constants.ZEROMQ_PORT + else: + self.port = port + + def connect(self): + ''' activates the connection object ''' + + if not HAVE_ZMQ: + raise errors.AnsibleError("zmq is not installed") + + # this is rough/temporary and will likely be optimized later ... + self.context = zmq.Context() + socket = self.context.socket(zmq.REQ) + addr = "tcp://%s:%s" % (self.host, self.port) + socket.connect(addr) + self.socket = socket + + return self + + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None, su_user=None, su=None): + ''' run a command on the remote host ''' + + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + + vvv("EXEC COMMAND %s" % cmd) + + if (self.runner.sudo and sudoable) or (self.runner.su and su): + raise errors.AnsibleError( + "When using fireball, do not specify sudo or su to run your tasks. " + + "Instead sudo the fireball action with sudo. " + + "Task will communicate with the fireball already running in sudo mode." + ) + + data = dict( + mode='command', + cmd=cmd, + tmp_path=tmp_path, + executable=executable, + ) + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + self.socket.send(data) + + response = self.socket.recv() + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + + return (response.get('rc',None), '', response.get('stdout',''), response.get('stderr','')) + + def put_file(self, in_path, out_path): + + ''' transfer a file from local to remote ''' + vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + data = file(in_path).read() + data = base64.b64encode(data) + + data = dict(mode='put', data=data, out_path=out_path) + # TODO: support chunked file transfer + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + self.socket.send(data) + + response = self.socket.recv() + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + + # no meaningful response needed for this + + def fetch_file(self, in_path, out_path): + ''' save a remote file to the specified path ''' + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + + data = dict(mode='fetch', in_path=in_path) + data = utils.jsonify(data) + data = utils.encrypt(self.key, data) + self.socket.send(data) + + response = self.socket.recv() + response = utils.decrypt(self.key, response) + response = utils.parse_json(response) + response = response['data'] + response = base64.b64decode(response) + + fh = open(out_path, "w") + fh.write(response) + fh.close() + + def close(self): + ''' terminate the connection ''' + # Be a good citizen + try: + self.socket.close() + self.context.term() + except: + pass + diff --git a/v2/ansible/plugins/connections/funcd.py b/v2/ansible/plugins/connections/funcd.py new file mode 100644 index 0000000000..7244abcbe9 --- /dev/null +++ b/v2/ansible/plugins/connections/funcd.py @@ -0,0 +1,99 @@ +# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# Based on chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com> +# (c) 2013, Michael Scherer <misc@zarb.org> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# --- +# The func transport permit to use ansible over func. For people who have already setup +# func and that wish to play with ansible, this permit to move gradually to ansible +# without having to redo completely the setup of the network. + +HAVE_FUNC=False +try: + import func.overlord.client as fc + HAVE_FUNC=True +except ImportError: + pass + +import os +from ansible.callbacks import vvv +from ansible import errors +import tempfile +import shutil + + +class Connection(object): + ''' Func-based connections ''' + + def __init__(self, runner, host, port, *args, **kwargs): + self.runner = runner + self.host = host + self.has_pipelining = False + # port is unused, this go on func + self.port = port + + def connect(self, port=None): + if not HAVE_FUNC: + raise errors.AnsibleError("func is not installed") + + self.client = fc.Client(self.host) + return self + + def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, + executable='/bin/sh', in_data=None, su=None, su_user=None): + ''' run a command on the remote minion ''' + + if su or su_user: + raise errors.AnsibleError("Internal Error: this module does not support running commands via su") + + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + + vvv("EXEC %s" % (cmd), host=self.host) + p = self.client.command.run(cmd)[self.host] + return (p[0], '', p[1], p[2]) + + def _normalize_path(self, path, prefix): + if not path.startswith(os.path.sep): + path = os.path.join(os.path.sep, path) + normpath = os.path.normpath(path) + return os.path.join(prefix, normpath[1:]) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to remote ''' + + out_path = self._normalize_path(out_path, '/') + vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + self.client.local.copyfile.send(in_path, out_path) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from remote to local ''' + + in_path = self._normalize_path(in_path, '/') + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + # need to use a tmp dir due to difference of semantic for getfile + # ( who take a # directory as destination) and fetch_file, who + # take a file directly + tmpdir = tempfile.mkdtemp(prefix="func_ansible") + self.client.local.getfile.get(in_path, tmpdir) + shutil.move(os.path.join(tmpdir, self.host, os.path.basename(in_path)), + out_path) + shutil.rmtree(tmpdir) + + def close(self): + ''' terminate the connection; nothing to do here ''' + pass diff --git a/v2/ansible/plugins/connections/jail.py b/v2/ansible/plugins/connections/jail.py new file mode 100644 index 0000000000..b721ad62b5 --- /dev/null +++ b/v2/ansible/plugins/connections/jail.py @@ -0,0 +1,151 @@ +# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# and chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com> +# (c) 2013, Michael Scherer <misc@zarb.org> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import distutils.spawn +import traceback +import os +import shutil +import subprocess +from ansible import errors +from ansible.callbacks import vvv + +class Connection(object): + ''' Local chroot based connections ''' + + def _search_executable(self, executable): + cmd = distutils.spawn.find_executable(executable) + if not cmd: + raise errors.AnsibleError("%s command not found in PATH") % executable + return cmd + + def list_jails(self): + p = subprocess.Popen([self.jls_cmd, '-q', 'name'], + cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, stderr = p.communicate() + + return stdout.split() + + def get_jail_path(self): + p = subprocess.Popen([self.jls_cmd, '-j', self.jail, '-q', 'path'], + cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, stderr = p.communicate() + # remove \n + return stdout[:-1] + + + + def __init__(self, runner, host, port, *args, **kwargs): + self.jail = host + self.runner = runner + self.host = host + self.has_pipelining = False + + if os.geteuid() != 0: + raise errors.AnsibleError("jail connection requires running as root") + + self.jls_cmd = self._search_executable('jls') + self.jexec_cmd = self._search_executable('jexec') + + if not self.jail in self.list_jails(): + raise errors.AnsibleError("incorrect jail name %s" % self.jail) + + + self.host = host + # port is unused, since this is local + self.port = port + + def connect(self, port=None): + ''' connect to the chroot; nothing to do here ''' + + vvv("THIS IS A LOCAL CHROOT DIR", host=self.jail) + + return self + + # a modifier + def _generate_cmd(self, executable, cmd): + if executable: + local_cmd = [self.jexec_cmd, self.jail, executable, '-c', cmd] + else: + local_cmd = '%s "%s" %s' % (self.jexec_cmd, self.jail, cmd) + return local_cmd + + def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None): + ''' run a command on the chroot ''' + + if su or su_user: + raise errors.AnsibleError("Internal Error: this module does not support running commands via su") + + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + + # We enter chroot as root so sudo stuff can be ignored + local_cmd = self._generate_cmd(executable, cmd) + + vvv("EXEC %s" % (local_cmd), host=self.jail) + p = subprocess.Popen(local_cmd, shell=isinstance(local_cmd, basestring), + cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, stderr = p.communicate() + return (p.returncode, '', stdout, stderr) + + def _normalize_path(self, path, prefix): + if not path.startswith(os.path.sep): + path = os.path.join(os.path.sep, path) + normpath = os.path.normpath(path) + return os.path.join(prefix, normpath[1:]) + + def _copy_file(self, in_path, out_path): + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + try: + shutil.copyfile(in_path, out_path) + except shutil.Error: + traceback.print_exc() + raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path)) + except IOError: + traceback.print_exc() + raise errors.AnsibleError("failed to transfer file to %s" % out_path) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to chroot ''' + + out_path = self._normalize_path(out_path, self.get_jail_path()) + vvv("PUT %s TO %s" % (in_path, out_path), host=self.jail) + + self._copy_file(in_path, out_path) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from chroot to local ''' + + in_path = self._normalize_path(in_path, self.get_jail_path()) + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.jail) + + self._copy_file(in_path, out_path) + + def close(self): + ''' terminate the connection; nothing to do here ''' + pass diff --git a/v2/ansible/plugins/connections/libvirt_lxc.py b/v2/ansible/plugins/connections/libvirt_lxc.py new file mode 100644 index 0000000000..c6cf11f266 --- /dev/null +++ b/v2/ansible/plugins/connections/libvirt_lxc.py @@ -0,0 +1,127 @@ +# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# Based on chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com> +# (c) 2013, Michael Scherer <misc@zarb.org> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import distutils.spawn +import os +import subprocess +from ansible import errors +from ansible.callbacks import vvv + +class Connection(object): + ''' Local lxc based connections ''' + + def _search_executable(self, executable): + cmd = distutils.spawn.find_executable(executable) + if not cmd: + raise errors.AnsibleError("%s command not found in PATH") % executable + return cmd + + def _check_domain(self, domain): + p = subprocess.Popen([self.cmd, '-q', '-c', 'lxc:///', 'dominfo', domain], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.communicate() + if p.returncode: + raise errors.AnsibleError("%s is not a lxc defined in libvirt" % domain) + + def __init__(self, runner, host, port, *args, **kwargs): + self.lxc = host + + self.cmd = self._search_executable('virsh') + + self._check_domain(host) + + self.runner = runner + self.host = host + # port is unused, since this is local + self.port = port + + def connect(self, port=None): + ''' connect to the lxc; nothing to do here ''' + + vvv("THIS IS A LOCAL LXC DIR", host=self.lxc) + + return self + + def _generate_cmd(self, executable, cmd): + if executable: + local_cmd = [self.cmd, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', executable , '-c', cmd] + else: + local_cmd = '%s -q -c lxc:/// lxc-enter-namespace %s -- %s' % (self.cmd, self.lxc, cmd) + return local_cmd + + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None): + ''' run a command on the chroot ''' + + if su or su_user: + raise errors.AnsibleError("Internal Error: this module does not support running commands via su") + + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + + # We enter lxc as root so sudo stuff can be ignored + local_cmd = self._generate_cmd(executable, cmd) + + vvv("EXEC %s" % (local_cmd), host=self.lxc) + p = subprocess.Popen(local_cmd, shell=isinstance(local_cmd, basestring), + cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, stderr = p.communicate() + return (p.returncode, '', stdout, stderr) + + def _normalize_path(self, path, prefix): + if not path.startswith(os.path.sep): + path = os.path.join(os.path.sep, path) + normpath = os.path.normpath(path) + return os.path.join(prefix, normpath[1:]) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to lxc ''' + + out_path = self._normalize_path(out_path, '/') + vvv("PUT %s TO %s" % (in_path, out_path), host=self.lxc) + + local_cmd = [self.cmd, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', '/bin/tee', out_path] + vvv("EXEC %s" % (local_cmd), host=self.lxc) + + p = subprocess.Popen(local_cmd, cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate(open(in_path,'rb').read()) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from lxc to local ''' + + in_path = self._normalize_path(in_path, '/') + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.lxc) + + local_cmd = [self.cmd, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', '/bin/cat', in_path] + vvv("EXEC %s" % (local_cmd), host=self.lxc) + + p = subprocess.Popen(local_cmd, cwd=self.runner.basedir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + open(out_path,'wb').write(stdout) + + + def close(self): + ''' terminate the connection; nothing to do here ''' + pass diff --git a/v2/ansible/plugins/connections/local.py b/v2/ansible/plugins/connections/local.py new file mode 100644 index 0000000000..58e8a20a2e --- /dev/null +++ b/v2/ansible/plugins/connections/local.py @@ -0,0 +1,139 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import traceback +import os +import pipes +import shutil +import subprocess +import select +import fcntl + +from ansible.errors import AnsibleError +from ansible.plugins.connections import ConnectionBase + +from ansible.utils.debug import debug + +class Connection(ConnectionBase): + ''' Local based connections ''' + + def get_transport(self): + ''' used to identify this connection object ''' + return 'local' + + def connect(self, port=None): + ''' connect to the local host; nothing to do here ''' + return self + + def exec_command(self, cmd, tmp_path, executable='/bin/sh', in_data=None): + ''' run a command on the local host ''' + + debug("in local.exec_command()") + # su requires to be run from a terminal, and therefore isn't supported here (yet?) + if self._connection_info.su: + raise AnsibleError("Internal Error: this module does not support running commands via su") + + if in_data: + raise AnsibleError("Internal Error: this module does not support optimized module pipelining") + + # FIXME: su/sudo stuff needs to be generalized + #if not self.runner.sudo or not sudoable: + # if executable: + # local_cmd = executable.split() + ['-c', cmd] + # else: + # local_cmd = cmd + #else: + # local_cmd, prompt, success_key = utils.make_sudo_cmd(self.runner.sudo_exe, sudo_user, executable, cmd) + if executable: + local_cmd = executable.split() + ['-c', cmd] + else: + local_cmd = cmd + + executable = executable.split()[0] if executable else None + + self._display.vvv("%s EXEC %s" % (self._host, local_cmd)) + # FIXME: cwd= needs to be set to the basedir of the playbook + debug("opening command with Popen()") + p = subprocess.Popen( + local_cmd, + shell=isinstance(local_cmd, basestring), + executable=executable, #cwd=... + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + debug("done running command with Popen()") + + # FIXME: more su/sudo stuff + #if self.runner.sudo and sudoable and self.runner.sudo_pass: + # fcntl.fcntl(p.stdout, fcntl.F_SETFL, + # fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) + # fcntl.fcntl(p.stderr, fcntl.F_SETFL, + # fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK) + # sudo_output = '' + # while not sudo_output.endswith(prompt) and success_key not in sudo_output: + # rfd, wfd, efd = select.select([p.stdout, p.stderr], [], + # [p.stdout, p.stderr], self.runner.timeout) + # if p.stdout in rfd: + # chunk = p.stdout.read() + # elif p.stderr in rfd: + # chunk = p.stderr.read() + # else: + # stdout, stderr = p.communicate() + # raise AnsibleError('timeout waiting for sudo password prompt:\n' + sudo_output) + # if not chunk: + # stdout, stderr = p.communicate() + # raise AnsibleError('sudo output closed while waiting for password prompt:\n' + sudo_output) + # sudo_output += chunk + # if success_key not in sudo_output: + # p.stdin.write(self.runner.sudo_pass + '\n') + # fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) + # fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK) + + debug("getting output with communicate()") + stdout, stderr = p.communicate() + debug("done communicating") + + debug("done with local.exec_command()") + return (p.returncode, '', stdout, stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to local ''' + + #vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + self._display.vvv("%s PUT %s TO %s" % (self._host, in_path, out_path)) + if not os.path.exists(in_path): + #raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) + raise AnsibleError("file or module does not exist: %s" % in_path) + try: + shutil.copyfile(in_path, out_path) + except shutil.Error: + traceback.print_exc() + raise AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path)) + except IOError: + traceback.print_exc() + raise AnsibleError("failed to transfer file to %s" % out_path) + + def fetch_file(self, in_path, out_path): + #vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + self._display.vvv("%s FETCH %s TO %s" % (self._host, in_path, out_path)) + ''' fetch a file from local to local -- for copatibility ''' + self.put_file(in_path, out_path) + + def close(self): + ''' terminate the connection; nothing to do here ''' + pass diff --git a/v2/ansible/plugins/connections/paramiko_ssh.py b/v2/ansible/plugins/connections/paramiko_ssh.py new file mode 100644 index 0000000000..4bb06e01c3 --- /dev/null +++ b/v2/ansible/plugins/connections/paramiko_ssh.py @@ -0,0 +1,417 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + + +# --- +# The paramiko transport is provided because many distributions, in particular EL6 and before +# do not support ControlPersist in their SSH implementations. This is needed on the Ansible +# control machine to be reasonably efficient with connections. Thus paramiko is faster +# for most users on these platforms. Users with ControlPersist capability can consider +# using -c ssh or configuring the transport in ansible.cfg. + +import warnings +import os +import pipes +import socket +import random +import logging +import tempfile +import traceback +import fcntl +import re +import sys +from termios import tcflush, TCIFLUSH +from binascii import hexlify +from ansible.callbacks import vvv +from ansible import errors +from ansible import utils +from ansible import constants as C + +AUTHENTICITY_MSG=""" +paramiko: The authenticity of host '%s' can't be established. +The %s key fingerprint is %s. +Are you sure you want to continue connecting (yes/no)? +""" + +# prevent paramiko warning noise -- see http://stackoverflow.com/questions/3920502/ +HAVE_PARAMIKO=False +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + import paramiko + HAVE_PARAMIKO=True + logging.getLogger("paramiko").setLevel(logging.WARNING) + except ImportError: + pass + +class MyAddPolicy(object): + """ + Based on AutoAddPolicy in paramiko so we can determine when keys are added + and also prompt for input. + + Policy for automatically adding the hostname and new host key to the + local L{HostKeys} object, and saving it. This is used by L{SSHClient}. + """ + + def __init__(self, runner): + self.runner = runner + + def missing_host_key(self, client, hostname, key): + + if C.HOST_KEY_CHECKING: + + fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) + fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX) + + old_stdin = sys.stdin + sys.stdin = self.runner._new_stdin + fingerprint = hexlify(key.get_fingerprint()) + ktype = key.get_name() + + # clear out any premature input on sys.stdin + tcflush(sys.stdin, TCIFLUSH) + + inp = raw_input(AUTHENTICITY_MSG % (hostname, ktype, fingerprint)) + sys.stdin = old_stdin + if inp not in ['yes','y','']: + fcntl.flock(self.runner.output_lockfile, fcntl.LOCK_UN) + fcntl.flock(self.runner.process_lockfile, fcntl.LOCK_UN) + raise errors.AnsibleError("host connection rejected by user") + + fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN) + fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN) + + + key._added_by_ansible_this_time = True + + # existing implementation below: + client._host_keys.add(hostname, key.get_name(), key) + + # host keys are actually saved in close() function below + # in order to control ordering. + + +# keep connection objects on a per host basis to avoid repeated attempts to reconnect + +SSH_CONNECTION_CACHE = {} +SFTP_CONNECTION_CACHE = {} + +class Connection(object): + ''' SSH based connections with Paramiko ''' + + def __init__(self, runner, host, port, user, password, private_key_file, *args, **kwargs): + + self.ssh = None + self.sftp = None + self.runner = runner + self.host = host + self.port = port or 22 + self.user = user + self.password = password + self.private_key_file = private_key_file + self.has_pipelining = False + + def _cache_key(self): + return "%s__%s__" % (self.host, self.user) + + def connect(self): + cache_key = self._cache_key() + if cache_key in SSH_CONNECTION_CACHE: + self.ssh = SSH_CONNECTION_CACHE[cache_key] + else: + self.ssh = SSH_CONNECTION_CACHE[cache_key] = self._connect_uncached() + return self + + def _connect_uncached(self): + ''' activates the connection object ''' + + if not HAVE_PARAMIKO: + raise errors.AnsibleError("paramiko is not installed") + + vvv("ESTABLISH CONNECTION FOR USER: %s on PORT %s TO %s" % (self.user, self.port, self.host), host=self.host) + + ssh = paramiko.SSHClient() + + self.keyfile = os.path.expanduser("~/.ssh/known_hosts") + + if C.HOST_KEY_CHECKING: + ssh.load_system_host_keys() + + ssh.set_missing_host_key_policy(MyAddPolicy(self.runner)) + + allow_agent = True + + if self.password is not None: + allow_agent = False + + try: + + if self.private_key_file: + key_filename = os.path.expanduser(self.private_key_file) + elif self.runner.private_key_file: + key_filename = os.path.expanduser(self.runner.private_key_file) + else: + key_filename = None + ssh.connect(self.host, username=self.user, allow_agent=allow_agent, look_for_keys=True, + key_filename=key_filename, password=self.password, + timeout=self.runner.timeout, port=self.port) + + except Exception, e: + + msg = str(e) + if "PID check failed" in msg: + raise errors.AnsibleError("paramiko version issue, please upgrade paramiko on the machine running ansible") + elif "Private key file is encrypted" in msg: + msg = 'ssh %s@%s:%s : %s\nTo connect as a different user, use -u <username>.' % ( + self.user, self.host, self.port, msg) + raise errors.AnsibleConnectionFailed(msg) + else: + raise errors.AnsibleConnectionFailed(msg) + + return ssh + + def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None): + ''' run a command on the remote host ''' + + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + + bufsize = 4096 + + try: + + self.ssh.get_transport().set_keepalive(5) + chan = self.ssh.get_transport().open_session() + + except Exception, e: + + msg = "Failed to open session" + if len(str(e)) > 0: + msg += ": %s" % str(e) + raise errors.AnsibleConnectionFailed(msg) + + no_prompt_out = '' + no_prompt_err = '' + if not (self.runner.sudo and sudoable) and not (self.runner.su and su): + + if executable: + quoted_command = executable + ' -c ' + pipes.quote(cmd) + else: + quoted_command = cmd + vvv("EXEC %s" % quoted_command, host=self.host) + chan.exec_command(quoted_command) + + else: + + # sudo usually requires a PTY (cf. requiretty option), therefore + # we give it one by default (pty=True in ansble.cfg), and we try + # to initialise from the calling environment + if C.PARAMIKO_PTY: + chan.get_pty(term=os.getenv('TERM', 'vt100'), + width=int(os.getenv('COLUMNS', 0)), + height=int(os.getenv('LINES', 0))) + if self.runner.sudo or sudoable: + shcmd, prompt, success_key = utils.make_sudo_cmd(self.runner.sudo_exe, sudo_user, executable, cmd) + elif self.runner.su or su: + shcmd, prompt, success_key = utils.make_su_cmd(su_user, executable, cmd) + + vvv("EXEC %s" % shcmd, host=self.host) + sudo_output = '' + + try: + + chan.exec_command(shcmd) + + if self.runner.sudo_pass or self.runner.su_pass: + + while True: + + if success_key in sudo_output or \ + (self.runner.sudo_pass and sudo_output.endswith(prompt)) or \ + (self.runner.su_pass and utils.su_prompts.check_su_prompt(sudo_output)): + break + chunk = chan.recv(bufsize) + + if not chunk: + if 'unknown user' in sudo_output: + raise errors.AnsibleError( + 'user %s does not exist' % sudo_user) + else: + raise errors.AnsibleError('ssh connection ' + + 'closed waiting for password prompt') + sudo_output += chunk + + if success_key not in sudo_output: + + if sudoable: + chan.sendall(self.runner.sudo_pass + '\n') + elif su: + chan.sendall(self.runner.su_pass + '\n') + else: + no_prompt_out += sudo_output + no_prompt_err += sudo_output + + except socket.timeout: + + raise errors.AnsibleError('ssh timed out waiting for sudo.\n' + sudo_output) + + stdout = ''.join(chan.makefile('rb', bufsize)) + stderr = ''.join(chan.makefile_stderr('rb', bufsize)) + + return (chan.recv_exit_status(), '', no_prompt_out + stdout, no_prompt_out + stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to remote ''' + + vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + + try: + self.sftp = self.ssh.open_sftp() + except Exception, e: + raise errors.AnsibleError("failed to open a SFTP connection (%s)" % e) + + try: + self.sftp.put(in_path, out_path) + except IOError: + raise errors.AnsibleError("failed to transfer file to %s" % out_path) + + def _connect_sftp(self): + + cache_key = "%s__%s__" % (self.host, self.user) + if cache_key in SFTP_CONNECTION_CACHE: + return SFTP_CONNECTION_CACHE[cache_key] + else: + result = SFTP_CONNECTION_CACHE[cache_key] = self.connect().ssh.open_sftp() + return result + + def fetch_file(self, in_path, out_path): + ''' save a remote file to the specified path ''' + + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + + try: + self.sftp = self._connect_sftp() + except Exception, e: + raise errors.AnsibleError("failed to open a SFTP connection (%s)", e) + + try: + self.sftp.get(in_path, out_path) + except IOError: + raise errors.AnsibleError("failed to transfer file from %s" % in_path) + + def _any_keys_added(self): + + added_any = False + for hostname, keys in self.ssh._host_keys.iteritems(): + for keytype, key in keys.iteritems(): + added_this_time = getattr(key, '_added_by_ansible_this_time', False) + if added_this_time: + return True + return False + + def _save_ssh_host_keys(self, filename): + ''' + not using the paramiko save_ssh_host_keys function as we want to add new SSH keys at the bottom so folks + don't complain about it :) + ''' + + if not self._any_keys_added(): + return False + + path = os.path.expanduser("~/.ssh") + if not os.path.exists(path): + os.makedirs(path) + + f = open(filename, 'w') + + for hostname, keys in self.ssh._host_keys.iteritems(): + + for keytype, key in keys.iteritems(): + + # was f.write + added_this_time = getattr(key, '_added_by_ansible_this_time', False) + if not added_this_time: + f.write("%s %s %s\n" % (hostname, keytype, key.get_base64())) + + for hostname, keys in self.ssh._host_keys.iteritems(): + + for keytype, key in keys.iteritems(): + added_this_time = getattr(key, '_added_by_ansible_this_time', False) + if added_this_time: + f.write("%s %s %s\n" % (hostname, keytype, key.get_base64())) + + f.close() + + def close(self): + ''' terminate the connection ''' + + cache_key = self._cache_key() + SSH_CONNECTION_CACHE.pop(cache_key, None) + SFTP_CONNECTION_CACHE.pop(cache_key, None) + + if self.sftp is not None: + self.sftp.close() + + if C.HOST_KEY_CHECKING and C.PARAMIKO_RECORD_HOST_KEYS and self._any_keys_added(): + + # add any new SSH host keys -- warning -- this could be slow + lockfile = self.keyfile.replace("known_hosts",".known_hosts.lock") + dirname = os.path.dirname(self.keyfile) + if not os.path.exists(dirname): + os.makedirs(dirname) + + KEY_LOCK = open(lockfile, 'w') + fcntl.lockf(KEY_LOCK, fcntl.LOCK_EX) + + try: + # just in case any were added recently + + self.ssh.load_system_host_keys() + self.ssh._host_keys.update(self.ssh._system_host_keys) + + # gather information about the current key file, so + # we can ensure the new file has the correct mode/owner + + key_dir = os.path.dirname(self.keyfile) + key_stat = os.stat(self.keyfile) + + # Save the new keys to a temporary file and move it into place + # rather than rewriting the file. We set delete=False because + # the file will be moved into place rather than cleaned up. + + tmp_keyfile = tempfile.NamedTemporaryFile(dir=key_dir, delete=False) + os.chmod(tmp_keyfile.name, key_stat.st_mode & 07777) + os.chown(tmp_keyfile.name, key_stat.st_uid, key_stat.st_gid) + + self._save_ssh_host_keys(tmp_keyfile.name) + tmp_keyfile.close() + + os.rename(tmp_keyfile.name, self.keyfile) + + except: + + # unable to save keys, including scenario when key was invalid + # and caught earlier + traceback.print_exc() + pass + fcntl.lockf(KEY_LOCK, fcntl.LOCK_UN) + + self.ssh.close() + diff --git a/v2/ansible/plugins/connections/ssh.py b/v2/ansible/plugins/connections/ssh.py new file mode 100644 index 0000000000..b8bbc5c46f --- /dev/null +++ b/v2/ansible/plugins/connections/ssh.py @@ -0,0 +1,493 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# + +import os +import re +import subprocess +import shlex +import pipes +import random +import select +import fcntl +import hmac +import pwd +import gettext +import pty +from hashlib import sha1 + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.plugins.connections import ConnectionBase + +class Connection(ConnectionBase): + ''' ssh based connections ''' + + def __init__(self, host, connection_info, *args, **kwargs): + super(Connection, self).__init__(host, connection_info) + + # SSH connection specific init stuff + self.HASHED_KEY_MAGIC = "|1|" + self._has_pipelining = True + + # FIXME: move the lockfile locations to ActionBase? + #fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) + #self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700) + self._cp_dir = '/tmp' + #fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN) + + def get_transport(self): + ''' used to identify this connection object from other classes ''' + return 'ssh' + + def connect(self): + ''' connect to the remote host ''' + + self._display.vvv("ESTABLISH CONNECTION FOR USER: %s" % self._connection_info.remote_user, host=self._host) + + self._common_args = [] + extra_args = C.ANSIBLE_SSH_ARGS + if extra_args is not None: + # make sure there is no empty string added as this can produce weird errors + self._common_args += [x.strip() for x in shlex.split(extra_args) if x.strip()] + else: + self._common_args += [ + "-o", "ControlMaster=auto", + "-o", "ControlPersist=60s", + "-o", "ControlPath=\"%s\"" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self._cp_dir)), + ] + + cp_in_use = False + cp_path_set = False + for arg in self._common_args: + if "ControlPersist" in arg: + cp_in_use = True + if "ControlPath" in arg: + cp_path_set = True + + if cp_in_use and not cp_path_set: + self._common_args += ["-o", "ControlPath=\"%s\"" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self._cp_dir))] + + if not C.HOST_KEY_CHECKING: + self._common_args += ["-o", "StrictHostKeyChecking=no"] + + if self._connection_info.port is not None: + self._common_args += ["-o", "Port=%d" % (self._connection_info.port)] + # FIXME: need to get this from connection info + #if self.private_key_file is not None: + # self._common_args += ["-o", "IdentityFile=\"%s\"" % os.path.expanduser(self.private_key_file)] + #elif self.runner.private_key_file is not None: + # self._common_args += ["-o", "IdentityFile=\"%s\"" % os.path.expanduser(self.runner.private_key_file)] + if self._connection_info.password: + self._common_args += ["-o", "GSSAPIAuthentication=no", + "-o", "PubkeyAuthentication=no"] + else: + self._common_args += ["-o", "KbdInteractiveAuthentication=no", + "-o", "PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey", + "-o", "PasswordAuthentication=no"] + if self._connection_info.remote_user != pwd.getpwuid(os.geteuid())[0]: + self._common_args += ["-o", "User="+self._connection_info.remote_user] + # FIXME: figure out where this goes + #self._common_args += ["-o", "ConnectTimeout=%d" % self.runner.timeout] + self._common_args += ["-o", "ConnectTimeout=15"] + + return self + + def _run(self, cmd, indata): + if indata: + # do not use pseudo-pty + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = p.stdin + else: + # try to use upseudo-pty + try: + # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors + master, slave = pty.openpty() + p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = os.fdopen(master, 'w', 0) + os.close(slave) + except: + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = p.stdin + + return (p, stdin) + + def _password_cmd(self): + if self._connection_info.password: + try: + p = subprocess.Popen(["sshpass"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.communicate() + except OSError: + raise AnsibleError("to use the 'ssh' connection type with passwords, you must install the sshpass program") + (self.rfd, self.wfd) = os.pipe() + return ["sshpass", "-d%d" % self.rfd] + return [] + + def _send_password(self): + if self._connection_info.password: + os.close(self.rfd) + os.write(self.wfd, "%s\n" % self._connection_info.password) + os.close(self.wfd) + + def _communicate(self, p, stdin, indata, su=False, sudoable=False, prompt=None): + fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) + fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK) + # We can't use p.communicate here because the ControlMaster may have stdout open as well + stdout = '' + stderr = '' + rpipes = [p.stdout, p.stderr] + if indata: + try: + stdin.write(indata) + stdin.close() + except: + raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh') + # Read stdout/stderr from process + while True: + rfd, wfd, efd = select.select(rpipes, [], rpipes, 1) + + # FIXME: su/sudo stuff + # fail early if the sudo/su password is wrong + #if self.runner.sudo and sudoable: + # if self.runner.sudo_pass: + # incorrect_password = gettext.dgettext( + # "sudo", "Sorry, try again.") + # if stdout.endswith("%s\r\n%s" % (incorrect_password, + # prompt)): + # raise AnsibleError('Incorrect sudo password') + # + # if stdout.endswith(prompt): + # raise AnsibleError('Missing sudo password') + # + #if self.runner.su and su and self.runner.su_pass: + # incorrect_password = gettext.dgettext( + # "su", "Sorry") + # if stdout.endswith("%s\r\n%s" % (incorrect_password, prompt)): + # raise AnsibleError('Incorrect su password') + + if p.stdout in rfd: + dat = os.read(p.stdout.fileno(), 9000) + stdout += dat + if dat == '': + rpipes.remove(p.stdout) + if p.stderr in rfd: + dat = os.read(p.stderr.fileno(), 9000) + stderr += dat + if dat == '': + rpipes.remove(p.stderr) + # only break out if no pipes are left to read or + # the pipes are completely read and + # the process is terminated + if (not rpipes or not rfd) and p.poll() is not None: + break + # No pipes are left to read but process is not yet terminated + # Only then it is safe to wait for the process to be finished + # NOTE: Actually p.poll() is always None here if rpipes is empty + elif not rpipes and p.poll() == None: + p.wait() + # The process is terminated. Since no pipes to read from are + # left, there is no need to call select() again. + break + # close stdin after process is terminated and stdout/stderr are read + # completely (see also issue #848) + stdin.close() + return (p.returncode, stdout, stderr) + + def not_in_host_file(self, host): + if 'USER' in os.environ: + user_host_file = os.path.expandvars("~${USER}/.ssh/known_hosts") + else: + user_host_file = "~/.ssh/known_hosts" + user_host_file = os.path.expanduser(user_host_file) + + host_file_list = [] + host_file_list.append(user_host_file) + host_file_list.append("/etc/ssh/ssh_known_hosts") + host_file_list.append("/etc/ssh/ssh_known_hosts2") + + hfiles_not_found = 0 + for hf in host_file_list: + if not os.path.exists(hf): + hfiles_not_found += 1 + continue + try: + host_fh = open(hf) + except IOError, e: + hfiles_not_found += 1 + continue + else: + data = host_fh.read() + host_fh.close() + + for line in data.split("\n"): + if line is None or " " not in line: + continue + tokens = line.split() + if tokens[0].find(self.HASHED_KEY_MAGIC) == 0: + # this is a hashed known host entry + try: + (kn_salt,kn_host) = tokens[0][len(self.HASHED_KEY_MAGIC):].split("|",2) + hash = hmac.new(kn_salt.decode('base64'), digestmod=sha1) + hash.update(host) + if hash.digest() == kn_host.decode('base64'): + return False + except: + # invalid hashed host key, skip it + continue + else: + # standard host file entry + if host in tokens[0]: + return False + + if (hfiles_not_found == len(host_file_list)): + self._display.vvv("EXEC previous known host file not found for %s" % host) + return True + + def exec_command(self, cmd, tmp_path, executable='/bin/sh', in_data=None): + ''' run a command on the remote host ''' + + ssh_cmd = self._password_cmd() + ssh_cmd += ["ssh", "-C"] + if not in_data: + # we can only use tty when we are not pipelining the modules. piping data into /usr/bin/python + # inside a tty automatically invokes the python interactive-mode but the modules are not + # compatible with the interactive-mode ("unexpected indent" mainly because of empty lines) + ssh_cmd += ["-tt"] + if self._connection_info.verbosity > 3: + ssh_cmd += ["-vvv"] + else: + ssh_cmd += ["-q"] + ssh_cmd += self._common_args + + # FIXME: ipv6 stuff needs to be figured out. It's in the connection info, however + # not sure if it's all working yet so this remains commented out + #if self._ipv6: + # ssh_cmd += ['-6'] + ssh_cmd += [self._host.ipv4_address] + + if not (self._connection_info.sudo or self._connection_info.su): + prompt = None + if executable: + ssh_cmd.append(executable + ' -c ' + pipes.quote(cmd)) + else: + ssh_cmd.append(cmd) + elif self._connection_info.su and self._connection_info.su_user: + su_cmd, prompt, success_key = self._connection_info.make_su_cmd(executable, cmd) + ssh_cmd.append(su_cmd) + else: + # FIXME: hard-coded sudo_exe here + sudo_cmd, prompt, success_key = self._connection_info.make_sudo_cmd('/usr/bin/sudo', executable, cmd) + ssh_cmd.append(sudo_cmd) + + self._display.vvv("EXEC %s" % ' '.join(ssh_cmd), host=self._host) + + not_in_host_file = self.not_in_host_file(self._host.get_name()) + + # FIXME: move the locations of these lock files, same as init above + #if C.HOST_KEY_CHECKING and not_in_host_file: + # # lock around the initial SSH connectivity so the user prompt about whether to add + # # the host to known hosts is not intermingled with multiprocess output. + # fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) + # fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX) + + # create process + (p, stdin) = self._run(ssh_cmd, in_data) + + self._send_password() + + no_prompt_out = '' + no_prompt_err = '' + # FIXME: su/sudo stuff + #if (self.runner.sudo and sudoable and self.runner.sudo_pass) or \ + # (self.runner.su and su and self.runner.su_pass): + # # several cases are handled for sudo privileges with password + # # * NOPASSWD (tty & no-tty): detect success_key on stdout + # # * without NOPASSWD: + # # * detect prompt on stdout (tty) + # # * detect prompt on stderr (no-tty) + # fcntl.fcntl(p.stdout, fcntl.F_SETFL, + # fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) + # fcntl.fcntl(p.stderr, fcntl.F_SETFL, + # fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK) + # sudo_output = '' + # sudo_errput = '' + # + # while True: + # if success_key in sudo_output or \ + # (self.runner.sudo_pass and sudo_output.endswith(prompt)) or \ + # (self.runner.su_pass and utils.su_prompts.check_su_prompt(sudo_output)): + # break + self._display.vvv("EXEC %s" % ' '.join(ssh_cmd), host=self._host) + + not_in_host_file = self.not_in_host_file(self._host.get_name()) + + # FIXME: file locations + #if C.HOST_KEY_CHECKING and not_in_host_file: + # # lock around the initial SSH connectivity so the user prompt about whether to add + # # the host to known hosts is not intermingled with multiprocess output. + # fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) + # fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX) + + # create process + (p, stdin) = self._run(ssh_cmd, in_data) + + self._send_password() + + no_prompt_out = '' + no_prompt_err = '' + # FIXME: su/sudo stuff + #if (self.runner.sudo and sudoable and self.runner.sudo_pass) or \ + # (self.runner.su and su and self.runner.su_pass): + # # several cases are handled for sudo privileges with password + # # * NOPASSWD (tty & no-tty): detect success_key on stdout + # # * without NOPASSWD: + # # * detect prompt on stdout (tty) + # # * detect prompt on stderr (no-tty) + # fcntl.fcntl(p.stdout, fcntl.F_SETFL, + # fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) + # fcntl.fcntl(p.stderr, fcntl.F_SETFL, + # fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK) + # sudo_output = '' + # sudo_errput = '' + # + # while True: + # if success_key in sudo_output or \ + # (self.runner.sudo_pass and sudo_output.endswith(prompt)) or \ + # (self.runner.su_pass and utils.su_prompts.check_su_prompt(sudo_output)): + # break + # + # rfd, wfd, efd = select.select([p.stdout, p.stderr], [], + # [p.stdout], self.runner.timeout) + # if p.stderr in rfd: + # chunk = p.stderr.read() + # if not chunk: + # raise AnsibleError('ssh connection closed waiting for sudo or su password prompt') + # sudo_errput += chunk + # incorrect_password = gettext.dgettext( + # "sudo", "Sorry, try again.") + # if sudo_errput.strip().endswith("%s%s" % (prompt, incorrect_password)): + # raise AnsibleError('Incorrect sudo password') + # elif sudo_errput.endswith(prompt): + # stdin.write(self.runner.sudo_pass + '\n') + # + # if p.stdout in rfd: + # chunk = p.stdout.read() + # if not chunk: + # raise AnsibleError('ssh connection closed waiting for sudo or su password prompt') + # sudo_output += chunk + # + # if not rfd: + # # timeout. wrap up process communication + # stdout = p.communicate() + # raise AnsibleError('ssh connection error waiting for sudo or su password prompt') + # + # if success_key not in sudo_output: + # if sudoable: + # stdin.write(self.runner.sudo_pass + '\n') + # elif su: + # stdin.write(self.runner.su_pass + '\n') + # else: + # no_prompt_out += sudo_output + # no_prompt_err += sudo_errput + + #(returncode, stdout, stderr) = self._communicate(p, stdin, in_data, su=su, sudoable=sudoable, prompt=prompt) + (returncode, stdout, stderr) = self._communicate(p, stdin, in_data, prompt=prompt) + + #if C.HOST_KEY_CHECKING and not_in_host_file: + # # lock around the initial SSH connectivity so the user prompt about whether to add + # # the host to known hosts is not intermingled with multiprocess output. + # fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN) + # fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN) + controlpersisterror = 'Bad configuration option: ControlPersist' in stderr or 'unknown configuration option: ControlPersist' in stderr + + if C.HOST_KEY_CHECKING: + if ssh_cmd[0] == "sshpass" and p.returncode == 6: + raise AnsibleError('Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this. Please add this host\'s fingerprint to your known_hosts file to manage this host.') + + if p.returncode != 0 and controlpersisterror: + raise AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ssh_args in [ssh_connection] section of the config file) before running again') + # FIXME: module name isn't in runner + #if p.returncode == 255 and (in_data or self.runner.module_name == 'raw'): + if p.returncode == 255 and in_data: + raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh') + + return (p.returncode, '', no_prompt_out + stdout, no_prompt_err + stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to remote ''' + self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._host) + if not os.path.exists(in_path): + raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) + cmd = self._password_cmd() + + # FIXME: make a function, used in all 3 methods EXEC/PUT/FETCH + host = self._host.ipv4_address + + # FIXME: ipv6 stuff needs to be figured out. It's in the connection info, however + # not sure if it's all working yet so this remains commented out + #if self._ipv6: + # host = '[%s]' % host + + if C.DEFAULT_SCP_IF_SSH: + cmd += ["scp"] + self._common_args + cmd += [in_path,host + ":" + pipes.quote(out_path)] + indata = None + else: + cmd += ["sftp"] + self._common_args + [host] + indata = "put %s %s\n" % (pipes.quote(in_path), pipes.quote(out_path)) + + (p, stdin) = self._run(cmd, indata) + + self._send_password() + + (returncode, stdout, stderr) = self._communicate(p, stdin, indata) + + if returncode != 0: + raise AnsibleError("failed to transfer file to %s:\n%s\n%s" % (out_path, stdout, stderr)) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from remote to local ''' + self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._host) + cmd = self._password_cmd() + + # FIXME: make a function, used in all 3 methods EXEC/PUT/FETCH + host = self._host.ipv4_address + + # FIXME: ipv6 stuff needs to be figured out. It's in the connection info, however + # not sure if it's all working yet so this remains commented out + #if self._ipv6: + # host = '[%s]' % self._host + + if C.DEFAULT_SCP_IF_SSH: + cmd += ["scp"] + self._common_args + cmd += [host + ":" + in_path, out_path] + indata = None + else: + cmd += ["sftp"] + self._common_args + [host] + indata = "get %s %s\n" % (in_path, out_path) + + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self._send_password() + stdout, stderr = p.communicate(indata) + + if p.returncode != 0: + raise AnsibleError("failed to transfer file from %s:\n%s\n%s" % (in_path, stdout, stderr)) + + def close(self): + ''' not applicable since we're executing openssh binaries ''' + pass + diff --git a/v2/ansible/plugins/connections/winrm.py b/v2/ansible/plugins/connections/winrm.py new file mode 100644 index 0000000000..d6e51710b5 --- /dev/null +++ b/v2/ansible/plugins/connections/winrm.py @@ -0,0 +1,258 @@ +# (c) 2014, Chris Church <chris@ninemoreminutes.com> +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import absolute_import + +import base64 +import hashlib +import imp +import os +import re +import shlex +import traceback +import urlparse +from ansible import errors +from ansible import utils +from ansible.callbacks import vvv, vvvv, verbose +from ansible.runner.shell_plugins import powershell + +try: + from winrm import Response + from winrm.exceptions import WinRMTransportError + from winrm.protocol import Protocol +except ImportError: + raise errors.AnsibleError("winrm is not installed") + +_winrm_cache = { + # 'user:pwhash@host:port': <protocol instance> +} + +def vvvvv(msg, host=None): + verbose(msg, host=host, caplevel=4) + +class Connection(object): + '''WinRM connections over HTTP/HTTPS.''' + + def __init__(self, runner, host, port, user, password, *args, **kwargs): + self.runner = runner + self.host = host + self.port = port + self.user = user + self.password = password + self.has_pipelining = False + self.default_shell = 'powershell' + self.default_suffixes = ['.ps1', ''] + self.protocol = None + self.shell_id = None + self.delegate = None + + def _winrm_connect(self): + ''' + Establish a WinRM connection over HTTP/HTTPS. + ''' + port = self.port or 5986 + vvv("ESTABLISH WINRM CONNECTION FOR USER: %s on PORT %s TO %s" % \ + (self.user, port, self.host), host=self.host) + netloc = '%s:%d' % (self.host, port) + cache_key = '%s:%s@%s:%d' % (self.user, hashlib.md5(self.password).hexdigest(), self.host, port) + if cache_key in _winrm_cache: + vvvv('WINRM REUSE EXISTING CONNECTION: %s' % cache_key, host=self.host) + return _winrm_cache[cache_key] + transport_schemes = [('plaintext', 'https'), ('plaintext', 'http')] # FIXME: ssl/kerberos + if port == 5985: + transport_schemes = reversed(transport_schemes) + exc = None + for transport, scheme in transport_schemes: + endpoint = urlparse.urlunsplit((scheme, netloc, '/wsman', '', '')) + vvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), + host=self.host) + protocol = Protocol(endpoint, transport=transport, + username=self.user, password=self.password) + try: + protocol.send_message('') + _winrm_cache[cache_key] = protocol + return protocol + except WinRMTransportError, exc: + err_msg = str(exc) + if re.search(r'Operation\s+?timed\s+?out', err_msg, re.I): + raise errors.AnsibleError("the connection attempt timed out") + m = re.search(r'Code\s+?(\d{3})', err_msg) + if m: + code = int(m.groups()[0]) + if code == 401: + raise errors.AnsibleError("the username/password specified for this server was incorrect") + elif code == 411: + _winrm_cache[cache_key] = protocol + return protocol + vvvv('WINRM CONNECTION ERROR: %s' % err_msg, host=self.host) + continue + if exc: + raise errors.AnsibleError(str(exc)) + + def _winrm_exec(self, command, args=(), from_exec=False): + if from_exec: + vvvv("WINRM EXEC %r %r" % (command, args), host=self.host) + else: + vvvvv("WINRM EXEC %r %r" % (command, args), host=self.host) + if not self.protocol: + self.protocol = self._winrm_connect() + if not self.shell_id: + self.shell_id = self.protocol.open_shell() + command_id = None + try: + command_id = self.protocol.run_command(self.shell_id, command, args) + response = Response(self.protocol.get_command_output(self.shell_id, command_id)) + if from_exec: + vvvv('WINRM RESULT %r' % response, host=self.host) + else: + vvvvv('WINRM RESULT %r' % response, host=self.host) + vvvvv('WINRM STDOUT %s' % response.std_out, host=self.host) + vvvvv('WINRM STDERR %s' % response.std_err, host=self.host) + return response + finally: + if command_id: + self.protocol.cleanup_command(self.shell_id, command_id) + + def connect(self): + if not self.protocol: + self.protocol = self._winrm_connect() + return self + + def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable=None, in_data=None, su=None, su_user=None): + cmd = cmd.encode('utf-8') + cmd_parts = shlex.split(cmd, posix=False) + if '-EncodedCommand' in cmd_parts: + encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1] + decoded_cmd = base64.b64decode(encoded_cmd) + vvv("EXEC %s" % decoded_cmd, host=self.host) + else: + vvv("EXEC %s" % cmd, host=self.host) + # For script/raw support. + if cmd_parts and cmd_parts[0].lower().endswith('.ps1'): + script = powershell._build_file_cmd(cmd_parts) + cmd_parts = powershell._encode_script(script, as_list=True) + try: + result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True) + except Exception, e: + traceback.print_exc() + raise errors.AnsibleError("failed to exec cmd %s" % cmd) + return (result.status_code, '', result.std_out.encode('utf-8'), result.std_err.encode('utf-8')) + + def put_file(self, in_path, out_path): + vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + with open(in_path) as in_file: + in_size = os.path.getsize(in_path) + script_template = ''' + $s = [System.IO.File]::OpenWrite("%s"); + [void]$s.Seek(%d, [System.IO.SeekOrigin]::Begin); + $b = [System.Convert]::FromBase64String("%s"); + [void]$s.Write($b, 0, $b.length); + [void]$s.SetLength(%d); + [void]$s.Close(); + ''' + # Determine max size of data we can pass per command. + script = script_template % (powershell._escape(out_path), in_size, '', in_size) + cmd = powershell._encode_script(script) + # Encode script with no data, subtract its length from 8190 (max + # windows command length), divide by 2.67 (UTF16LE base64 command + # encoding), then by 1.35 again (data base64 encoding). + buffer_size = int(((8190 - len(cmd)) / 2.67) / 1.35) + for offset in xrange(0, in_size, buffer_size): + try: + out_data = in_file.read(buffer_size) + if offset == 0: + if out_data.lower().startswith('#!powershell') and not out_path.lower().endswith('.ps1'): + out_path = out_path + '.ps1' + b64_data = base64.b64encode(out_data) + script = script_template % (powershell._escape(out_path), offset, b64_data, in_size) + vvvv("WINRM PUT %s to %s (offset=%d size=%d)" % (in_path, out_path, offset, len(out_data)), host=self.host) + cmd_parts = powershell._encode_script(script, as_list=True) + result = self._winrm_exec(cmd_parts[0], cmd_parts[1:]) + if result.status_code != 0: + raise IOError(result.std_err.encode('utf-8')) + except Exception: + traceback.print_exc() + raise errors.AnsibleError("failed to transfer file to %s" % out_path) + + def fetch_file(self, in_path, out_path): + out_path = out_path.replace('\\', '/') + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + buffer_size = 2**20 # 1MB chunks + if not os.path.exists(os.path.dirname(out_path)): + os.makedirs(os.path.dirname(out_path)) + out_file = None + try: + offset = 0 + while True: + try: + script = ''' + If (Test-Path -PathType Leaf "%(path)s") + { + $stream = [System.IO.File]::OpenRead("%(path)s"); + $stream.Seek(%(offset)d, [System.IO.SeekOrigin]::Begin) | Out-Null; + $buffer = New-Object Byte[] %(buffer_size)d; + $bytesRead = $stream.Read($buffer, 0, %(buffer_size)d); + $bytes = $buffer[0..($bytesRead-1)]; + [System.Convert]::ToBase64String($bytes); + $stream.Close() | Out-Null; + } + ElseIf (Test-Path -PathType Container "%(path)s") + { + Write-Host "[DIR]"; + } + Else + { + Write-Error "%(path)s does not exist"; + Exit 1; + } + ''' % dict(buffer_size=buffer_size, path=powershell._escape(in_path), offset=offset) + vvvv("WINRM FETCH %s to %s (offset=%d)" % (in_path, out_path, offset), host=self.host) + cmd_parts = powershell._encode_script(script, as_list=True) + result = self._winrm_exec(cmd_parts[0], cmd_parts[1:]) + if result.status_code != 0: + raise IOError(result.std_err.encode('utf-8')) + if result.std_out.strip() == '[DIR]': + data = None + else: + data = base64.b64decode(result.std_out.strip()) + if data is None: + if not os.path.exists(out_path): + os.makedirs(out_path) + break + else: + if not out_file: + # If out_path is a directory and we're expecting a file, bail out now. + if os.path.isdir(out_path): + break + out_file = open(out_path, 'wb') + out_file.write(data) + if len(data) < buffer_size: + break + offset += len(data) + except Exception: + traceback.print_exc() + raise errors.AnsibleError("failed to transfer file to %s" % out_path) + finally: + if out_file: + out_file.close() + + def close(self): + if self.protocol and self.shell_id: + self.protocol.close_shell(self.shell_id) + self.shell_id = None diff --git a/v2/ansible/plugins/filter b/v2/ansible/plugins/filter new file mode 120000 index 0000000000..fa1d588570 --- /dev/null +++ b/v2/ansible/plugins/filter @@ -0,0 +1 @@ +../../../lib/ansible/runner/filter_plugins
\ No newline at end of file diff --git a/v2/ansible/plugins/inventory/__init__.py b/v2/ansible/plugins/inventory/__init__.py index 785fc45992..41e8578ee7 100644 --- a/v2/ansible/plugins/inventory/__init__.py +++ b/v2/ansible/plugins/inventory/__init__.py @@ -15,7 +15,66 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. +############################################# + # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from abc import ABCMeta, abstractmethod + +class InventoryParser: + '''Abstract Base Class for retrieving inventory information + + Any InventoryParser functions by taking an inven_source. The caller then + calls the parser() method. Once parser is called, the caller can access + InventoryParser.hosts for a mapping of Host objects and + InventoryParser.Groups for a mapping of Group objects. + ''' + __metaclass__ = ABCMeta + + def __init__(self, inven_source): + ''' + InventoryParser contructors take a source of inventory information + that they will parse the host and group information from. + ''' + self.inven_source = inven_source + self.reset_parser() + + @abstractmethod + def reset_parser(self): + ''' + InventoryParsers generally cache their data once parser() is + called. This method initializes any parser state before calling parser + again. + ''' + self.hosts = dict() + self.groups = dict() + self.parsed = False + + def _merge(self, target, addition): + ''' + This method is provided to InventoryParsers to merge host or group + dicts since it may take several passes to get all of the data + + Example usage: + self.hosts = self.from_ini(filename) + new_hosts = self.from_script(scriptname) + self._merge(self.hosts, new_hosts) + ''' + for i in addition: + if i in target: + target[i].merge(addition[i]) + else: + target[i] = addition[i] + + @abstractmethod + def parse(self, refresh=False): + if refresh: + self.reset_parser() + if self.parsed: + return self.parsed + + # Parse self.inven_sources here + pass + diff --git a/v2/ansible/plugins/inventory/directory.py b/v2/ansible/plugins/inventory/directory.py new file mode 100644 index 0000000000..d340ed7538 --- /dev/null +++ b/v2/ansible/plugins/inventory/directory.py @@ -0,0 +1,52 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +# Make coding more python3-ish +from __future__ import (division, print_function) +__metaclass__ = type + +import os + +from . aggregate import InventoryAggregateParser + +class InventoryDirectoryParser(InventoryAggregateParser): + + def __init__(self, inven_directory): + directory = inven_directory + names = os.listdir(inven_directory) + filtered_names = [] + + # Clean up the list of filenames + for filename in names: + # Skip files that end with certain extensions or characters + if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")): + continue + # Skip hidden files + if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)): + continue + # These are things inside of an inventory basedir + if filename in ("host_vars", "group_vars", "vars_plugins"): + continue + fullpath = os.path.join(directory, filename) + new_names.append(fullpath) + + super(InventoryDirectoryParser, self).__init__(new_names) + + def parse(self): + return super(InventoryDirectoryParser, self).parse() diff --git a/v2/ansible/plugins/inventory/ini.py b/v2/ansible/plugins/inventory/ini.py new file mode 100644 index 0000000000..e185c1a785 --- /dev/null +++ b/v2/ansible/plugins/inventory/ini.py @@ -0,0 +1,60 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +############################################# + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from . import InventoryParser + +class InventoryIniParser(InventoryAggregateParser): + + def __init__(self, inven_directory): + directory = inven_directory + names = os.listdir(inven_directory) + filtered_names = [] + + # Clean up the list of filenames + for filename in names: + # Skip files that end with certain extensions or characters + if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")): + continue + # Skip hidden files + if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)): + continue + # These are things inside of an inventory basedir + if filename in ("host_vars", "group_vars", "vars_plugins"): + continue + fullpath = os.path.join(directory, filename) + new_names.append(fullpath) + + super(InventoryDirectoryParser, self).__init__(new_names) + + def parse(self): + return super(InventoryDirectoryParser, self).parse() + + def _before_comment(self, msg): + ''' what's the part of a string before a comment? ''' + msg = msg.replace("\#","**NOT_A_COMMENT**") + msg = msg.split("#")[0] + msg = msg.replace("**NOT_A_COMMENT**","#") + return msg + diff --git a/v2/ansible/plugins/lookup/__init__.py b/v2/ansible/plugins/lookup/__init__.py index 785fc45992..8c841c81d2 100644 --- a/v2/ansible/plugins/lookup/__init__.py +++ b/v2/ansible/plugins/lookup/__init__.py @@ -19,3 +19,31 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +__all__ = ['LookupBase'] + +class LookupBase: + def __init__(self, loader=None, **kwargs): + self._loader = loader + + def _flatten(self, terms): + ret = [] + for term in terms: + if isinstance(term, (list, tuple)): + ret.extend(term) + else: + ret.append(term) + return ret + + def _combine(self, a, b): + results = [] + for x in a: + for y in b: + results.append(self._flatten([x,y])) + return results + + def _flatten_hash_to_list(self, terms): + ret = [] + for key in terms: + ret.append({'key': key, 'value': terms[key]}) + return ret + diff --git a/v2/ansible/plugins/lookup/csvfile.py b/v2/ansible/plugins/lookup/csvfile.py index ce5a2b77d2..87757399ce 100644 --- a/v2/ansible/plugins/lookup/csvfile.py +++ b/v2/ansible/plugins/lookup/csvfile.py @@ -15,15 +15,14 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils, errors import os import codecs import csv -class LookupModule(object): +from ansible.errors import * +from ansible.plugins.lookup import LookupBase - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): def read_csv(self, filename, key, delimiter, dflt=None, col=1): @@ -35,13 +34,11 @@ class LookupModule(object): if row[0] == key: return row[int(col)] except Exception, e: - raise errors.AnsibleError("csvfile: %s" % str(e)) + raise AnsibleError("csvfile: %s" % str(e)) return dflt - def run(self, terms, inject=None, **kwargs): - - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables=None, **kwargs): if isinstance(terms, basestring): terms = [ terms ] @@ -65,12 +62,12 @@ class LookupModule(object): assert(name in paramvals) paramvals[name] = value except (ValueError, AssertionError), e: - raise errors.AnsibleError(e) + raise AnsibleError(e) if paramvals['delimiter'] == 'TAB': paramvals['delimiter'] = "\t" - path = utils.path_dwim(self.basedir, paramvals['file']) + path = self._loader.path_dwim(paramvals['file']) var = self.read_csv(path, key, paramvals['delimiter'], paramvals['default'], paramvals['col']) if var is not None: diff --git a/v2/ansible/plugins/lookup/dict.py b/v2/ansible/plugins/lookup/dict.py index cda1546598..61389df7c2 100644 --- a/v2/ansible/plugins/lookup/dict.py +++ b/v2/ansible/plugins/lookup/dict.py @@ -15,25 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible.utils import safe_eval -import ansible.utils as utils -import ansible.errors as errors +from ansible.plugins.lookup import LookupBase -def flatten_hash_to_list(terms): - ret = [] - for key in terms: - ret.append({'key': key, 'value': terms[key]}) - return ret +class LookupModule(LookupBase): -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - - def run(self, terms, inject=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, varibles=None, **kwargs): if not isinstance(terms, dict): raise errors.AnsibleError("with_dict expects a dict") - return flatten_hash_to_list(terms) + return self._flatten_hash_to_list(terms) diff --git a/v2/ansible/plugins/lookup/dnstxt.py b/v2/ansible/plugins/lookup/dnstxt.py index 4fa47bf4ee..7100f8d96d 100644 --- a/v2/ansible/plugins/lookup/dnstxt.py +++ b/v2/ansible/plugins/lookup/dnstxt.py @@ -15,8 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils, errors import os + HAVE_DNS=False try: import dns.resolver @@ -25,6 +25,9 @@ try: except ImportError: pass +from ansible.errors import * +from ansible.plugins.lookup import LookupBase + # ============================================================== # DNSTXT: DNS TXT records # @@ -32,17 +35,12 @@ except ImportError: # TODO: configurable resolver IPs # -------------------------------------------------------------- -class LookupModule(object): +class LookupModule(LookupBase): - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir + def run(self, terms, variables=None, **kwargs): if HAVE_DNS == False: - raise errors.AnsibleError("Can't LOOKUP(dnstxt): module dns.resolver is not installed") - - def run(self, terms, inject=None, **kwargs): - - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + raise AnsibleError("Can't LOOKUP(dnstxt): module dns.resolver is not installed") if isinstance(terms, basestring): terms = [ terms ] @@ -62,7 +60,9 @@ class LookupModule(object): except dns.resolver.Timeout: string = '' except dns.exception.DNSException, e: - raise errors.AnsibleError("dns.resolver unhandled exception", e) + raise AnsibleError("dns.resolver unhandled exception", e) ret.append(''.join(string)) + return ret + diff --git a/v2/ansible/plugins/lookup/env.py b/v2/ansible/plugins/lookup/env.py index d4f85356ed..896f95e13a 100644 --- a/v2/ansible/plugins/lookup/env.py +++ b/v2/ansible/plugins/lookup/env.py @@ -15,21 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils, errors -from ansible.utils import template import os -class LookupModule(object): +from ansible.plugins.lookup import LookupBase - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): - - try: - terms = template.template(self.basedir, terms, inject) - except Exception, e: - pass + def run(self, terms, variables, **kwargs): if isinstance(terms, basestring): terms = [ terms ] @@ -38,4 +30,5 @@ class LookupModule(object): for term in terms: var = term.split()[0] ret.append(os.getenv(var, '')) + return ret diff --git a/v2/ansible/plugins/lookup/etcd.py b/v2/ansible/plugins/lookup/etcd.py index a758a2fb0b..5b54788985 100644 --- a/v2/ansible/plugins/lookup/etcd.py +++ b/v2/ansible/plugins/lookup/etcd.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils import os import urllib2 try: @@ -23,6 +22,8 @@ try: except ImportError: import simplejson as json +from ansible.plugins.lookup import LookupBase + # this can be made configurable, not should not use ansible.cfg ANSIBLE_ETCD_URL = 'http://127.0.0.1:4001' if os.getenv('ANSIBLE_ETCD_URL') is not None: @@ -57,22 +58,18 @@ class etcd(): return value -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - self.etcd = etcd() +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): - - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): if isinstance(terms, basestring): terms = [ terms ] + etcd = etcd() + ret = [] for term in terms: key = term.split()[0] - value = self.etcd.get(key) + value = etcd.get(key) ret.append(value) return ret diff --git a/v2/ansible/plugins/lookup/file.py b/v2/ansible/plugins/lookup/file.py index 70bae6653a..add4da7f47 100644 --- a/v2/ansible/plugins/lookup/file.py +++ b/v2/ansible/plugins/lookup/file.py @@ -15,27 +15,22 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils, errors import os import codecs -class LookupModule(object): +from ansible.errors import * +from ansible.plugins.lookup import LookupBase - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): + def run(self, terms, variables=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - ret = [] - - # this can happen if the variable contains a string, strictly not desired for lookup - # plugins, but users may try it, so make it work. if not isinstance(terms, list): terms = [ terms ] + ret = [] for term in terms: - basedir_path = utils.path_dwim(self.basedir, term) + basedir_path = self._loader.path_dwim(term) relative_path = None playbook_path = None @@ -44,16 +39,20 @@ class LookupModule(object): # basedir of the current file, use dwim_relative to look in the # role/files/ directory, and finally the playbook directory # itself (which will be relative to the current working dir) - if '_original_file' in inject: - relative_path = utils.path_dwim_relative(inject['_original_file'], 'files', term, self.basedir, check=False) - if 'playbook_dir' in inject: - playbook_path = os.path.join(inject['playbook_dir'], term) + + # FIXME: the original file stuff still needs to be worked out, but the + # playbook_dir stuff should be able to be removed as it should + # be covered by the fact that the loader contains that info + #if '_original_file' in variables: + # relative_path = self._loader.path_dwim_relative(variables['_original_file'], 'files', term, self.basedir, check=False) + #if 'playbook_dir' in variables: + # playbook_path = os.path.join(variables['playbook_dir'], term) for path in (basedir_path, relative_path, playbook_path): if path and os.path.exists(path): ret.append(codecs.open(path, encoding="utf8").read().rstrip()) break else: - raise errors.AnsibleError("could not locate file in lookup: %s" % term) + raise AnsibleError("could not locate file in lookup: %s" % term) return ret diff --git a/v2/ansible/plugins/lookup/fileglob.py b/v2/ansible/plugins/lookup/fileglob.py index 7d3cbb92be..bde016af9e 100644 --- a/v2/ansible/plugins/lookup/fileglob.py +++ b/v2/ansible/plugins/lookup/fileglob.py @@ -17,23 +17,16 @@ import os import glob -from ansible import utils -class LookupModule(object): +from ansible.plugins.lookup import LookupBase - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): - - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables=None, **kwargs): ret = [] - for term in terms: - - dwimmed = utils.path_dwim(self.basedir, term) + dwimmed = self._loader.path_dwim(term) globbed = glob.glob(dwimmed) ret.extend(g for g in globbed if os.path.isfile(g)) - return ret diff --git a/v2/ansible/plugins/lookup/first_found.py b/v2/ansible/plugins/lookup/first_found.py index a48b56a3c2..ea43e13c4d 100644 --- a/v2/ansible/plugins/lookup/first_found.py +++ b/v2/ansible/plugins/lookup/first_found.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. - # take a list of files and (optionally) a list of paths # return the first existing file found in the paths # [file1, file2, file3], [path1, path2, path3] @@ -118,17 +117,15 @@ # ignore_errors: true -from ansible import utils, errors import os -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +from ansible.plugins.lookup import LookupBase +from ansible.template import Templar +from ansible.utils.boolean import boolean - def run(self, terms, inject=None, **kwargs): +class LookupModule(LookupBase): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): result = None anydict = False @@ -144,7 +141,7 @@ class LookupModule(object): if isinstance(term, dict): files = term.get('files', []) paths = term.get('paths', []) - skip = utils.boolean(term.get('skip', False)) + skip = boolean(term.get('skip', False)) filelist = files if isinstance(files, basestring): @@ -172,20 +169,26 @@ class LookupModule(object): else: total_search = terms + templar = Templar(loader=self._loader, variables=variables) + roledir = variables.get('roledir') for fn in total_search: - if inject and '_original_file' in inject: - # check the templates and vars directories too, - # if they exist - for roledir in ('templates', 'vars'): - path = utils.path_dwim(os.path.join(self.basedir, '..', roledir), fn) - if os.path.exists(path): - return [path] - # if none of the above were found, just check the - # current filename against the basedir (this will already - # have ../files from runner, if it's a role task - path = utils.path_dwim(self.basedir, fn) - if os.path.exists(path): - return [path] + fn = templar.template(fn) + if os.path.isabs(fn) and os.path.exists(fn): + return [fn] + else: + if roledir is not None: + # check the templates and vars directories too,if they exist + for subdir in ('templates', 'vars'): + path = self._loader.path_dwim_relative(roledir, subdir, fn) + if os.path.exists(path): + return [path] + + # if none of the above were found, just check the + # current filename against the basedir (this will already + # have ../files from runner, if it's a role task + path = self._loader.path_dwim(fn) + if os.path.exists(path): + return [path] else: if skip: return [] diff --git a/v2/ansible/plugins/lookup/flattened.py b/v2/ansible/plugins/lookup/flattened.py index 831b2e9130..24f1a9ac95 100644 --- a/v2/ansible/plugins/lookup/flattened.py +++ b/v2/ansible/plugins/lookup/flattened.py @@ -15,34 +15,29 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -import ansible.utils as utils -import ansible.errors as errors +from ansible.errors import * +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms -def check_list_of_one_list(term): - # make sure term is not a list of one (list of one..) item - # return the final non list item if so +class LookupModule(LookupBase): - if isinstance(term,list) and len(term) == 1: - term = term[0] - if isinstance(term,list): - term = check_list_of_one_list(term) + def _check_list_of_one_list(self, term): + # make sure term is not a list of one (list of one..) item + # return the final non list item if so - return term + if isinstance(term,list) and len(term) == 1: + term = term[0] + if isinstance(term,list): + term = self._check_list_of_one_list(term) + return term - -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - - - def flatten(self, terms, inject): + def _do_flatten(self, terms, variables): ret = [] for term in terms: - term = check_list_of_one_list(term) + term = self._check_list_of_one_list(term) if term == 'None' or term == 'null': # ignore undefined items @@ -50,29 +45,25 @@ class LookupModule(object): if isinstance(term, basestring): # convert a variable to a list - term2 = utils.listify_lookup_plugin_terms(term, self.basedir, inject) + term2 = listify_lookup_plugin_terms(term, variables, loader=self._loader) # but avoid converting a plain string to a list of one string if term2 != [ term ]: term = term2 if isinstance(term, list): # if it's a list, check recursively for items that are a list - term = self.flatten(term, inject) + term = self._do_flatten(term, variables) ret.extend(term) - else: + else: ret.append(term) return ret - def run(self, terms, inject=None, **kwargs): - - # see if the string represents a list and convert to list if so - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): if not isinstance(terms, list): - raise errors.AnsibleError("with_flattened expects a list") + raise AnsibleError("with_flattened expects a list") - ret = self.flatten(terms, inject) - return ret + return self._do_flatten(terms, variables) diff --git a/v2/ansible/plugins/lookup/indexed_items.py b/v2/ansible/plugins/lookup/indexed_items.py index c1db1fdee2..1731dc0e84 100644 --- a/v2/ansible/plugins/lookup/indexed_items.py +++ b/v2/ansible/plugins/lookup/indexed_items.py @@ -15,30 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible.utils import safe_eval -import ansible.utils as utils -import ansible.errors as errors +from ansible.plugins.lookup import LookupBase -def flatten(terms): - ret = [] - for term in terms: - if isinstance(term, list): - ret.extend(term) - else: - ret.append(term) - return ret - -class LookupModule(object): +class LookupModule(LookupBase): def __init__(self, basedir=None, **kwargs): self.basedir = basedir - def run(self, terms, inject=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): if not isinstance(terms, list): raise errors.AnsibleError("with_indexed_items expects a list") - items = flatten(terms) + items = self._flatten(terms) return zip(range(len(items)), items) diff --git a/v2/ansible/plugins/lookup/inventory_hostnames.py b/v2/ansible/plugins/lookup/inventory_hostnames.py index 98523e1398..faffe47eb8 100644 --- a/v2/ansible/plugins/lookup/inventory_hostnames.py +++ b/v2/ansible/plugins/lookup/inventory_hostnames.py @@ -16,33 +16,19 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible.utils import safe_eval -import ansible.utils as utils -import ansible.errors as errors -import ansible.inventory as inventory +from ansible.errors import * +from ansible.plugins.lookup import LookupBase -def flatten(terms): - ret = [] - for term in terms: - if isinstance(term, list): - ret.extend(term) - else: - ret.append(term) - return ret - -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - if 'runner' in kwargs: - self.host_list = kwargs['runner'].inventory.host_list - else: - raise errors.AnsibleError("inventory_hostnames must be used as a loop. Example: \"with_inventory_hostnames: \'all\'\"") +class LookupModule(LookupBase): def run(self, terms, inject=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - if not isinstance(terms, list): - raise errors.AnsibleError("with_inventory_hostnames expects a list") - return flatten(inventory.Inventory(self.host_list).list_hosts(terms)) + raise AnsibleError("with_inventory_hostnames expects a list") + + # FIXME: the inventory is no longer available this way, so we may have + # to dump the host list into the list of variables and read it back + # in here (or the inventory sources, so we can recreate the list + # of hosts) + #return self._flatten(inventory.Inventory(self.host_list).list_hosts(terms)) + return terms diff --git a/v2/ansible/plugins/lookup/items.py b/v2/ansible/plugins/lookup/items.py index 85e77d5380..46925d2a8b 100644 --- a/v2/ansible/plugins/lookup/items.py +++ b/v2/ansible/plugins/lookup/items.py @@ -15,30 +15,14 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible.utils import safe_eval -import ansible.utils as utils -import ansible.errors as errors +from ansible.plugins.lookup import LookupBase -def flatten(terms): - ret = [] - for term in terms: - if isinstance(term, list): - ret.extend(term) - else: - ret.append(term) - return ret +class LookupModule(LookupBase): -class LookupModule(object): + def run(self, terms, **kwargs): - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - - def run(self, terms, inject=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - - if not isinstance(terms, list) and not isinstance(terms,set): - raise errors.AnsibleError("with_items expects a list or a set") - - return flatten(terms) + if not isinstance(terms, list): + terms = [ terms ] + return self._flatten(terms) diff --git a/v2/ansible/plugins/lookup/lines.py b/v2/ansible/plugins/lookup/lines.py index 5d4b70a857..507793b18e 100644 --- a/v2/ansible/plugins/lookup/lines.py +++ b/v2/ansible/plugins/lookup/lines.py @@ -16,23 +16,20 @@ # along with Ansible. If not, see <http://www.gnu.org/licenses/>. import subprocess -from ansible import utils, errors -class LookupModule(object): +from ansible.errors import * +from ansible.plugins.lookup import LookupBase - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): - - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): ret = [] for term in terms: - p = subprocess.Popen(term, cwd=self.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() if p.returncode == 0: ret.extend(stdout.splitlines()) else: - raise errors.AnsibleError("lookup_plugin.lines(%s) returned %d" % (term, p.returncode)) + raise AnsibleError("lookup_plugin.lines(%s) returned %d" % (term, p.returncode)) return ret diff --git a/v2/ansible/plugins/lookup/nested.py b/v2/ansible/plugins/lookup/nested.py index 29c4a7d21c..0f2d146b47 100644 --- a/v2/ansible/plugins/lookup/nested.py +++ b/v2/ansible/plugins/lookup/nested.py @@ -15,59 +15,35 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -import ansible.utils as utils -from ansible.utils import safe_eval -import ansible.errors as errors +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms -def flatten(terms): - ret = [] - for term in terms: - if isinstance(term, list): - ret.extend(term) - elif isinstance(term, tuple): - ret.extend(term) - else: - ret.append(term) - return ret +class LookupModule(LookupBase): -def combine(a,b): - results = [] - for x in a: - for y in b: - results.append(flatten([x,y])) - return results - -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - - def __lookup_injects(self, terms, inject): + def __lookup_variabless(self, terms, variables): results = [] for x in terms: - intermediate = utils.listify_lookup_plugin_terms(x, self.basedir, inject) + intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader) results.append(intermediate) return results - def run(self, terms, inject=None, **kwargs): - - # this code is common with 'items.py' consider moving to utils if we need it again + def run(self, terms, variables=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - terms = self.__lookup_injects(terms, inject) + terms = self.__lookup_variabless(terms, variables) my_list = terms[:] my_list.reverse() result = [] if len(my_list) == 0: - raise errors.AnsibleError("with_nested requires at least one element in the nested list") + raise AnsibleError("with_nested requires at least one element in the nested list") result = my_list.pop() while len(my_list) > 0: - result2 = combine(result, my_list.pop()) + result2 = self._combine(result, my_list.pop()) result = result2 new_result = [] for x in result: - new_result.append(flatten(x)) + new_result.append(self._flatten(x)) return new_result diff --git a/v2/ansible/plugins/lookup/password.py b/v2/ansible/plugins/lookup/password.py index a066887e2c..6e13410e1a 100644 --- a/v2/ansible/plugins/lookup/password.py +++ b/v2/ansible/plugins/lookup/password.py @@ -17,38 +17,55 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils, errors import os import errno -from string import ascii_letters, digits import string import random +from string import ascii_letters, digits + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.utils.encrypt import do_encrypt + +DEFAULT_LENGTH = 20 -class LookupModule(object): +class LookupModule(LookupBase): - LENGTH = 20 + def random_password(self, length=DEFAULT_LENGTH, chars=C.DEFAULT_PASSWORD_CHARS): + ''' + Return a random password string of length containing only chars. + NOTE: this was moved from the old ansible utils code, as nothing + else appeared to use it. + ''' - def __init__(self, length=None, encrypt=None, basedir=None, **kwargs): - self.basedir = basedir + password = [] + while len(password) < length: + new_char = os.urandom(1) + if new_char in chars: + password.append(new_char) + + return ''.join(password) def random_salt(self): salt_chars = ascii_letters + digits + './' - return utils.random_password(length=8, chars=salt_chars) - - def run(self, terms, inject=None, **kwargs): + return self.random_password(length=8, chars=salt_chars) - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): ret = [] + if not isinstance(terms, list): + terms = [ terms ] + for term in terms: # you can't have escaped spaces in yor pathname params = term.split() relpath = params[0] paramvals = { - 'length': LookupModule.LENGTH, + 'length': DEFAULT_LENGTH, 'encrypt': None, 'chars': ['ascii_letters','digits',".,:-_"], } @@ -69,21 +86,21 @@ class LookupModule(object): else: paramvals[name] = value except (ValueError, AssertionError), e: - raise errors.AnsibleError(e) + raise AnsibleError(e) length = paramvals['length'] encrypt = paramvals['encrypt'] use_chars = paramvals['chars'] # get password or create it if file doesn't exist - path = utils.path_dwim(self.basedir, relpath) + path = self._loader.path_dwim(relpath) if not os.path.exists(path): pathdir = os.path.dirname(path) if not os.path.isdir(pathdir): try: os.makedirs(pathdir, mode=0700) except OSError, e: - raise errors.AnsibleError("cannot create the path for the password lookup: %s (error was %s)" % (pathdir, str(e))) + raise AnsibleError("cannot create the path for the password lookup: %s (error was %s)" % (pathdir, str(e))) chars = "".join([getattr(string,c,c) for c in use_chars]).replace('"','').replace("'",'') password = ''.join(random.choice(chars) for _ in range(length)) @@ -121,7 +138,7 @@ class LookupModule(object): f.write(password + '\n') if encrypt: - password = utils.do_encrypt(password, encrypt, salt=salt) + password = do_encrypt(password, encrypt, salt=salt) ret.append(password) diff --git a/v2/ansible/plugins/lookup/pipe.py b/v2/ansible/plugins/lookup/pipe.py index 0cd9e1cda5..0a7e5cb31a 100644 --- a/v2/ansible/plugins/lookup/pipe.py +++ b/v2/ansible/plugins/lookup/pipe.py @@ -16,16 +16,13 @@ # along with Ansible. If not, see <http://www.gnu.org/licenses/>. import subprocess -from ansible import utils, errors -class LookupModule(object): +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): - - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + def run(self, terms, variables, **kwargs): if isinstance(terms, basestring): terms = [ terms ] @@ -43,10 +40,10 @@ class LookupModule(object): ''' term = str(term) - p = subprocess.Popen(term, cwd=self.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() if p.returncode == 0: ret.append(stdout.decode("utf-8").rstrip()) else: - raise errors.AnsibleError("lookup_plugin.pipe(%s) returned %d" % (term, p.returncode)) + raise AnsibleError("lookup_plugin.pipe(%s) returned %d" % (term, p.returncode)) return ret diff --git a/v2/ansible/plugins/lookup/random_choice.py b/v2/ansible/plugins/lookup/random_choice.py index 9b32c2f119..e899a2dbe3 100644 --- a/v2/ansible/plugins/lookup/random_choice.py +++ b/v2/ansible/plugins/lookup/random_choice.py @@ -16,7 +16,8 @@ # along with Ansible. If not, see <http://www.gnu.org/licenses/>. import random -from ansible import utils + +from ansible.plugins.lookup import LookupBase # useful for introducing chaos ... or just somewhat reasonably fair selection # amongst available mirrors @@ -28,14 +29,9 @@ from ansible import utils # - two # - three -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): def run(self, terms, inject=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - return [ random.choice(terms) ] diff --git a/v2/ansible/plugins/lookup/redis_kv.py b/v2/ansible/plugins/lookup/redis_kv.py index 22c5c3754f..08895d4c4e 100644 --- a/v2/ansible/plugins/lookup/redis_kv.py +++ b/v2/ansible/plugins/lookup/redis_kv.py @@ -15,15 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible import utils, errors import os +import re + HAVE_REDIS=False try: import redis # https://github.com/andymccurdy/redis-py/ HAVE_REDIS=True except ImportError: pass -import re + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase # ============================================================== # REDISGET: Obtain value from a GET on a Redis key. Terms @@ -31,17 +34,15 @@ import re # URL may be empty, in which case redis://localhost:6379 assumed # -------------------------------------------------------------- -class LookupModule(object): - - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - if HAVE_REDIS == False: - raise errors.AnsibleError("Can't LOOKUP(redis_kv): module redis is not installed") + def run(self, terms, variables, **kwargs): - def run(self, terms, inject=None, **kwargs): + if not HAVE_REDIS: + raise AnsibleError("Can't LOOKUP(redis_kv): module redis is not installed") - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + if not isinstance(terms, list): + terms = [ terms ] ret = [] for term in terms: @@ -59,7 +60,7 @@ class LookupModule(object): host = m.group('host') port = int(m.group('port')) except AttributeError: - raise errors.AnsibleError("Bad URI in redis lookup") + raise AnsibleError("Bad URI in redis lookup") try: conn = redis.Redis(host=host, port=port) diff --git a/v2/ansible/plugins/lookup/sequence.py b/v2/ansible/plugins/lookup/sequence.py index b162b3069e..99783cf566 100644 --- a/v2/ansible/plugins/lookup/sequence.py +++ b/v2/ansible/plugins/lookup/sequence.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible.errors import AnsibleError -import ansible.utils as utils from re import compile as re_compile, IGNORECASE +from ansible.errors import * +from ansible.parsing.splitter import parse_kv +from ansible.plugins.lookup import LookupBase +from ansible.template import Templar + # shortcut format NUM = "(0?x?[0-9a-f]+)" SHORTCUT = re_compile( @@ -34,7 +37,7 @@ SHORTCUT = re_compile( ) -class LookupModule(object): +class LookupModule(LookupBase): """ sequence lookup module @@ -73,10 +76,6 @@ class LookupModule(object): calculating the number of entries in a sequence when a stride is specified. """ - def __init__(self, basedir, **kwargs): - """absorb any keyword args""" - self.basedir = basedir - def reset(self): """set sensible defaults""" self.start = 1 @@ -170,26 +169,24 @@ class LookupModule(object): "problem formatting %r with %r" % self.format ) - def run(self, terms, inject=None, **kwargs): + def run(self, terms, variables, **kwargs): results = [] - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - if isinstance(terms, basestring): terms = [ terms ] + templar = Templar(loader=self._loader, variables=variables) + for term in terms: try: self.reset() # clear out things for this iteration + term = templar.template(term) try: if not self.parse_simple_args(term): - self.parse_kv_args(utils.parse_kv(term)) - except Exception: - raise AnsibleError( - "unknown error parsing with_sequence arguments: %r" - % term - ) + self.parse_kv_args(parse_kv(term)) + except Exception, e: + raise AnsibleError("unknown error parsing with_sequence arguments: %r. Error was: %s" % (term, e)) self.sanity_check() diff --git a/v2/ansible/plugins/lookup/subelements.py b/v2/ansible/plugins/lookup/subelements.py index f33aae717d..93e9e570c4 100644 --- a/v2/ansible/plugins/lookup/subelements.py +++ b/v2/ansible/plugins/lookup/subelements.py @@ -15,27 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -import ansible.utils as utils -import ansible.errors as errors +from ansible.errors import * +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms +class LookupModule(LookupBase): -class LookupModule(object): + def run(self, terms, variables, **kwargs): - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - - - def run(self, terms, inject=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - terms[0] = utils.listify_lookup_plugin_terms(terms[0], self.basedir, inject) + terms[0] = listify_lookup_plugin_terms(terms[0], variables, loader=self._loader) if not isinstance(terms, list) or not len(terms) == 2: - raise errors.AnsibleError( - "subelements lookup expects a list of two items, first a dict or a list, and second a string") - terms[0] = utils.listify_lookup_plugin_terms(terms[0], self.basedir, inject) - if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], basestring): - raise errors.AnsibleError( - "subelements lookup expects a list of two items, first a dict or a list, and second a string") + raise AnsibleError("subelements lookup expects a list of two items, first a dict or a list, and second a string") if isinstance(terms[0], dict): # convert to list: if terms[0].get('skipped',False) != False: @@ -46,19 +37,20 @@ class LookupModule(object): elementlist.append(terms[0][key]) else: elementlist = terms[0] + subelement = terms[1] ret = [] for item0 in elementlist: if not isinstance(item0, dict): - raise errors.AnsibleError("subelements lookup expects a dictionary, got '%s'" %item0) - if item0.get('skipped',False) != False: + raise AnsibleError("subelements lookup expects a dictionary, got '%s'" %item0) + if item0.get('skipped', False) != False: # this particular item is to be skipped continue if not subelement in item0: - raise errors.AnsibleError("could not find '%s' key in iterated item '%s'" % (subelement, item0)) + raise AnsibleError("could not find '%s' key in iterated item '%s'" % (subelement, item0)) if not isinstance(item0[subelement], list): - raise errors.AnsibleError("the key %s should point to a list, got '%s'" % (subelement, item0[subelement])) + raise AnsibleError("the key %s should point to a list, got '%s'" % (subelement, item0[subelement])) sublist = item0.pop(subelement, []) for item1 in sublist: ret.append((item0, item1)) diff --git a/v2/ansible/plugins/lookup/template.py b/v2/ansible/plugins/lookup/template.py index e009b6b76b..74406f6445 100644 --- a/v2/ansible/plugins/lookup/template.py +++ b/v2/ansible/plugins/lookup/template.py @@ -15,19 +15,29 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -from ansible.utils import template -import ansible.utils as utils +import os -class LookupModule(object): +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.template import Templar - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir +class LookupModule(LookupBase): - def run(self, terms, inject=None, **kwargs): + def run(self, terms, variables, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) + if not isinstance(terms, list): + terms = [ terms ] + + templar = Templar(loader=self._loader, variables=variables) ret = [] for term in terms: - ret.append(template.template_from_file(self.basedir, term, inject)) + path = self._loader.path_dwim(term) + if os.path.exists(path): + with open(path, 'r') as f: + template_data = f.read() + res = templar.template(template_data, preserve_trailing_newlines=True) + ret.append(res) + else: + raise AnsibleError("the template file %s could not be found for the lookup" % term) return ret diff --git a/v2/ansible/plugins/lookup/together.py b/v2/ansible/plugins/lookup/together.py index 07332c9fb9..8b5ff5c891 100644 --- a/v2/ansible/plugins/lookup/together.py +++ b/v2/ansible/plugins/lookup/together.py @@ -15,23 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -import ansible.utils as utils -from ansible.utils import safe_eval -import ansible.errors as errors from itertools import izip_longest -def flatten(terms): - ret = [] - for term in terms: - if isinstance(term, list): - ret.extend(term) - elif isinstance(term, tuple): - ret.extend(term) - else: - ret.append(term) - return ret +from ansible.errors import * +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms -class LookupModule(object): +class LookupModule(LookupBase): """ Transpose a list of arrays: [1, 2, 3], [4, 5, 6] -> [1, 4], [2, 5], [3, 6] @@ -39,26 +29,20 @@ class LookupModule(object): [1, 2], [3] -> [1, 3], [2, None] """ - def __init__(self, basedir=None, **kwargs): - self.basedir = basedir - - def __lookup_injects(self, terms, inject): + def __lookup_variabless(self, terms, variables): results = [] for x in terms: - intermediate = utils.listify_lookup_plugin_terms(x, self.basedir, inject) + intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader) results.append(intermediate) return results - def run(self, terms, inject=None, **kwargs): - - # this code is common with 'items.py' consider moving to utils if we need it again + def run(self, terms, variables=None, **kwargs): - terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) - terms = self.__lookup_injects(terms, inject) + terms = self.__lookup_variabless(terms, variables) my_list = terms[:] if len(my_list) == 0: raise errors.AnsibleError("with_together requires at least one element in each list") - return [flatten(x) for x in izip_longest(*my_list, fillvalue=None)] + return [self._flatten(x) for x in izip_longest(*my_list, fillvalue=None)] diff --git a/v2/ansible/plugins/shell/csh.py b/v2/ansible/plugins/shell/csh.py new file mode 100644 index 0000000000..137c013c12 --- /dev/null +++ b/v2/ansible/plugins/shell/csh.py @@ -0,0 +1,23 @@ +# (c) 2014, Chris Church <chris@ninemoreminutes.com> +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.runner.shell_plugins.sh import ShellModule as ShModule + +class ShellModule(ShModule): + + def env_prefix(self, **kwargs): + return 'env %s' % super(ShellModule, self).env_prefix(**kwargs) diff --git a/v2/ansible/plugins/shell/fish.py b/v2/ansible/plugins/shell/fish.py new file mode 100644 index 0000000000..137c013c12 --- /dev/null +++ b/v2/ansible/plugins/shell/fish.py @@ -0,0 +1,23 @@ +# (c) 2014, Chris Church <chris@ninemoreminutes.com> +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from ansible.runner.shell_plugins.sh import ShellModule as ShModule + +class ShellModule(ShModule): + + def env_prefix(self, **kwargs): + return 'env %s' % super(ShellModule, self).env_prefix(**kwargs) diff --git a/v2/ansible/plugins/shell/powershell.py b/v2/ansible/plugins/shell/powershell.py new file mode 100644 index 0000000000..7254df6f7e --- /dev/null +++ b/v2/ansible/plugins/shell/powershell.py @@ -0,0 +1,117 @@ +# (c) 2014, Chris Church <chris@ninemoreminutes.com> +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import base64 +import os +import re +import random +import shlex +import time + +_common_args = ['PowerShell', '-NoProfile', '-NonInteractive'] + +# Primarily for testing, allow explicitly specifying PowerShell version via +# an environment variable. +_powershell_version = os.environ.get('POWERSHELL_VERSION', None) +if _powershell_version: + _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:] + +def _escape(value, include_vars=False): + '''Return value escaped for use in PowerShell command.''' + # http://www.techotopia.com/index.php/Windows_PowerShell_1.0_String_Quoting_and_Escape_Sequences + # http://stackoverflow.com/questions/764360/a-list-of-string-replacements-in-python + subs = [('\n', '`n'), ('\r', '`r'), ('\t', '`t'), ('\a', '`a'), + ('\b', '`b'), ('\f', '`f'), ('\v', '`v'), ('"', '`"'), + ('\'', '`\''), ('`', '``'), ('\x00', '`0')] + if include_vars: + subs.append(('$', '`$')) + pattern = '|'.join('(%s)' % re.escape(p) for p, s in subs) + substs = [s for p, s in subs] + replace = lambda m: substs[m.lastindex - 1] + return re.sub(pattern, replace, value) + +def _encode_script(script, as_list=False): + '''Convert a PowerShell script to a single base64-encoded command.''' + script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()]) + encoded_script = base64.b64encode(script.encode('utf-16-le')) + cmd_parts = _common_args + ['-EncodedCommand', encoded_script] + if as_list: + return cmd_parts + return ' '.join(cmd_parts) + +def _build_file_cmd(cmd_parts): + '''Build command line to run a file, given list of file name plus args.''' + return ' '.join(_common_args + ['-ExecutionPolicy', 'Unrestricted', '-File'] + ['"%s"' % x for x in cmd_parts]) + +class ShellModule(object): + + def env_prefix(self, **kwargs): + return '' + + def join_path(self, *args): + return os.path.join(*args).replace('/', '\\') + + def path_has_trailing_slash(self, path): + # Allow Windows paths to be specified using either slash. + return path.endswith('/') or path.endswith('\\') + + def chmod(self, mode, path): + return '' + + def remove(self, path, recurse=False): + path = _escape(path) + if recurse: + return _encode_script('''Remove-Item "%s" -Force -Recurse;''' % path) + else: + return _encode_script('''Remove-Item "%s" -Force;''' % path) + + def mkdtemp(self, basefile, system=False, mode=None): + basefile = _escape(basefile) + # FIXME: Support system temp path! + return _encode_script('''(New-Item -Type Directory -Path $env:temp -Name "%s").FullName | Write-Host -Separator '';''' % basefile) + + def md5(self, path): + path = _escape(path) + script = ''' + If (Test-Path -PathType Leaf "%(path)s") + { + $sp = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider; + $fp = [System.IO.File]::Open("%(path)s", [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read); + [System.BitConverter]::ToString($sp.ComputeHash($fp)).Replace("-", "").ToLower(); + $fp.Dispose(); + } + ElseIf (Test-Path -PathType Container "%(path)s") + { + Write-Host "3"; + } + Else + { + Write-Host "1"; + } + ''' % dict(path=path) + return _encode_script(script) + + def build_module_command(self, env_string, shebang, cmd, rm_tmp=None): + cmd = cmd.encode('utf-8') + cmd_parts = shlex.split(cmd, posix=False) + if not cmd_parts[0].lower().endswith('.ps1'): + cmd_parts[0] = '%s.ps1' % cmd_parts[0] + script = _build_file_cmd(cmd_parts) + if rm_tmp: + rm_tmp = _escape(rm_tmp) + script = '%s; Remove-Item "%s" -Force -Recurse;' % (script, rm_tmp) + return _encode_script(script) diff --git a/v2/ansible/plugins/shell/sh.py b/v2/ansible/plugins/shell/sh.py new file mode 100644 index 0000000000..5462429743 --- /dev/null +++ b/v2/ansible/plugins/shell/sh.py @@ -0,0 +1,114 @@ +# (c) 2014, Chris Church <chris@ninemoreminutes.com> +# +# This file is part of Ansible. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os +import re +import pipes +import ansible.constants as C + +_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$') + +class ShellModule(object): + + def env_prefix(self, **kwargs): + '''Build command prefix with environment variables.''' + env = dict( + LANG = C.DEFAULT_MODULE_LANG, + LC_CTYPE = C.DEFAULT_MODULE_LANG, + ) + env.update(kwargs) + return ' '.join(['%s=%s' % (k, pipes.quote(unicode(v))) for k,v in env.items()]) + + def join_path(self, *args): + return os.path.join(*args) + + def path_has_trailing_slash(self, path): + return path.endswith('/') + + def chmod(self, mode, path): + path = pipes.quote(path) + return 'chmod %s %s' % (mode, path) + + def remove(self, path, recurse=False): + path = pipes.quote(path) + if recurse: + return "rm -rf %s >/dev/null 2>&1" % path + else: + return "rm -f %s >/dev/null 2>&1" % path + + def mkdtemp(self, basefile=None, system=False, mode=None): + if not basefile: + basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48)) + basetmp = self.join_path(C.DEFAULT_REMOTE_TMP, basefile) + if system and basetmp.startswith('$HOME'): + basetmp = self.join_path('/tmp', basefile) + cmd = 'mkdir -p %s' % basetmp + if mode: + cmd += ' && chmod %s %s' % (mode, basetmp) + cmd += ' && echo %s' % basetmp + return cmd + + def expand_user(self, user_home_path): + ''' Return a command to expand tildes in a path + + It can be either "~" or "~username". We use the POSIX definition of + a username: + http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426 + http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276 + ''' + + # Check that the user_path to expand is safe + if user_home_path != '~': + if not _USER_HOME_PATH_RE.match(user_home_path): + # pipes.quote will make the shell return the string verbatim + user_home_path = pipes.quote(user_home_path) + return 'echo %s' % user_home_path + + def checksum(self, path, python_interp): + # The following test needs to be SH-compliant. BASH-isms will + # not work if /bin/sh points to a non-BASH shell. + # + # In the following test, each condition is a check and logical + # comparison (|| or &&) that sets the rc value. Every check is run so + # the last check in the series to fail will be the rc that is + # returned. + # + # If a check fails we error before invoking the hash functions because + # hash functions may successfully take the hash of a directory on BSDs + # (UFS filesystem?) which is not what the rest of the ansible code + # expects + # + # If all of the available hashing methods fail we fail with an rc of + # 0. This logic is added to the end of the cmd at the bottom of this + # function. + + test = "rc=flag; [ -r \'%(p)s\' ] || rc=2; [ -f \'%(p)s\' ] || rc=1; [ -d \'%(p)s\' ] && rc=3; %(i)s -V 2>/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] && echo \"${rc}\"\' %(p)s\' && exit 0" % dict(p=path, i=python_interp) + csums = [ + "(%s -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();\nafile = open(\"%s\", \"rb\")\nbuf = afile.read(BLOCKSIZE)\nwhile len(buf) > 0:\n\thasher.update(buf)\n\tbuf = afile.read(BLOCKSIZE)\nafile.close()\nprint(hasher.hexdigest())' 2>/dev/null)" % (python_interp, path), # Python > 2.4 (including python3) + "(%s -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha1();\nafile = open(\"%s\", \"rb\")\nbuf = afile.read(BLOCKSIZE)\nwhile len(buf) > 0:\n\thasher.update(buf)\n\tbuf = afile.read(BLOCKSIZE)\nafile.close()\nprint(hasher.hexdigest())' 2>/dev/null)" % (python_interp, path), # Python == 2.4 + ] + + cmd = " || ".join(csums) + cmd = "%s; %s || (echo \'0 %s\')" % (test, cmd, path) + return cmd + + def build_module_command(self, env_string, shebang, cmd, rm_tmp=None): + cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd] + new_cmd = " ".join(cmd_parts) + if rm_tmp: + new_cmd = '%s; rm -rf %s >/dev/null 2>&1' % (new_cmd, rm_tmp) + return new_cmd diff --git a/v2/ansible/plugins/strategies/__init__.py b/v2/ansible/plugins/strategies/__init__.py new file mode 100644 index 0000000000..b8ae6ffa85 --- /dev/null +++ b/v2/ansible/plugins/strategies/__init__.py @@ -0,0 +1,367 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import Queue +import time + +from ansible.errors import * + +from ansible.inventory.host import Host +from ansible.inventory.group import Group + +from ansible.playbook.helpers import compile_block_list +from ansible.playbook.role import ROLE_CACHE, hash_params +from ansible.plugins import module_loader +from ansible.utils.debug import debug + + +__all__ = ['StrategyBase'] + + +class StrategyBase: + + ''' + This is the base class for strategy plugins, which contains some common + code useful to all strategies like running handlers, cleanup actions, etc. + ''' + + def __init__(self, tqm): + self._tqm = tqm + self._inventory = tqm.get_inventory() + self._workers = tqm.get_workers() + self._notified_handlers = tqm.get_notified_handlers() + self._callback = tqm.get_callback() + self._variable_manager = tqm.get_variable_manager() + self._loader = tqm.get_loader() + self._final_q = tqm._final_q + + # internal counters + self._pending_results = 0 + self._cur_worker = 0 + + # this dictionary is used to keep track of hosts that have + # outstanding tasks still in queue + self._blocked_hosts = dict() + + def run(self, iterator, connection_info, result=True): + # save the counts on failed/unreachable hosts, as the cleanup/handler + # methods will clear that information during their runs + num_failed = len(self._tqm._failed_hosts) + num_unreachable = len(self._tqm._unreachable_hosts) + + debug("running the cleanup portion of the play") + result &= self.cleanup(iterator, connection_info) + debug("running handlers") + result &= self.run_handlers(iterator, connection_info) + + if not result: + if num_unreachable > 0: + return 3 + elif num_failed > 0: + return 2 + else: + return 1 + else: + return 0 + + def get_hosts_remaining(self, play): + return [host for host in self._inventory.get_hosts(play.hosts) if host.name not in self._tqm._failed_hosts and host.get_name() not in self._tqm._unreachable_hosts] + + def get_failed_hosts(self): + return [host for host in self._inventory.get_hosts() if host.name in self._tqm._failed_hosts] + + def _queue_task(self, host, task, task_vars, connection_info): + ''' handles queueing the task up to be sent to a worker ''' + + debug("entering _queue_task() for %s/%s" % (host, task)) + + # and then queue the new task + debug("%s - putting task (%s) in queue" % (host, task)) + try: + debug("worker is %d (out of %d available)" % (self._cur_worker+1, len(self._workers))) + + (worker_prc, main_q, rslt_q) = self._workers[self._cur_worker] + self._cur_worker += 1 + if self._cur_worker >= len(self._workers): + self._cur_worker = 0 + + self._pending_results += 1 + main_q.put((host, task, self._loader.get_basedir(), task_vars, connection_info, module_loader), block=False) + except (EOFError, IOError, AssertionError), e: + # most likely an abort + debug("got an error while queuing: %s" % e) + return + debug("exiting _queue_task() for %s/%s" % (host, task)) + + def _process_pending_results(self): + ''' + Reads results off the final queue and takes appropriate action + based on the result (executing callbacks, updating state, etc.). + ''' + + while not self._final_q.empty() and not self._tqm._terminated: + try: + result = self._final_q.get(block=False) + debug("got result from result worker: %s" % (result,)) + + # all host status messages contain 2 entries: (msg, task_result) + if result[0] in ('host_task_ok', 'host_task_failed', 'host_task_skipped', 'host_unreachable'): + task_result = result[1] + host = task_result._host + task = task_result._task + if result[0] == 'host_task_failed': + if not task.ignore_errors: + self._tqm._failed_hosts[host.get_name()] = True + self._callback.runner_on_failed(task, task_result) + elif result[0] == 'host_unreachable': + self._tqm._unreachable_hosts[host.get_name()] = True + self._callback.runner_on_unreachable(task, task_result) + elif result[0] == 'host_task_skipped': + self._callback.runner_on_skipped(task, task_result) + elif result[0] == 'host_task_ok': + self._callback.runner_on_ok(task, task_result) + + self._pending_results -= 1 + if host.name in self._blocked_hosts: + del self._blocked_hosts[host.name] + + # If this is a role task, mark the parent role as being run (if + # the task was ok or failed, but not skipped or unreachable) + if task_result._task._role is not None and result[0] in ('host_task_ok', 'host_task_failed'): + # lookup the role in the ROLE_CACHE to make sure we're dealing + # with the correct object and mark it as executed + for (entry, role_obj) in ROLE_CACHE[task_result._task._role._role_name].iteritems(): + #hashed_entry = frozenset(task_result._task._role._role_params.iteritems()) + hashed_entry = hash_params(task_result._task._role._role_params) + if entry == hashed_entry : + role_obj._had_task_run = True + + elif result[0] == 'add_host': + task_result = result[1] + new_host_info = task_result.get('add_host', dict()) + + self._add_host(new_host_info) + + elif result[0] == 'add_group': + host = result[1] + task_result = result[2] + group_name = task_result.get('add_group') + + self._add_group(host, group_name) + + elif result[0] == 'notify_handler': + host = result[1] + handler_name = result[2] + + if handler_name not in self._notified_handlers: + self._notified_handlers[handler_name] = [] + + if host not in self._notified_handlers[handler_name]: + self._notified_handlers[handler_name].append(host) + + elif result[0] == 'set_host_var': + host = result[1] + var_name = result[2] + var_value = result[3] + self._variable_manager.set_host_variable(host, var_name, var_value) + + elif result[0] == 'set_host_facts': + host = result[1] + facts = result[2] + self._variable_manager.set_host_facts(host, facts) + + else: + raise AnsibleError("unknown result message received: %s" % result[0]) + except Queue.Empty: + pass + + def _wait_on_pending_results(self): + ''' + Wait for the shared counter to drop to zero, using a short sleep + between checks to ensure we don't spin lock + ''' + + while self._pending_results > 0 and not self._tqm._terminated: + debug("waiting for pending results (%d left)" % self._pending_results) + self._process_pending_results() + if self._tqm._terminated: + break + time.sleep(0.01) + + def _add_host(self, host_info): + ''' + Helper function to add a new host to inventory based on a task result. + ''' + + host_name = host_info.get('host_name') + + # Check if host in cache, add if not + if host_name in self._inventory._hosts_cache: + new_host = self._inventory._hosts_cache[host_name] + else: + new_host = Host(host_name) + self._inventory._hosts_cache[host_name] = new_host + + allgroup = self._inventory.get_group('all') + allgroup.add_host(new_host) + + # Set/update the vars for this host + # FIXME: probably should have a set vars method for the host? + new_vars = host_info.get('host_vars', dict()) + new_host.vars.update(new_vars) + + new_groups = host_info.get('groups', []) + for group_name in new_groups: + if not self._inventory.get_group(group_name): + new_group = Group(group_name) + self._inventory.add_group(new_group) + new_group.vars = self._inventory.get_group_variables(group_name) + else: + new_group = self._inventory.get_group(group_name) + + new_group.add_host(new_host) + + # add this host to the group cache + if self._inventory._groups_list is not None: + if group_name in self._inventory._groups_list: + if new_host.name not in self._inventory._groups_list[group_name]: + self._inventory._groups_list[group_name].append(new_host.name) + + # clear pattern caching completely since it's unpredictable what + # patterns may have referenced the group + # FIXME: is this still required? + self._inventory.clear_pattern_cache() + + def _add_group(self, host, group_name): + ''' + Helper function to add a group (if it does not exist), and to assign the + specified host to that group. + ''' + + new_group = self._inventory.get_group(group_name) + if not new_group: + # create the new group and add it to inventory + new_group = Group(group_name) + self._inventory.add_group(new_group) + + # and add the group to the proper hierarchy + allgroup = self._inventory.get_group('all') + allgroup.add_child_group(new_group) + + # the host here is from the executor side, which means it was a + # serialized/cloned copy and we'll need to look up the proper + # host object from the master inventory + actual_host = self._inventory.get_host(host.name) + + # and add the host to the group + new_group.add_host(actual_host) + + def cleanup(self, iterator, connection_info): + ''' + Iterates through failed hosts and runs any outstanding rescue/always blocks + and handlers which may still need to be run after a failure. + ''' + + debug("in cleanup") + result = True + + debug("getting failed hosts") + failed_hosts = self.get_failed_hosts() + if len(failed_hosts) == 0: + debug("there are no failed hosts") + return result + + debug("marking hosts failed in the iterator") + # mark the host as failed in the iterator so it will take + # any required rescue paths which may be outstanding + for host in failed_hosts: + iterator.mark_host_failed(host) + + debug("clearing the failed hosts list") + # clear the failed hosts dictionary now while also + for entry in self._tqm._failed_hosts.keys(): + del self._tqm._failed_hosts[entry] + + work_to_do = True + while work_to_do: + work_to_do = False + for host in failed_hosts: + host_name = host.get_name() + + if host_name in self._tqm._failed_hosts: + iterator.mark_host_failed(host) + del self._tqm._failed_hosts[host_name] + + if host_name not in self._tqm._unreachable_hosts and iterator.get_next_task_for_host(host, peek=True): + work_to_do = True + # check to see if this host is blocked (still executing a previous task) + if not host_name in self._blocked_hosts: + # pop the task, mark the host blocked, and queue it + self._blocked_hosts[host_name] = True + task = iterator.get_next_task_for_host(host) + self._callback.playbook_on_cleanup_task_start(task.get_name()) + self._queue_task(iterator._play, host, task, connection_info) + + self._process_pending_results() + + # no more work, wait until the queue is drained + self._wait_on_pending_results() + + return result + + def run_handlers(self, iterator, connection_info): + ''' + Runs handlers on those hosts which have been notified. + ''' + + result = True + + # FIXME: getting the handlers from the iterators play should be + # a method on the iterator, which may also filter the list + # of handlers based on the notified list + handlers = compile_block_list(iterator._play.handlers) + + debug("handlers are: %s" % handlers) + for handler in handlers: + handler_name = handler.get_name() + + if handler_name in self._notified_handlers and len(self._notified_handlers[handler_name]): + if not len(self.get_hosts_remaining()): + self._callback.playbook_on_no_hosts_remaining() + result = False + break + + self._callback.playbook_on_handler_task_start(handler_name) + for host in self._notified_handlers[handler_name]: + if not handler.has_triggered(host): + temp_data = handler.serialize() + self._queue_task(iterator._play, host, handler, connection_info) + handler.flag_for_host(host) + + self._process_pending_results() + + self._wait_on_pending_results() + + # wipe the notification list + self._notified_handlers[handler_name] = [] + + debug("done running handlers, result is: %s" % result) + return result diff --git a/v2/ansible/plugins/strategies/free.py b/v2/ansible/plugins/strategies/free.py new file mode 100644 index 0000000000..6aab495fec --- /dev/null +++ b/v2/ansible/plugins/strategies/free.py @@ -0,0 +1,110 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import time + +from ansible.plugins.strategies import StrategyBase + +class StrategyModule(StrategyBase): + + def run(self, iterator, connection_info): + ''' + The "free" strategy is a bit more complex, in that it allows tasks to + be sent to hosts as quickly as they can be processed. This means that + some hosts may finish very quickly if run tasks result in little or no + work being done versus other systems. + + The algorithm used here also tries to be more "fair" when iterating + through hosts by remembering the last host in the list to be given a task + and starting the search from there as opposed to the top of the hosts + list again, which would end up favoring hosts near the beginning of the + list. + ''' + + # the last host to be given a task + last_host = 0 + + work_to_do = True + while work_to_do and not self._tqm._terminated: + + hosts_left = self.get_hosts_remaining() + if len(hosts_left) == 0: + self._callback.playbook_on_no_hosts_remaining() + break + + # using .qsize() is a best estimate anyway, due to the + # multiprocessing/threading concerns (per the python docs) + if 1: #if self._job_queue.qsize() < len(hosts_left): + + work_to_do = False # assume we have no more work to do + starting_host = last_host # save current position so we know when we've + # looped back around and need to break + + # try and find an unblocked host with a task to run + while True: + host = hosts_left[last_host] + host_name = host.get_name() + + # peek at the next task for the host, to see if there's + # anything to do do for this host + if host_name not in self._tqm._failed_hosts and host_name not in self._tqm._unreachable_hosts and iterator.get_next_task_for_host(host, peek=True): + + # FIXME: check task tags, etc. here as we do in linear + # FIXME: handle meta tasks here, which will require a tweak + # to run_handlers so that only the handlers on this host + # are flushed and not all + + # set the flag so the outer loop knows we've still found + # some work which needs to be done + work_to_do = True + + # check to see if this host is blocked (still executing a previous task) + if not host_name in self._blocked_hosts: + # pop the task, mark the host blocked, and queue it + self._blocked_hosts[host_name] = True + task = iterator.get_next_task_for_host(host) + #self._callback.playbook_on_task_start(task.get_name(), False) + self._queue_task(iterator._play, host, task, connection_info) + + # move on to the next host and make sure we + # haven't gone past the end of our hosts list + last_host += 1 + if last_host > len(hosts_left) - 1: + last_host = 0 + + # if we've looped around back to the start, break out + if last_host == starting_host: + break + + # pause briefly so we don't spin lock + time.sleep(0.05) + + try: + self._wait_for_pending_results() + except: + # FIXME: ctrl+c can cause some failures here, so catch them + # with the appropriate error type + pass + + # run the base class run() method, which executes the cleanup function + # and runs any outstanding handlers which have been triggered + super(StrategyModule, self).run(iterator, connection_info) + diff --git a/v2/ansible/plugins/strategies/linear.py b/v2/ansible/plugins/strategies/linear.py new file mode 100644 index 0000000000..b77381ce80 --- /dev/null +++ b/v2/ansible/plugins/strategies/linear.py @@ -0,0 +1,113 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.errors import AnsibleError +from ansible.plugins.strategies import StrategyBase +from ansible.utils.debug import debug + +class StrategyModule(StrategyBase): + + def run(self, iterator, connection_info): + ''' + The linear strategy is simple - get the next task and queue + it for all hosts, then wait for the queue to drain before + moving on to the next task + ''' + + result = True + + # iteratate over each task, while there is one left to run + work_to_do = True + while work_to_do and not self._tqm._terminated: + + try: + debug("getting the remaining hosts for this loop") + hosts_left = self.get_hosts_remaining(iterator._play) + debug("done getting the remaining hosts for this loop") + if len(hosts_left) == 0: + debug("out of hosts to run on") + self._callback.playbook_on_no_hosts_remaining() + result = False + break + + # queue up this task for each host in the inventory + callback_sent = False + work_to_do = False + for host in hosts_left: + while True: + task = iterator.get_next_task_for_host(host) + if not task: + break + + debug("getting variables") + task_vars = self._variable_manager.get_vars(loader=self._loader, play=iterator._play, host=host, task=task) + debug("done getting variables") + + # check to see if this task should be skipped, due to it being a member of a + # role which has already run (and whether that role allows duplicate execution) + if task._role and task._role.has_run(): + # If there is no metadata, the default behavior is to not allow duplicates, + # if there is metadata, check to see if the allow_duplicates flag was set to true + if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: + debug("'%s' skipped because role has already run" % task) + continue + + if not task.evaluate_tags(connection_info.only_tags, connection_info.skip_tags, task_vars) and task.action != 'setup': + debug("'%s' failed tag evaluation" % task) + continue + + break + + if not task: + continue + + work_to_do = True + if task.action == 'meta': + # meta tasks store their args in the _raw_params field of args, + # since they do not use k=v pairs, so get that + meta_action = task.args.get('_raw_params') + if meta_action == 'flush_handlers': + self.run_handlers(iterator, connection_info) + else: + raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds) + else: + if not callback_sent: + self._callback.playbook_on_task_start(task.get_name(), False) + callback_sent = True + + self._blocked_hosts[host.get_name()] = True + self._queue_task(host, task, task_vars, connection_info) + + self._process_pending_results() + + debug("done queuing things up, now waiting for results queue to drain") + self._wait_on_pending_results() + debug("results queue empty") + except (IOError, EOFError), e: + debug("got IOError/EOFError in task loop: %s" % e) + # most likely an abort, return failed + return 1 + + # run the base class run() method, which executes the cleanup function + # and runs any outstanding handlers which have been triggered + + return super(StrategyModule, self).run(iterator, connection_info, result) + diff --git a/v2/ansible/template/__init__.py b/v2/ansible/template/__init__.py new file mode 100644 index 0000000000..46bbc06a07 --- /dev/null +++ b/v2/ansible/template/__init__.py @@ -0,0 +1,265 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from jinja2 import Environment +from jinja2.exceptions import TemplateSyntaxError, UndefinedError +from jinja2.utils import concat as j2_concat +from jinja2.runtime import StrictUndefined + +from ansible import constants as C +from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable +from ansible.plugins import filter_loader, lookup_loader +from ansible.template.safe_eval import safe_eval +from ansible.template.template import AnsibleJ2Template +from ansible.template.vars import AnsibleJ2Vars +from ansible.utils.debug import debug + +__all__ = ['Templar'] + +JINJA2_OVERRIDE = '#jinja2:' +JINJA2_ALLOWED_OVERRIDES = ['trim_blocks', 'lstrip_blocks', 'newline_sequence', 'keep_trailing_newline'] + +class Templar: + ''' + The main class for templating, with the main entry-point of template(). + ''' + + def __init__(self, loader, variables=dict(), fail_on_undefined=C.DEFAULT_UNDEFINED_VAR_BEHAVIOR): + self._loader = loader + self._basedir = loader.get_basedir() + self._filters = None + self._available_variables = variables + + # flags to determine whether certain failures during templating + # should result in fatal errors being raised + self._fail_on_lookup_errors = True + self._fail_on_filter_errors = True + self._fail_on_undefined_errors = fail_on_undefined + + def _count_newlines_from_end(self, in_str): + ''' + Counts the number of newlines at the end of a string. This is used during + the jinja2 templating to ensure the count matches the input, since some newlines + may be thrown away during the templating. + ''' + + i = len(in_str) + while i > 0: + if in_str[i-1] != '\n': + break + i -= 1 + + return len(in_str) - i + + def _get_filters(self): + ''' + Returns filter plugins, after loading and caching them if need be + ''' + + if self._filters is not None: + return self._filters.copy() + + plugins = [x for x in filter_loader.all()] + + self._filters = dict() + for fp in plugins: + self._filters.update(fp.filters()) + + return self._filters.copy() + + def _get_extensions(self): + ''' + Return jinja2 extensions to load. + + If some extensions are set via jinja_extensions in ansible.cfg, we try + to load them with the jinja environment. + ''' + + jinja_exts = [] + if C.DEFAULT_JINJA2_EXTENSIONS: + # make sure the configuration directive doesn't contain spaces + # and split extensions in an array + jinja_exts = C.DEFAULT_JINJA2_EXTENSIONS.replace(" ", "").split(',') + + return jinja_exts + + def set_available_variables(self, variables): + ''' + Sets the list of template variables this Templar instance will use + to template things, so we don't have to pass them around between + internal methods. + ''' + + assert isinstance(variables, dict) + self._available_variables = variables.copy() + + def template(self, variable, convert_bare=False, preserve_trailing_newlines=False): + ''' + Templates (possibly recursively) any given data as input. If convert_bare is + set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}') + before being sent through the template engine. + ''' + + try: + if convert_bare: + variable = self._convert_bare_variable(variable) + + if isinstance(variable, basestring): + result = variable + if self._contains_vars(variable): + result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines) + + # if this looks like a dictionary or list, convert it to such using the safe_eval method + if (result.startswith("{") and not result.startswith("{{")) or result.startswith("["): + eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True) + if eval_results[1] is None: + result = eval_results[0] + else: + # FIXME: if the safe_eval raised an error, should we do something with it? + pass + + return result + + elif isinstance(variable, (list, tuple)): + return [self.template(v, convert_bare=convert_bare) for v in variable] + elif isinstance(variable, dict): + d = {} + for (k, v) in variable.iteritems(): + d[k] = self.template(v, convert_bare=convert_bare) + return d + else: + return variable + + except AnsibleFilterError: + if self._fail_on_filter_errors: + raise + else: + return variable + + def _contains_vars(self, data): + ''' + returns True if the data contains a variable pattern + ''' + return "$" in data or "{{" in data or '{%' in data + + def _convert_bare_variable(self, variable): + ''' + Wraps a bare string, which may have an attribute portion (ie. foo.bar) + in jinja2 variable braces so that it is evaluated properly. + ''' + + if isinstance(variable, basestring): + first_part = variable.split(".")[0].split("[")[0] + if first_part in self._available_variables and '{{' not in variable and '$' not in variable: + return "{{%s}}" % variable + + # the variable didn't meet the conditions to be converted, + # so just return it as-is + return variable + + def _finalize(self, thing): + ''' + A custom finalize method for jinja2, which prevents None from being returned + ''' + return thing if thing is not None else '' + + def _lookup(self, name, *args, **kwargs): + instance = lookup_loader.get(name.lower(), loader=self._loader) + + if instance is not None: + # safely catch run failures per #5059 + try: + ran = instance.run(*args, variables=self._available_variables, **kwargs) + except AnsibleUndefinedVariable: + raise + except Exception, e: + if self._fail_on_lookup_errors: + raise + ran = None + if ran: + ran = ",".join(ran) + return ran + else: + raise AnsibleError("lookup plugin (%s) not found" % name) + + def _do_template(self, data, preserve_trailing_newlines=False): + + try: + + environment = Environment(trim_blocks=True, undefined=StrictUndefined, extensions=self._get_extensions(), finalize=self._finalize) + environment.filters.update(self._get_filters()) + environment.template_class = AnsibleJ2Template + + # FIXME: may not be required anymore, as the basedir stuff will + # be handled by the loader? + #if '_original_file' in vars: + # basedir = os.path.dirname(vars['_original_file']) + # filesdir = os.path.abspath(os.path.join(basedir, '..', 'files')) + # if os.path.exists(filesdir): + # basedir = filesdir + + try: + t = environment.from_string(data) + except TemplateSyntaxError, e: + raise AnsibleError("template error while templating string: %s" % str(e)) + except Exception, e: + if 'recursion' in str(e): + raise AnsibleError("recursive loop detected in template string: %s" % data) + else: + return data + + t.globals['lookup'] = self._lookup + t.globals['finalize'] = self._finalize + + jvars = AnsibleJ2Vars(self, t.globals) + + new_context = t.new_context(jvars, shared=True) + rf = t.root_render_func(new_context) + + try: + res = j2_concat(rf) + except TypeError, te: + if 'StrictUndefined' in str(te): + raise AnsibleUndefinedVariable( + "Unable to look up a name or access an attribute in template string. " + \ + "Make sure your variable name does not contain invalid characters like '-'." + ) + else: + debug("failing because of a type error, template data is: %s" % data) + raise AnsibleError("an unexpected type error occurred. Error was %s" % te) + + if preserve_trailing_newlines: + # The low level calls above do not preserve the newline + # characters at the end of the input data, so we use the + # calculate the difference in newlines and append them + # to the resulting output for parity + res_newlines = self._count_newlines_from_end(res) + data_newlines = self._count_newlines_from_end(data) + if data_newlines > res_newlines: + res += '\n' * (data_newlines - res_newlines) + + return res + except (UndefinedError, AnsibleUndefinedVariable), e: + if self._fail_on_undefined_errors: + raise + else: + return data + diff --git a/v2/ansible/template/safe_eval.py b/v2/ansible/template/safe_eval.py new file mode 100644 index 0000000000..ba377054d7 --- /dev/null +++ b/v2/ansible/template/safe_eval.py @@ -0,0 +1,118 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import ast +import sys + +from ansible import constants as C +from ansible.plugins import filter_loader + +def safe_eval(expr, locals={}, include_exceptions=False): + ''' + This is intended for allowing things like: + with_items: a_list_variable + + Where Jinja2 would return a string but we do not want to allow it to + call functions (outside of Jinja2, where the env is constrained). If + the input data to this function came from an untrusted (remote) source, + it should first be run through _clean_data_struct() to ensure the data + is further sanitized prior to evaluation. + + Based on: + http://stackoverflow.com/questions/12523516/using-ast-and-whitelists-to-make-pythons-eval-safe + ''' + + # this is the whitelist of AST nodes we are going to + # allow in the evaluation. Any node type other than + # those listed here will raise an exception in our custom + # visitor class defined below. + SAFE_NODES = set( + ( + ast.Add, + ast.BinOp, + ast.Call, + ast.Compare, + ast.Dict, + ast.Div, + ast.Expression, + ast.List, + ast.Load, + ast.Mult, + ast.Num, + ast.Name, + ast.Str, + ast.Sub, + ast.Tuple, + ast.UnaryOp, + ) + ) + + # AST node types were expanded after 2.6 + if not sys.version.startswith('2.6'): + SAFE_NODES.union( + set( + (ast.Set,) + ) + ) + + filter_list = [] + for filter in filter_loader.all(): + filter_list.extend(filter.filters().keys()) + + CALL_WHITELIST = C.DEFAULT_CALLABLE_WHITELIST + filter_list + + class CleansingNodeVisitor(ast.NodeVisitor): + def generic_visit(self, node, inside_call=False): + if type(node) not in SAFE_NODES: + raise Exception("invalid expression (%s)" % expr) + elif isinstance(node, ast.Call): + inside_call = True + elif isinstance(node, ast.Name) and inside_call: + if hasattr(builtin, node.id) and node.id not in CALL_WHITELIST: + raise Exception("invalid function: %s" % node.id) + # iterate over all child nodes + for child_node in ast.iter_child_nodes(node): + self.generic_visit(child_node, inside_call) + + if not isinstance(expr, basestring): + # already templated to a datastructure, perhaps? + if include_exceptions: + return (expr, None) + return expr + + cnv = CleansingNodeVisitor() + try: + parsed_tree = ast.parse(expr, mode='eval') + cnv.visit(parsed_tree) + compiled = compile(parsed_tree, expr, 'eval') + result = eval(compiled, {}, locals) + + if include_exceptions: + return (result, None) + else: + return result + except SyntaxError, e: + # special handling for syntax errors, we just return + # the expression string back as-is + if include_exceptions: + return (expr, None) + return expr + except Exception, e: + if include_exceptions: + return (expr, e) + return expr + diff --git a/v2/ansible/template/template.py b/v2/ansible/template/template.py new file mode 100644 index 0000000000..a111bec0a5 --- /dev/null +++ b/v2/ansible/template/template.py @@ -0,0 +1,37 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import jinja2 + +__all__ = ['AnsibleJ2Template'] + + +class AnsibleJ2Template(jinja2.environment.Template): + ''' + A helper class, which prevents Jinja2 from running _jinja2_vars through dict(). + Without this, {% include %} and similar will create new contexts unlike the special + one created in template_from_file. This ensures they are all alike, except for + potential locals. + ''' + + def new_context(self, vars=None, shared=False, locals=None): + return jinja2.runtime.Context(self.environment, vars.add_locals(locals), self.name, self.blocks) + diff --git a/v2/ansible/template/vars.py b/v2/ansible/template/vars.py new file mode 100644 index 0000000000..3c0bb61ecb --- /dev/null +++ b/v2/ansible/template/vars.py @@ -0,0 +1,88 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +__all__ = ['AnsibleJ2Vars'] + + +class AnsibleJ2Vars: + ''' + Helper class to template all variable content before jinja2 sees it. This is + done by hijacking the variable storage that jinja2 uses, and overriding __contains__ + and __getitem__ to look like a dict. Added bonus is avoiding duplicating the large + hashes that inject tends to be. + + To facilitate using builtin jinja2 things like range, globals are also handled here. + ''' + + def __init__(self, templar, globals, *extras): + ''' + Initializes this object with a valid Templar() object, as + well as several dictionaries of variables representing + different scopes (in jinja2 terminology). + ''' + + self._templar = templar + self._globals = globals + self._extras = extras + + def __contains__(self, k): + if k in self._templar._available_variables: + return True + for i in self._extras: + if k in i: + return True + if k in self._globals: + return True + return False + + def __getitem__(self, varname): + # FIXME: are we still going to need HostVars? + #from ansible.runner import HostVars + + if varname not in self._templar._available_variables: + for i in self._extras: + if varname in i: + return i[varname] + if varname in self._globals: + return self._globals[varname] + else: + raise KeyError("undefined variable: %s" % varname) + + variable = self._templar._available_variables[varname] + + # HostVars is special, return it as-is, as is the special variable + # 'vars', which contains the vars structure + from ansible.vars.hostvars import HostVars + if isinstance(variable, dict) and varname == "vars" or isinstance(variable, HostVars): + return variable + else: + return self._templar.template(variable) + + def add_locals(self, locals): + ''' + If locals are provided, create a copy of self containing those + locals in addition to what is already in this variable proxy. + ''' + if locals is None: + return self + return AnsibleJ2Vars(self._templar, self._globals, locals, *self._extras) + diff --git a/v2/ansible/playbook/include.py b/v2/ansible/utils/__init__.py index ae8ccff595..ae8ccff595 100644 --- a/v2/ansible/playbook/include.py +++ b/v2/ansible/utils/__init__.py diff --git a/v2/ansible/utils/boolean.py b/v2/ansible/utils/boolean.py new file mode 100644 index 0000000000..bf15be346d --- /dev/null +++ b/v2/ansible/utils/boolean.py @@ -0,0 +1,29 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def boolean(value): + val = str(value) + if val.lower() in [ "true", "t", "y", "1", "yes" ]: + return True + else: + return False + diff --git a/v2/ansible/utils/cli.py b/v2/ansible/utils/cli.py new file mode 100644 index 0000000000..43aa21470d --- /dev/null +++ b/v2/ansible/utils/cli.py @@ -0,0 +1,214 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import operator +import optparse +import os +import time +import yaml + +from ansible import __version__ +from ansible import constants as C + +# FIXME: documentation for methods here, which have mostly been +# copied directly over from the old utils/__init__.py + +class SortedOptParser(optparse.OptionParser): + '''Optparser which sorts the options by opt before outputting --help''' + + def format_help(self, formatter=None): + self.option_list.sort(key=operator.methodcaller('get_opt_string')) + return optparse.OptionParser.format_help(self, formatter=None) + +def base_parser(usage="", output_opts=False, runas_opts=False, + async_opts=False, connect_opts=False, subset_opts=False, check_opts=False, diff_opts=False): + ''' create an options parser for any ansible script ''' + + parser = SortedOptParser(usage, version=version("%prog")) + + parser.add_option('-v','--verbose', dest='verbosity', default=0, action="count", + help="verbose mode (-vvv for more, -vvvv to enable connection debugging)") + parser.add_option('-f','--forks', dest='forks', default=C.DEFAULT_FORKS, type='int', + help="specify number of parallel processes to use (default=%s)" % C.DEFAULT_FORKS) + parser.add_option('-i', '--inventory-file', dest='inventory', + help="specify inventory host file (default=%s)" % C.DEFAULT_HOST_LIST, + default=C.DEFAULT_HOST_LIST) + parser.add_option('-k', '--ask-pass', default=False, dest='ask_pass', action='store_true', + help='ask for SSH password') + parser.add_option('--private-key', default=C.DEFAULT_PRIVATE_KEY_FILE, dest='private_key_file', + help='use this file to authenticate the connection') + parser.add_option('-K', '--ask-sudo-pass', default=False, dest='ask_sudo_pass', action='store_true', + help='ask for sudo password') + parser.add_option('--ask-su-pass', default=False, dest='ask_su_pass', action='store_true', + help='ask for su password') + parser.add_option('--ask-vault-pass', default=False, dest='ask_vault_pass', action='store_true', + help='ask for vault password') + parser.add_option('--vault-password-file', default=C.DEFAULT_VAULT_PASSWORD_FILE, + dest='vault_password_file', help="vault password file") + parser.add_option('--list-hosts', dest='listhosts', action='store_true', + help='outputs a list of matching hosts; does not execute anything else') + parser.add_option('-M', '--module-path', dest='module_path', + help="specify path(s) to module library (default=%s)" % C.DEFAULT_MODULE_PATH, + default=None) + + if subset_opts: + parser.add_option('-l', '--limit', default=C.DEFAULT_SUBSET, dest='subset', + help='further limit selected hosts to an additional pattern') + + parser.add_option('-T', '--timeout', default=C.DEFAULT_TIMEOUT, type='int', + dest='timeout', + help="override the SSH timeout in seconds (default=%s)" % C.DEFAULT_TIMEOUT) + + if output_opts: + parser.add_option('-o', '--one-line', dest='one_line', action='store_true', + help='condense output') + parser.add_option('-t', '--tree', dest='tree', default=None, + help='log output to this directory') + + if runas_opts: + parser.add_option("-s", "--sudo", default=C.DEFAULT_SUDO, action="store_true", + dest='sudo', help="run operations with sudo (nopasswd)") + parser.add_option('-U', '--sudo-user', dest='sudo_user', default=None, + help='desired sudo user (default=root)') # Can't default to root because we need to detect when this option was given + parser.add_option('-u', '--user', default=C.DEFAULT_REMOTE_USER, + dest='remote_user', help='connect as this user (default=%s)' % C.DEFAULT_REMOTE_USER) + + parser.add_option('-S', '--su', default=C.DEFAULT_SU, + action='store_true', help='run operations with su') + parser.add_option('-R', '--su-user', help='run operations with su as this ' + 'user (default=%s)' % C.DEFAULT_SU_USER) + + if connect_opts: + parser.add_option('-c', '--connection', dest='connection', + default=C.DEFAULT_TRANSPORT, + help="connection type to use (default=%s)" % C.DEFAULT_TRANSPORT) + + if async_opts: + parser.add_option('-P', '--poll', default=C.DEFAULT_POLL_INTERVAL, type='int', + dest='poll_interval', + help="set the poll interval if using -B (default=%s)" % C.DEFAULT_POLL_INTERVAL) + parser.add_option('-B', '--background', dest='seconds', type='int', default=0, + help='run asynchronously, failing after X seconds (default=N/A)') + + if check_opts: + parser.add_option("-C", "--check", default=False, dest='check', action='store_true', + help="don't make any changes; instead, try to predict some of the changes that may occur" + ) + + if diff_opts: + parser.add_option("-D", "--diff", default=False, dest='diff', action='store_true', + help="when changing (small) files and templates, show the differences in those files; works great with --check" + ) + + + return parser + +def version(prog): + result = "{0} {1}".format(prog, __version__) + gitinfo = _gitinfo() + if gitinfo: + result = result + " {0}".format(gitinfo) + result = result + "\n configured module search path = %s" % C.DEFAULT_MODULE_PATH + return result + +def version_info(gitinfo=False): + if gitinfo: + # expensive call, user with care + ansible_version_string = version('') + else: + ansible_version_string = __version__ + ansible_version = ansible_version_string.split()[0] + ansible_versions = ansible_version.split('.') + for counter in range(len(ansible_versions)): + if ansible_versions[counter] == "": + ansible_versions[counter] = 0 + try: + ansible_versions[counter] = int(ansible_versions[counter]) + except: + pass + if len(ansible_versions) < 3: + for counter in range(len(ansible_versions), 3): + ansible_versions.append(0) + return {'string': ansible_version_string.strip(), + 'full': ansible_version, + 'major': ansible_versions[0], + 'minor': ansible_versions[1], + 'revision': ansible_versions[2]} + +def _git_repo_info(repo_path): + ''' returns a string containing git branch, commit id and commit date ''' + result = None + if os.path.exists(repo_path): + # Check if the .git is a file. If it is a file, it means that we are in a submodule structure. + if os.path.isfile(repo_path): + try: + gitdir = yaml.safe_load(open(repo_path)).get('gitdir') + # There is a possibility the .git file to have an absolute path. + if os.path.isabs(gitdir): + repo_path = gitdir + else: + repo_path = os.path.join(repo_path[:-4], gitdir) + except (IOError, AttributeError): + return '' + f = open(os.path.join(repo_path, "HEAD")) + branch = f.readline().split('/')[-1].rstrip("\n") + f.close() + branch_path = os.path.join(repo_path, "refs", "heads", branch) + if os.path.exists(branch_path): + f = open(branch_path) + commit = f.readline()[:10] + f.close() + else: + # detached HEAD + commit = branch[:10] + branch = 'detached HEAD' + branch_path = os.path.join(repo_path, "HEAD") + + date = time.localtime(os.stat(branch_path).st_mtime) + if time.daylight == 0: + offset = time.timezone + else: + offset = time.altzone + result = "({0} {1}) last updated {2} (GMT {3:+04d})".format(branch, commit, + time.strftime("%Y/%m/%d %H:%M:%S", date), int(offset / -36)) + else: + result = '' + return result + +def _gitinfo(): + basedir = os.path.join(os.path.dirname(__file__), '..', '..', '..') + repo_path = os.path.join(basedir, '.git') + result = _git_repo_info(repo_path) + submodules = os.path.join(basedir, '.gitmodules') + if not os.path.exists(submodules): + return result + f = open(submodules) + for line in f: + tokens = line.strip().split(' ') + if tokens[0] == 'path': + submodule_path = tokens[2] + submodule_info =_git_repo_info(os.path.join(basedir, submodule_path, '.git')) + if not submodule_info: + submodule_info = ' not found - use git submodule update --init ' + submodule_path + result += "\n {0}: {1}".format(submodule_path, submodule_info) + f.close() + return result + diff --git a/v2/ansible/utils/color.py b/v2/ansible/utils/color.py new file mode 100644 index 0000000000..ebcb4317f7 --- /dev/null +++ b/v2/ansible/utils/color.py @@ -0,0 +1,75 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import sys + +from ansible import constants as C + +ANSIBLE_COLOR=True +if C.ANSIBLE_NOCOLOR: + ANSIBLE_COLOR=False +elif not hasattr(sys.stdout, 'isatty') or not sys.stdout.isatty(): + ANSIBLE_COLOR=False +else: + try: + import curses + curses.setupterm() + if curses.tigetnum('colors') < 0: + ANSIBLE_COLOR=False + except ImportError: + # curses library was not found + pass + except curses.error: + # curses returns an error (e.g. could not find terminal) + ANSIBLE_COLOR=False + +if C.ANSIBLE_FORCE_COLOR: + ANSIBLE_COLOR=True + +# --- begin "pretty" +# +# pretty - A miniature library that provides a Python print and stdout +# wrapper that makes colored terminal text easier to use (e.g. without +# having to mess around with ANSI escape sequences). This code is public +# domain - there is no license except that you must leave this header. +# +# Copyright (C) 2008 Brian Nez <thedude at bri1 dot com> +# +# http://nezzen.net/2008/06/23/colored-text-in-python-using-ansi-escape-sequences/ + +codeCodes = { + 'black': '0;30', 'bright gray': '0;37', + 'blue': '0;34', 'white': '1;37', + 'green': '0;32', 'bright blue': '1;34', + 'cyan': '0;36', 'bright green': '1;32', + 'red': '0;31', 'bright cyan': '1;36', + 'purple': '0;35', 'bright red': '1;31', + 'yellow': '0;33', 'bright purple': '1;35', + 'dark gray': '1;30', 'bright yellow': '1;33', + 'normal': '0' +} + +def stringc(text, color): + """String in color.""" + + if ANSIBLE_COLOR: + return "\033["+codeCodes[color]+"m"+text+"\033[0m" + else: + return text + +# --- end "pretty" + diff --git a/v2/ansible/utils/debug.py b/v2/ansible/utils/debug.py new file mode 100644 index 0000000000..3b37ac50a7 --- /dev/null +++ b/v2/ansible/utils/debug.py @@ -0,0 +1,15 @@ +import os +import time +import sys + +from multiprocessing import Lock + +from ansible import constants as C + +global_debug_lock = Lock() +def debug(msg): + if C.DEFAULT_DEBUG: + global_debug_lock.acquire() + print("%6d %0.5f: %s" % (os.getpid(), time.time(), msg)) + sys.stdout.flush() + global_debug_lock.release() diff --git a/v2/ansible/utils/display.py b/v2/ansible/utils/display.py new file mode 100644 index 0000000000..085d52b2c8 --- /dev/null +++ b/v2/ansible/utils/display.py @@ -0,0 +1,114 @@ +# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# FIXME: copied mostly from old code, needs py3 improvements + +import textwrap + +from ansible import constants as C +from ansible.errors import * +from ansible.utils.color import stringc + +class Display: + + def __init__(self, conn_info=None): + if conn_info: + self._verbosity = conn_info.verbosity + else: + self._verbosity = 0 + + # list of all deprecation messages to prevent duplicate display + self._deprecations = {} + self._warns = {} + + def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False): + msg2 = msg + if color: + msg2 = stringc(msg, color) + if not log_only: + if not stderr: + try: + print msg2 + except UnicodeEncodeError: + print msg2.encode('utf-8') + else: + try: + print >>sys.stderr, msg2 + except UnicodeEncodeError: + print >>sys.stderr, msg2.encode('utf-8') + if C.DEFAULT_LOG_PATH != '': + while msg.startswith("\n"): + msg = msg.replace("\n","") + # FIXME: logger stuff needs to be implemented + #if not screen_only: + # if color == 'red': + # logger.error(msg) + # else: + # logger.info(msg) + + def vv(self, msg, host=None): + return self.verbose(msg, host=host, caplevel=1) + + def vvv(self, msg, host=None): + return self.verbose(msg, host=host, caplevel=2) + + def vvvv(self, msg, host=None): + return self.verbose(msg, host=host, caplevel=3) + + def verbose(self, msg, host=None, caplevel=2): + # FIXME: this needs to be implemented + #msg = utils.sanitize_output(msg) + if self._verbosity > caplevel: + if host is None: + self.display(msg, color='blue') + else: + self.display("<%s> %s" % (host.name, msg), color='blue') + + def deprecated(self, msg, version, removed=False): + ''' used to print out a deprecation message.''' + + if not removed and not C.DEPRECATION_WARNINGS: + return + + if not removed: + if version: + new_msg = "\n[DEPRECATION WARNING]: %s. This feature will be removed in version %s." % (msg, version) + else: + new_msg = "\n[DEPRECATION WARNING]: %s. This feature will be removed in a future release." % (msg) + new_msg = new_msg + " Deprecation warnings can be disabled by setting deprecation_warnings=False in ansible.cfg.\n\n" + else: + raise AnsibleError("[DEPRECATED]: %s. Please update your playbooks." % msg) + + wrapped = textwrap.wrap(new_msg, 79) + new_msg = "\n".join(wrapped) + "\n" + + if new_msg not in deprecations: + self._display(new_msg, color='purple', stderr=True) + self._deprecations[new_msg] = 1 + + def warning(self, msg): + new_msg = "\n[WARNING]: %s" % msg + wrapped = textwrap.wrap(new_msg, 79) + new_msg = "\n".join(wrapped) + "\n" + if new_msg not in warns: + self._display(new_msg, color='bright purple', stderr=True) + self._warns[new_msg] = 1 + + def system_warning(self, msg): + if C.SYSTEM_WARNINGS: + self._warning(msg) + diff --git a/v2/ansible/utils/encrypt.py b/v2/ansible/utils/encrypt.py new file mode 100644 index 0000000000..878b461c86 --- /dev/null +++ b/v2/ansible/utils/encrypt.py @@ -0,0 +1,46 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +PASSLIB_AVAILABLE = False +try: + import passlib.hash + PASSLIB_AVAILABLE = True +except: + pass + +from ansible.errors import AnsibleError + +__all__ = ['do_encrypt'] + +def do_encrypt(result, encrypt, salt_size=None, salt=None): + if PASSLIB_AVAILABLE: + try: + crypt = getattr(passlib.hash, encrypt) + except: + raise AnsibleError("passlib does not support '%s' algorithm" % encrypt) + + if salt_size: + result = crypt.encrypt(result, salt_size=salt_size) + elif salt: + result = crypt.encrypt(result, salt=salt) + else: + result = crypt.encrypt(result) + else: + raise AnsibleError("passlib must be installed to encrypt vars_prompt values") + + return result + diff --git a/v2/ansible/utils/hashing.py b/v2/ansible/utils/hashing.py new file mode 100644 index 0000000000..a7d142e5bd --- /dev/null +++ b/v2/ansible/utils/hashing.py @@ -0,0 +1,91 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +# Note, sha1 is the only hash algorithm compatible with python2.4 and with +# FIPS-140 mode (as of 11-2014) +try: + from hashlib import sha1 as sha1 +except ImportError: + from sha import sha as sha1 + +# Backwards compat only +try: + from hashlib import md5 as _md5 +except ImportError: + try: + from md5 import md5 as _md5 + except ImportError: + # Assume we're running in FIPS mode here + _md5 = None + +def secure_hash_s(data, hash_func=sha1): + ''' Return a secure hash hex digest of data. ''' + + digest = hash_func() + try: + digest.update(data) + except UnicodeEncodeError: + digest.update(data.encode('utf-8')) + return digest.hexdigest() + +def secure_hash(filename, hash_func=sha1): + ''' Return a secure hash hex digest of local file, None if file is not present or a directory. ''' + + if not os.path.exists(filename) or os.path.isdir(filename): + return None + digest = hash_func() + blocksize = 64 * 1024 + try: + infile = open(filename, 'rb') + block = infile.read(blocksize) + while block: + digest.update(block) + block = infile.read(blocksize) + infile.close() + except IOError, e: + raise errors.AnsibleError("error while accessing the file %s, error was: %s" % (filename, e)) + return digest.hexdigest() + +# The checksum algorithm must match with the algorithm in ShellModule.checksum() method +checksum = secure_hash +checksum_s = secure_hash_s + +# Backwards compat functions. Some modules include md5s in their return values +# Continue to support that for now. As of ansible-1.8, all of those modules +# should also return "checksum" (sha1 for now) +# Do not use md5 unless it is needed for: +# 1) Optional backwards compatibility +# 2) Compliance with a third party protocol +# +# MD5 will not work on systems which are FIPS-140-2 compliant. + +def md5s(data): + if not _md5: + raise ValueError('MD5 not available. Possibly running in FIPS mode') + return secure_hash_s(data, _md5) + +def md5(filename): + if not _md5: + raise ValueError('MD5 not available. Possibly running in FIPS mode') + return secure_hash(filename, _md5) + diff --git a/v2/ansible/utils/listify.py b/v2/ansible/utils/listify.py new file mode 100644 index 0000000000..800b99b8ec --- /dev/null +++ b/v2/ansible/utils/listify.py @@ -0,0 +1,67 @@ +# (c) 2014 Michael DeHaan, <michael@ansible.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from six import iteritems, string_types + +import re + +from ansible.template import Templar +from ansible.template.safe_eval import safe_eval + +__all__ = ['listify_lookup_plugin_terms'] + +LOOKUP_REGEX = re.compile(r'lookup\s*\(') + +def listify_lookup_plugin_terms(terms, variables, loader): + + if isinstance(terms, basestring): + # someone did: + # with_items: alist + # OR + # with_items: {{ alist }} + + stripped = terms.strip() + if not (stripped.startswith('{') or stripped.startswith('[')) and \ + not stripped.startswith("/") and \ + not stripped.startswith('set([') and \ + not LOOKUP_REGEX.search(terms): + # if not already a list, get ready to evaluate with Jinja2 + # not sure why the "/" is in above code :) + try: + templar = Templar(loader=loader, variables=variables) + new_terms = templar.template("{{ %s }}" % terms) + if isinstance(new_terms, basestring) and "{{" in new_terms: + pass + else: + terms = new_terms + except: + pass + + if '{' in terms or '[' in terms: + # Jinja2 already evaluated a variable to a list. + # Jinja2-ified list needs to be converted back to a real type + return safe_eval(terms) + + if isinstance(terms, basestring): + terms = [ terms ] + + return terms + diff --git a/v2/ansible/utils/path.py b/v2/ansible/utils/path.py new file mode 100644 index 0000000000..ea7fc201a8 --- /dev/null +++ b/v2/ansible/utils/path.py @@ -0,0 +1,35 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +import os +import stat + +__all__ = ['is_executable', 'unfrackpath'] + +def is_executable(path): + '''is the given path executable?''' + return (stat.S_IXUSR & os.stat(path)[stat.ST_MODE] or stat.S_IXGRP & os.stat(path)[stat.ST_MODE] or stat.S_IXOTH & os.stat(path)[stat.ST_MODE]) + +def unfrackpath(path): + ''' + returns a path that is free of symlinks, environment + variables, relative path traversals and symbols (~) + example: + '$HOME/../../var/mail' becomes '/var/spool/mail' + ''' + return os.path.normpath(os.path.realpath(os.path.expandvars(os.path.expanduser(path)))) + diff --git a/v2/ansible/utils/unicode.py b/v2/ansible/utils/unicode.py new file mode 100644 index 0000000000..b2fcf65161 --- /dev/null +++ b/v2/ansible/utils/unicode.py @@ -0,0 +1,248 @@ +# (c) 2012-2014, Toshio Kuraotmi <a.badger@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# to_bytes and to_unicode were written by Toshio Kuratomi for the +# python-kitchen library https://pypi.python.org/pypi/kitchen +# They are licensed in kitchen under the terms of the GPLv2+ +# They were copied and modified for use in ansible by Toshio in Jan 2015 +# (simply removing the deprecated features) + +#: Aliases for the utf-8 codec +_UTF8_ALIASES = frozenset(('utf-8', 'UTF-8', 'utf8', 'UTF8', 'utf_8', 'UTF_8', + 'utf', 'UTF', 'u8', 'U8')) +#: Aliases for the latin-1 codec +_LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1', + 'latin', 'LATIN', 'l1', 'L1', 'cp819', 'CP819', '8859', 'iso8859-1', + 'ISO8859-1', 'iso-8859-1', 'ISO-8859-1')) + +# EXCEPTION_CONVERTERS is defined below due to using to_unicode + +def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None): + '''Convert an object into a :class:`unicode` string + + :arg obj: Object to convert to a :class:`unicode` string. This should + normally be a byte :class:`str` + :kwarg encoding: What encoding to try converting the byte :class:`str` as. + Defaults to :term:`utf-8` + :kwarg errors: If errors are found while decoding, perform this action. + Defaults to ``replace`` which replaces the invalid bytes with + a character that means the bytes were unable to be decoded. Other + values are the same as the error handling schemes in the `codec base + classes + <http://docs.python.org/library/codecs.html#codec-base-classes>`_. + For instance ``strict`` which raises an exception and ``ignore`` which + simply omits the non-decodable characters. + :kwarg nonstring: How to treat nonstring values. Possible values are: + + :simplerepr: Attempt to call the object's "simple representation" + method and return that value. Python-2.3+ has two methods that + try to return a simple representation: :meth:`object.__unicode__` + and :meth:`object.__str__`. We first try to get a usable value + from :meth:`object.__unicode__`. If that fails we try the same + with :meth:`object.__str__`. + :empty: Return an empty :class:`unicode` string + :strict: Raise a :exc:`TypeError` + :passthru: Return the object unchanged + :repr: Attempt to return a :class:`unicode` string of the repr of the + object + + Default is ``simplerepr`` + + :raises TypeError: if :attr:`nonstring` is ``strict`` and + a non-:class:`basestring` object is passed in or if :attr:`nonstring` + is set to an unknown value + :raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and + :attr:`obj` is not decodable using the given encoding + :returns: :class:`unicode` string or the original object depending on the + value of :attr:`nonstring`. + + Usually this should be used on a byte :class:`str` but it can take both + byte :class:`str` and :class:`unicode` strings intelligently. Nonstring + objects are handled in different ways depending on the setting of the + :attr:`nonstring` parameter. + + The default values of this function are set so as to always return + a :class:`unicode` string and never raise an error when converting from + a byte :class:`str` to a :class:`unicode` string. However, when you do + not pass validly encoded text (or a nonstring object), you may end up with + output that you don't expect. Be sure you understand the requirements of + your data, not just ignore errors by passing it through this function. + ''' + # Could use isbasestring/isunicode here but we want this code to be as + # fast as possible + if isinstance(obj, basestring): + if isinstance(obj, unicode): + return obj + if encoding in _UTF8_ALIASES: + return unicode(obj, 'utf-8', errors) + if encoding in _LATIN1_ALIASES: + return unicode(obj, 'latin-1', errors) + return obj.decode(encoding, errors) + + if not nonstring: + nonstring = 'simplerepr' + if nonstring == 'empty': + return u'' + elif nonstring == 'passthru': + return obj + elif nonstring == 'simplerepr': + try: + simple = obj.__unicode__() + except (AttributeError, UnicodeError): + simple = None + if not simple: + try: + simple = str(obj) + except UnicodeError: + try: + simple = obj.__str__() + except (UnicodeError, AttributeError): + simple = u'' + if isbytestring(simple): + return unicode(simple, encoding, errors) + return simple + elif nonstring in ('repr', 'strict'): + obj_repr = repr(obj) + if isbytestring(obj_repr): + obj_repr = unicode(obj_repr, encoding, errors) + if nonstring == 'repr': + return obj_repr + raise TypeError('to_unicode was given "%(obj)s" which is neither' + ' a byte string (str) or a unicode string' % + {'obj': obj_repr.encode(encoding, 'replace')}) + + raise TypeError('nonstring value, %(param)s, is not set to a valid' + ' action' % {'param': nonstring}) + +def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None): + '''Convert an object into a byte :class:`str` + + :arg obj: Object to convert to a byte :class:`str`. This should normally + be a :class:`unicode` string. + :kwarg encoding: Encoding to use to convert the :class:`unicode` string + into a byte :class:`str`. Defaults to :term:`utf-8`. + :kwarg errors: If errors are found while encoding, perform this action. + Defaults to ``replace`` which replaces the invalid bytes with + a character that means the bytes were unable to be encoded. Other + values are the same as the error handling schemes in the `codec base + classes + <http://docs.python.org/library/codecs.html#codec-base-classes>`_. + For instance ``strict`` which raises an exception and ``ignore`` which + simply omits the non-encodable characters. + :kwarg nonstring: How to treat nonstring values. Possible values are: + + :simplerepr: Attempt to call the object's "simple representation" + method and return that value. Python-2.3+ has two methods that + try to return a simple representation: :meth:`object.__unicode__` + and :meth:`object.__str__`. We first try to get a usable value + from :meth:`object.__str__`. If that fails we try the same + with :meth:`object.__unicode__`. + :empty: Return an empty byte :class:`str` + :strict: Raise a :exc:`TypeError` + :passthru: Return the object unchanged + :repr: Attempt to return a byte :class:`str` of the :func:`repr` of the + object + + Default is ``simplerepr``. + + :raises TypeError: if :attr:`nonstring` is ``strict`` and + a non-:class:`basestring` object is passed in or if :attr:`nonstring` + is set to an unknown value. + :raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the + bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`. + :returns: byte :class:`str` or the original object depending on the value + of :attr:`nonstring`. + + .. warning:: + + If you pass a byte :class:`str` into this function the byte + :class:`str` is returned unmodified. It is **not** re-encoded with + the specified :attr:`encoding`. The easiest way to achieve that is:: + + to_bytes(to_unicode(text), encoding='utf-8') + + The initial :func:`to_unicode` call will ensure text is + a :class:`unicode` string. Then, :func:`to_bytes` will turn that into + a byte :class:`str` with the specified encoding. + + Usually, this should be used on a :class:`unicode` string but it can take + either a byte :class:`str` or a :class:`unicode` string intelligently. + Nonstring objects are handled in different ways depending on the setting + of the :attr:`nonstring` parameter. + + The default values of this function are set so as to always return a byte + :class:`str` and never raise an error when converting from unicode to + bytes. However, when you do not pass an encoding that can validly encode + the object (or a non-string object), you may end up with output that you + don't expect. Be sure you understand the requirements of your data, not + just ignore errors by passing it through this function. + ''' + # Could use isbasestring, isbytestring here but we want this to be as fast + # as possible + if isinstance(obj, basestring): + if isinstance(obj, str): + return obj + return obj.encode(encoding, errors) + if not nonstring: + nonstring = 'simplerepr' + + if nonstring == 'empty': + return '' + elif nonstring == 'passthru': + return obj + elif nonstring == 'simplerepr': + try: + simple = str(obj) + except UnicodeError: + try: + simple = obj.__str__() + except (AttributeError, UnicodeError): + simple = None + if not simple: + try: + simple = obj.__unicode__() + except (AttributeError, UnicodeError): + simple = '' + if isunicodestring(simple): + simple = simple.encode(encoding, 'replace') + return simple + elif nonstring in ('repr', 'strict'): + try: + obj_repr = obj.__repr__() + except (AttributeError, UnicodeError): + obj_repr = '' + if isunicodestring(obj_repr): + obj_repr = obj_repr.encode(encoding, errors) + else: + obj_repr = str(obj_repr) + if nonstring == 'repr': + return obj_repr + raise TypeError('to_bytes was given "%(obj)s" which is neither' + ' a unicode string or a byte string (str)' % {'obj': obj_repr}) + + raise TypeError('nonstring value, %(param)s, is not set to a valid' + ' action' % {'param': nonstring}) + + +# force the return value of a function to be unicode. Use with partial to +# ensure that a filter will return unicode values. +def unicode_wrap(func, *args, **kwargs): + return to_unicode(func(*args, **kwargs), nonstring='passthru') diff --git a/v2/ansible/utils/vars.py b/v2/ansible/utils/vars.py new file mode 100644 index 0000000000..c033c0c258 --- /dev/null +++ b/v2/ansible/utils/vars.py @@ -0,0 +1,51 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +from ansible import constants as C + +def combine_vars(a, b): + + if C.DEFAULT_HASH_BEHAVIOUR == "merge": + return merge_hash(a, b) + else: + return dict(a.items() + b.items()) + +def merge_hash(a, b): + ''' recursively merges hash b into a + keys from b take precedence over keys from a ''' + + result = {} + + for dicts in a, b: + # next, iterate over b keys and values + for k, v in dicts.iteritems(): + # if there's already such key in a + # and that key contains dict + if k in result and isinstance(result[k], dict): + # merge those dicts recursively + result[k] = merge_hash(a[k], v) + else: + # otherwise, just copy a value from b to a + result[k] = v + + return result + diff --git a/v2/ansible/vars/__init__.py b/v2/ansible/vars/__init__.py index af81b12b2e..f9e7cba9cd 100644 --- a/v2/ansible/vars/__init__.py +++ b/v2/ansible/vars/__init__.py @@ -23,23 +23,47 @@ import os from collections import defaultdict -from ansible.parsing.yaml import DataLoader +try: + from hashlib import sha1 +except ImportError: + from sha import sha as sha1 + +from ansible import constants as C +from ansible.parsing import DataLoader from ansible.plugins.cache import FactCache +from ansible.template import Templar +from ansible.utils.debug import debug +from ansible.vars.hostvars import HostVars + +CACHED_VARS = dict() class VariableManager: - def __init__(self, inventory_path=None, loader=None): + def __init__(self): self._fact_cache = FactCache() self._vars_cache = defaultdict(dict) self._extra_vars = defaultdict(dict) self._host_vars_files = defaultdict(dict) self._group_vars_files = defaultdict(dict) + self._inventory = None - if not loader: - self._loader = DataLoader() - else: - self._loader = loader + self._omit_token = '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest() + + def _get_cache_entry(self, play=None, host=None, task=None): + play_id = "NONE" + if play: + play_id = play._uuid + + host_id = "NONE" + if host: + host_id = host.get_name() + + task_id = "NONE" + if task: + task_id = task._uuid + + return "PLAY:%s;HOST:%s;TASK:%s" % (play_id, host_id, task_id) @property def extra_vars(self): @@ -51,6 +75,23 @@ class VariableManager: assert isinstance(value, dict) self._extra_vars = value.copy() + def set_inventory(self, inventory): + self._inventory = inventory + + def _combine_vars(self, a, b): + ''' + Combines dictionaries of variables, based on the hash behavior + ''' + + # FIXME: do we need this from utils, or should it just + # be merged into this definition? + #_validate_both_dicts(a, b) + + if C.DEFAULT_HASH_BEHAVIOUR == "merge": + return self._merge_dicts(a, b) + else: + return dict(a.items() + b.items()) + def _merge_dicts(self, a, b): ''' Recursively merges dict b into a, so that keys @@ -77,7 +118,7 @@ class VariableManager: return result - def get_vars(self, play=None, host=None, task=None): + def get_vars(self, loader, play=None, host=None, task=None): ''' Returns the variables, with optional "context" given via the parameters for the play, host, and task (which could possibly result in different @@ -97,49 +138,88 @@ class VariableManager: - extra vars ''' - vars = defaultdict(dict) + debug("in VariableManager get_vars()") + cache_entry = self._get_cache_entry(play=play, host=host, task=task) + if cache_entry in CACHED_VARS: + debug("vars are cached, returning them now") + return CACHED_VARS[cache_entry] + + all_vars = defaultdict(dict) if play: # first we compile any vars specified in defaults/main.yml # for all roles within the specified play for role in play.get_roles(): - vars = self._merge_dicts(vars, role.get_default_vars()) + all_vars = self._combine_vars(all_vars, role.get_default_vars()) if host: # next, if a host is specified, we load any vars from group_vars # files and then any vars from host_vars files which may apply to # this host or the groups it belongs to + + # we merge in the special 'all' group_vars first, if they exist + if 'all' in self._group_vars_files: + all_vars = self._combine_vars(all_vars, self._group_vars_files['all']) + for group in host.get_groups(): - if group in self._group_vars_files: - vars = self._merge_dicts(vars, self._group_vars_files[group]) + group_name = group.get_name() + all_vars = self._combine_vars(all_vars, group.get_vars()) + if group_name in self._group_vars_files and group_name != 'all': + all_vars = self._combine_vars(all_vars, self._group_vars_files[group_name]) host_name = host.get_name() if host_name in self._host_vars_files: - vars = self._merge_dicts(vars, self._host_vars_files[host_name]) + all_vars = self._combine_vars(all_vars, self._host_vars_files[host_name]) # then we merge in vars specified for this host - vars = self._merge_dicts(vars, host.get_vars()) + all_vars = self._combine_vars(all_vars, host.get_vars()) # next comes the facts cache and the vars cache, respectively - vars = self._merge_dicts(vars, self._fact_cache.get(host.get_name(), dict())) - vars = self._merge_dicts(vars, self._vars_cache.get(host.get_name(), dict())) + all_vars = self._combine_vars(all_vars, self._fact_cache.get(host.get_name(), dict())) if play: - vars = self._merge_dicts(vars, play.get_vars()) + all_vars = self._combine_vars(all_vars, play.get_vars()) + templar = Templar(loader=loader, variables=all_vars) for vars_file in play.get_vars_files(): - # Try templating the vars_file. If an unknown var error is raised, - # ignore it - unless a host is specified - # TODO ... + try: + vars_file = templar.template(vars_file) + data = loader.load_from_file(vars_file) + all_vars = self._combine_vars(all_vars, data) + except: + # FIXME: get_vars should probably be taking a flag to determine + # whether or not vars files errors should be fatal at this + # stage, or just base it on whether a host was specified? + pass + for role in play.get_roles(): + all_vars = self._combine_vars(all_vars, role.get_vars()) - data = self._loader.load_from_file(vars_file) - vars = self._merge_dicts(vars, data) + if host: + all_vars = self._combine_vars(all_vars, self._vars_cache.get(host.get_name(), dict())) if task: - vars = self._merge_dicts(vars, task.get_vars()) + if task._role: + all_vars = self._combine_vars(all_vars, task._role.get_vars()) + all_vars = self._combine_vars(all_vars, task.get_vars()) + + all_vars = self._combine_vars(all_vars, self._extra_vars) + + # FIXME: make sure all special vars are here + # Finally, we create special vars + + if host and self._inventory is not None: + hostvars = HostVars(vars_manager=self, inventory=self._inventory, loader=loader) + all_vars['hostvars'] = hostvars + + if self._inventory is not None: + all_vars['inventory_dir'] = self._inventory.basedir() + + # the 'omit' value alows params to be left out if the variable they are based on is undefined + all_vars['omit'] = self._omit_token - vars = self._merge_dicts(vars, self._extra_vars) + CACHED_VARS[cache_entry] = all_vars - return vars + debug("done with get_vars()") + return all_vars def _get_inventory_basename(self, path): ''' @@ -148,35 +228,83 @@ class VariableManager: ''' (name, ext) = os.path.splitext(os.path.basename(path)) - return name + if ext not in ('yml', 'yaml'): + return os.path.basename(path) + else: + return name - def _load_inventory_file(self, path): + def _load_inventory_file(self, path, loader): ''' helper function, which loads the file and gets the basename of the file without the extension ''' - data = self._loader.load_from_file(path) + if os.path.isdir(path): + data = dict() + + try: + names = os.listdir(path) + except os.error, err: + raise AnsibleError("This folder cannot be listed: %s: %s." % (path, err.strerror)) + + # evaluate files in a stable order rather than whatever + # order the filesystem lists them. + names.sort() + + # do not parse hidden files or dirs, e.g. .svn/ + paths = [os.path.join(path, name) for name in names if not name.startswith('.')] + for p in paths: + _found, results = self._load_inventory_file(path=p, loader=loader) + data = self._combine_vars(data, results) + + else: + data = loader.load_from_file(path) + name = self._get_inventory_basename(path) return (name, data) - def add_host_vars_file(self, path): + def add_host_vars_file(self, path, loader): ''' Loads and caches a host_vars file in the _host_vars_files dict, where the key to that dictionary is the basename of the file, minus the extension, for matching against a given inventory host name ''' - (name, data) = self._load_inventory_file(path) - self._host_vars_files[name] = data + if os.path.exists(path): + (name, data) = self._load_inventory_file(path, loader) + self._host_vars_files[name] = data - def add_group_vars_file(self, path): + def add_group_vars_file(self, path, loader): ''' Loads and caches a host_vars file in the _host_vars_files dict, where the key to that dictionary is the basename of the file, minus the extension, for matching against a given inventory host name ''' - (name, data) = self._load_inventory_file(path) - self._group_vars_files[name] = data + if os.path.exists(path): + (name, data) = self._load_inventory_file(path, loader) + self._group_vars_files[name] = data + + def set_host_facts(self, host, facts): + ''' + Sets or updates the given facts for a host in the fact cache. + ''' + + assert isinstance(facts, dict) + + host_name = host.get_name() + if host_name not in self._fact_cache: + self._fact_cache[host_name] = facts + else: + self._fact_cache[host_name].update(facts) + + def set_host_variable(self, host, varname, value): + ''' + Sets a value in the vars_cache for a host. + ''' + + host_name = host.get_name() + if host_name not in self._vars_cache: + self._vars_cache[host_name] = dict() + self._vars_cache[host_name][varname] = value diff --git a/v2/ansible/vars/hostvars.py b/v2/ansible/vars/hostvars.py new file mode 100644 index 0000000000..45b3340229 --- /dev/null +++ b/v2/ansible/vars/hostvars.py @@ -0,0 +1,47 @@ +# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.template import Templar + +__all__ = ['HostVars'] + +class HostVars(dict): + ''' A special view of vars_cache that adds values from the inventory when needed. ''' + + def __init__(self, vars_manager, inventory, loader): + self._vars_manager = vars_manager + self._inventory = inventory + self._loader = loader + self._lookup = {} + + #self.update(vars_cache) + + def __getitem__(self, host_name): + + if host_name not in self._lookup: + host = self._inventory.get_host(host_name) + result = self._vars_manager.get_vars(loader=self._loader, host=host) + #result.update(self._vars_cache.get(host, {})) + #templar = Templar(variables=self._vars_cache, loader=self._loader) + #self._lookup[host] = templar.template(result) + self._lookup[host_name] = result + return self._lookup[host_name] + diff --git a/v2/bin/ansible b/v2/bin/ansible new file mode 100755 index 0000000000..c51040c6a8 --- /dev/null +++ b/v2/bin/ansible @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +######################################################## + +import os +import sys + +from ansible import constants as C +from ansible.errors import * +from ansible.executor.task_queue_manager import TaskQueueManager +from ansible.inventory import Inventory +from ansible.parsing import DataLoader +from ansible.parsing.splitter import parse_kv +from ansible.playbook.play import Play +from ansible.utils.cli import base_parser +from ansible.vars import VariableManager + +######################################################## + +class Cli(object): + ''' code behind bin/ansible ''' + + def __init__(self): + pass + + def parse(self): + ''' create an options parser for bin/ansible ''' + + parser = base_parser( + usage='%prog <host-pattern> [options]', + runas_opts=True, + subset_opts=True, + async_opts=True, + output_opts=True, + connect_opts=True, + check_opts=True, + diff_opts=False, + ) + + parser.add_option('-a', '--args', dest='module_args', + help="module arguments", default=C.DEFAULT_MODULE_ARGS) + parser.add_option('-m', '--module-name', dest='module_name', + help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME, + default=C.DEFAULT_MODULE_NAME) + + options, args = parser.parse_args() + + if len(args) == 0 or len(args) > 1: + parser.print_help() + sys.exit(1) + + # su and sudo command line arguments need to be mutually exclusive + if (options.su or options.su_user or options.ask_su_pass) and \ + (options.sudo or options.sudo_user or options.ask_sudo_pass): + parser.error("Sudo arguments ('--sudo', '--sudo-user', and '--ask-sudo-pass') " + "and su arguments ('-su', '--su-user', and '--ask-su-pass') are " + "mutually exclusive") + + if (options.ask_vault_pass and options.vault_password_file): + parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive") + + return (options, args) + + # ---------------------------------------------- + + def run(self, options, args): + ''' use Runner lib to do SSH things ''' + + pattern = args[0] + + #------------------------------------------------------------------------------- + # FIXME: the password asking stuff needs to be ported over still + #------------------------------------------------------------------------------- + #sshpass = None + #sudopass = None + #su_pass = None + #vault_pass = None + # + #options.ask_pass = options.ask_pass or C.DEFAULT_ASK_PASS + ## Never ask for an SSH password when we run with local connection + #if options.connection == "local": + # options.ask_pass = False + #options.ask_sudo_pass = options.ask_sudo_pass or C.DEFAULT_ASK_SUDO_PASS + #options.ask_su_pass = options.ask_su_pass or C.DEFAULT_ASK_SU_PASS + #options.ask_vault_pass = options.ask_vault_pass or C.DEFAULT_ASK_VAULT_PASS + # + #(sshpass, sudopass, su_pass, vault_pass) = utils.ask_passwords(ask_pass=options.ask_pass, ask_sudo_pass=options.ask_sudo_pass, ask_su_pass=options.ask_su_pass, ask_vault_pass=options.ask_vault_pass) + # + # read vault_pass from a file + #if not options.ask_vault_pass and options.vault_password_file: + # vault_pass = utils.read_vault_file(options.vault_password_file) + #------------------------------------------------------------------------------- + + # FIXME: needs vault password, after the above is fixed + loader = DataLoader() + variable_manager = VariableManager() + + inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) + if options.subset: + inventory.subset(options.subset) + + hosts = inventory.list_hosts(pattern) + if len(hosts) == 0: + raise AnsibleError("provided hosts list is empty") + + if options.listhosts: + for host in hosts: + print(' %s' % host.name) + sys.exit(0) + + if ((options.module_name == 'command' or options.module_name == 'shell') and not options.module_args): + raise AnsibleError("No argument passed to %s module" % options.module_name) + + # FIXME: async support needed + #if options.seconds: + # callbacks.display("background launch...\n\n", color='cyan') + # results, poller = runner.run_async(options.seconds) + # results = self.poll_while_needed(poller, options) + #else: + # results = runner.run() + + # create a pseudo-play to execute the specified module via a single task + play_ds = dict( + hosts = pattern, + gather_facts = 'no', + tasks = [ + dict(action=dict(module=options.module_name, args=parse_kv(options.module_args))), + ] + ) + + play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) + + # now create a task queue manager to execute the play + try: + tqm = TaskQueueManager(inventory=inventory, callback='minimal', variable_manager=variable_manager, loader=loader, options=options) + result = tqm.run(play) + tqm.cleanup() + except AnsibleError: + tqm.cleanup() + raise + + return result + + # ---------------------------------------------- + + def poll_while_needed(self, poller, options): + ''' summarize results from Runner ''' + + # BACKGROUND POLL LOGIC when -B and -P are specified + if options.seconds and options.poll_interval > 0: + poller.wait(options.seconds, options.poll_interval) + + return poller.results + + +######################################################## + +if __name__ == '__main__': + #callbacks.display("", log_only=True) + #callbacks.display(" ".join(sys.argv), log_only=True) + #callbacks.display("", log_only=True) + + try: + cli = Cli() + (options, args) = cli.parse() + result = cli.run(options, args) + + except AnsibleError, e: + print(e) + sys.exit(1) + + except Exception, e: + # Generic handler for errors + print("ERROR: %s" % str(e)) + sys.exit(1) + + sys.exit(result) diff --git a/v2/bin/ansible-playbook b/v2/bin/ansible-playbook new file mode 100755 index 0000000000..eafccedcba --- /dev/null +++ b/v2/bin/ansible-playbook @@ -0,0 +1,180 @@ +#!/usr/bin/env python + +import os +import stat +import sys + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.executor.playbook_executor import PlaybookExecutor +from ansible.inventory import Inventory +from ansible.parsing import DataLoader +from ansible.parsing.splitter import parse_kv +from ansible.playbook import Playbook +from ansible.playbook.task import Task +from ansible.utils.cli import base_parser +from ansible.utils.unicode import to_unicode +from ansible.utils.vars import combine_vars +from ansible.vars import VariableManager + +# Implement an ansible.utils.warning() function later +warning = getattr(__builtins__, 'print') + +#--------------------------------------------------------------------------------------------------- + +def main(args): + ''' run ansible-playbook operations ''' + + # create parser for CLI options + parser = base_parser( + usage = "%prog playbook.yml", + connect_opts=True, + runas_opts=True, + subset_opts=True, + check_opts=True, + diff_opts=True + ) + #parser.add_option('--vault-password', dest="vault_password", + # help="password for vault encrypted files") + parser.add_option('-e', '--extra-vars', dest="extra_vars", action="append", + help="set additional variables as key=value or YAML/JSON", default=[]) + parser.add_option('-t', '--tags', dest='tags', default='all', + help="only run plays and tasks tagged with these values") + parser.add_option('--skip-tags', dest='skip_tags', + help="only run plays and tasks whose tags do not match these values") + parser.add_option('--syntax-check', dest='syntax', action='store_true', + help="perform a syntax check on the playbook, but do not execute it") + parser.add_option('--list-tasks', dest='listtasks', action='store_true', + help="list all tasks that would be executed") + parser.add_option('--step', dest='step', action='store_true', + help="one-step-at-a-time: confirm each task before running") + parser.add_option('--start-at-task', dest='start_at', + help="start the playbook at the task matching this name") + parser.add_option('--force-handlers', dest='force_handlers', action='store_true', + help="run handlers even if a task fails") + parser.add_option('--flush-cache', dest='flush_cache', action='store_true', + help="clear the fact cache") + + options, args = parser.parse_args(args) + + if len(args) == 0: + parser.print_help(file=sys.stderr) + return 1 + + #--------------------------------------------------------------------------------------------------- + # FIXME: su/sudo stuff needs to be generalized + # su and sudo command line arguments need to be mutually exclusive + #if (options.su or options.su_user or options.ask_su_pass) and \ + # (options.sudo or options.sudo_user or options.ask_sudo_pass): + # parser.error("Sudo arguments ('--sudo', '--sudo-user', and '--ask-sudo-pass') " + # "and su arguments ('-su', '--su-user', and '--ask-su-pass') are " + # "mutually exclusive") + # + #if (options.ask_vault_pass and options.vault_password_file): + # parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive") + # + #sshpass = None + #sudopass = None + #su_pass = None + #vault_pass = None + # + #options.ask_vault_pass = options.ask_vault_pass or C.DEFAULT_ASK_VAULT_PASS + # + #if options.listhosts or options.syntax or options.listtasks: + # (_, _, _, vault_pass) = utils.ask_passwords(ask_vault_pass=options.ask_vault_pass) + #else: + # options.ask_pass = options.ask_pass or C.DEFAULT_ASK_PASS + # # Never ask for an SSH password when we run with local connection + # if options.connection == "local": + # options.ask_pass = False + # options.ask_sudo_pass = options.ask_sudo_pass or C.DEFAULT_ASK_SUDO_PASS + # options.ask_su_pass = options.ask_su_pass or C.DEFAULT_ASK_SU_PASS + # (sshpass, sudopass, su_pass, vault_pass) = utils.ask_passwords(ask_pass=options.ask_pass, ask_sudo_pass=options.ask_sudo_pass, ask_su_pass=options.ask_su_pass, ask_vault_pass=options.ask_vault_pass) + # options.sudo_user = options.sudo_user or C.DEFAULT_SUDO_USER + # options.su_user = options.su_user or C.DEFAULT_SU_USER + # + ## read vault_pass from a file + #if not options.ask_vault_pass and options.vault_password_file: + # vault_pass = utils.read_vault_file(options.vault_password_file) + # END FIXME + #--------------------------------------------------------------------------------------------------- + + # FIXME: this hard-coded value will be removed after fixing the removed block + # above, which dealt wtih asking for passwords during runtime + vault_pass = 'testing' + loader = DataLoader(vault_password=vault_pass) + + extra_vars = {} + for extra_vars_opt in options.extra_vars: + extra_vars_opt = to_unicode(extra_vars_opt, errors='strict') + if extra_vars_opt.startswith(u"@"): + # Argument is a YAML file (JSON is a subset of YAML) + data = loader.load_from_file(extra_vars_opt[1:]) + elif extra_vars_opt and extra_vars_opt[0] in u'[{': + # Arguments as YAML + data = loader.load(extra_vars_opt) + else: + # Arguments as Key-value + data = parse_kv(extra_vars_opt) + extra_vars = combine_vars(extra_vars, data) + + # FIXME: this should be moved inside the playbook executor code + only_tags = options.tags.split(",") + skip_tags = options.skip_tags + if options.skip_tags is not None: + skip_tags = options.skip_tags.split(",") + + # initial error check, to make sure all specified playbooks are accessible + # before we start running anything through the playbook executor + for playbook in args: + if not os.path.exists(playbook): + raise AnsibleError("the playbook: %s could not be found" % playbook) + if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): + raise AnsibleError("the playbook: %s does not appear to be a file" % playbook) + + # create the variable manager, which will be shared throughout + # the code, ensuring a consistent view of global variables + variable_manager = VariableManager() + variable_manager.set_extra_vars(extra_vars) + + # create the inventory, and filter it based on the subset specified (if any) + inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) + variable_manager.set_inventory(inventory) + + # Note: slightly wrong, this is written so that implicit localhost + # (which is not returned in list_hosts()) is taken into account for + # warning if inventory is empty. But it can't be taken into account for + # checking if limit doesn't match any hosts. Instead we don't worry about + # limit if only implicit localhost was in inventory to start with. + # + # Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts()) + no_hosts = False + if len(inventory.list_hosts()) == 0: + # Empty inventory + warning("provided hosts list is empty, only localhost is available") + no_hosts = True + inventory.subset(options.subset) + if len(inventory.list_hosts()) == 0 and no_hosts is False: + # Invalid limit + raise errors.AnsibleError("Specified --limit does not match any hosts") + + # create the playbook executor, which manages running the plays + # via a task queue manager + pbex = PlaybookExecutor(playbooks=args, inventory=inventory, variable_manager=variable_manager, loader=loader, options=options) + return pbex.run() + +if __name__ == "__main__": + #display(" ", log_only=True) + #display(" ".join(sys.argv), log_only=True) + #display(" ", log_only=True) + try: + sys.exit(main(sys.argv[1:])) + except AnsibleError, e: + #display("ERROR: %s" % e, color='red', stderr=True) + print e + sys.exit(1) + except KeyboardInterrupt, ke: + #display("ERROR: interrupted", color='red', stderr=True) + print "keyboard interrupt" + sys.exit(1) + diff --git a/v2/hacking/README.md b/v2/hacking/README.md new file mode 100644 index 0000000000..6d65464eee --- /dev/null +++ b/v2/hacking/README.md @@ -0,0 +1,48 @@ +'Hacking' directory tools +========================= + +Env-setup +--------- + +The 'env-setup' script modifies your environment to allow you to run +ansible from a git checkout using python 2.6+. (You may not use +python 3 at this time). + +First, set up your environment to run from the checkout: + + $ source ./hacking/env-setup + +You will need some basic prerequisites installed. If you do not already have them +and do not wish to install them from your operating system package manager, you +can install them from pip + + $ easy_install pip # if pip is not already available + $ pip install pyyaml jinja2 nose passlib pycrypto + +From there, follow ansible instructions on docs.ansible.com as normal. + +Test-module +----------- + +'test-module' is a simple program that allows module developers (or testers) to run +a module outside of the ansible program, locally, on the current machine. + +Example: + + $ ./hacking/test-module -m library/commands/shell -a "echo hi" + +This is a good way to insert a breakpoint into a module, for instance. + +Module-formatter +---------------- + +The module formatter is a script used to generate manpages and online +module documentation. This is used by the system makefiles and rarely +needs to be run directly. + +Authors +------- +'authors' is a simple script that generates a list of everyone who has +contributed code to the ansible repository. + + diff --git a/v2/hacking/authors.sh b/v2/hacking/authors.sh new file mode 100755 index 0000000000..7c97840b2f --- /dev/null +++ b/v2/hacking/authors.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# script from http://stackoverflow.com/questions/12133583 +set -e + +# Get a list of authors ordered by number of commits +# and remove the commit count column +AUTHORS=$(git --no-pager shortlog -nse | cut -f 2- | sort -f) +if [ -z "$AUTHORS" ] ; then + echo "Authors list was empty" + exit 1 +fi + +# Display the authors list and write it to the file +echo "$AUTHORS" | tee "$(git rev-parse --show-toplevel)/AUTHORS.TXT" diff --git a/v2/hacking/env-setup b/v2/hacking/env-setup new file mode 100644 index 0000000000..c573a56878 --- /dev/null +++ b/v2/hacking/env-setup @@ -0,0 +1,76 @@ +# usage: source hacking/env-setup [-q] +# modifies environment for running Ansible from checkout + +# Default values for shell variables we use +PYTHONPATH=${PYTHONPATH-""} +PATH=${PATH-""} +MANPATH=${MANPATH-""} +verbosity=${1-info} # Defaults to `info' if unspecified + +if [ "$verbosity" = -q ]; then + verbosity=silent +fi + +# When run using source as directed, $0 gets set to bash, so we must use $BASH_SOURCE +if [ -n "$BASH_SOURCE" ] ; then + HACKING_DIR=$(dirname "$BASH_SOURCE") +elif [ $(basename "$0") = "env-setup" ]; then + HACKING_DIR=$(dirname "$0") +else + HACKING_DIR="$PWD/hacking" +fi +# The below is an alternative to readlink -fn which doesn't exist on OS X +# Source: http://stackoverflow.com/a/1678636 +FULL_PATH=$(python -c "import os; print(os.path.realpath('$HACKING_DIR'))") +ANSIBLE_HOME=$(dirname "$FULL_PATH") + +PREFIX_PYTHONPATH="$ANSIBLE_HOME" +PREFIX_PATH="$ANSIBLE_HOME/bin" +PREFIX_MANPATH="$ANSIBLE_HOME/docs/man" + +expr "$PYTHONPATH" : "${PREFIX_PYTHONPATH}.*" > /dev/null || export PYTHONPATH="$PREFIX_PYTHONPATH:$PYTHONPATH" +expr "$PATH" : "${PREFIX_PATH}.*" > /dev/null || export PATH="$PREFIX_PATH:$PATH" +expr "$MANPATH" : "${PREFIX_MANPATH}.*" > /dev/null || export MANPATH="$PREFIX_MANPATH:$MANPATH" + +# +# Generate egg_info so that pkg_resources works +# + +# Do the work in a function so we don't repeat ourselves later +gen_egg_info() +{ + python setup.py egg_info + if [ -e "$PREFIX_PYTHONPATH/ansible.egg-info" ] ; then + rm -r "$PREFIX_PYTHONPATH/ansible.egg-info" + fi + mv "ansible.egg-info" "$PREFIX_PYTHONPATH" +} + +if [ "$ANSIBLE_HOME" != "$PWD" ] ; then + current_dir="$PWD" +else + current_dir="$ANSIBLE_HOME" +fi +cd "$ANSIBLE_HOME" +#if [ "$verbosity" = silent ] ; then +# gen_egg_info > /dev/null 2>&1 +#else +# gen_egg_info +#fi +cd "$current_dir" + +if [ "$verbosity" != silent ] ; then + cat <<- EOF + + Setting up Ansible to run out of checkout... + + PATH=$PATH + PYTHONPATH=$PYTHONPATH + MANPATH=$MANPATH + + Remember, you may wish to specify your host file with -i + + Done! + + EOF +fi diff --git a/v2/hacking/env-setup.fish b/v2/hacking/env-setup.fish new file mode 100644 index 0000000000..05fb60672d --- /dev/null +++ b/v2/hacking/env-setup.fish @@ -0,0 +1,57 @@ +#!/usr/bin/env fish +# usage: . ./hacking/env-setup [-q] +# modifies environment for running Ansible from checkout +set HACKING_DIR (dirname (status -f)) +set FULL_PATH (python -c "import os; print(os.path.realpath('$HACKING_DIR'))") +set ANSIBLE_HOME (dirname $FULL_PATH) +set PREFIX_PYTHONPATH $ANSIBLE_HOME/lib +set PREFIX_PATH $ANSIBLE_HOME/bin +set PREFIX_MANPATH $ANSIBLE_HOME/docs/man + +# Set PYTHONPATH +if not set -q PYTHONPATH + set -gx PYTHONPATH $PREFIX_PYTHONPATH +else + switch PYTHONPATH + case "$PREFIX_PYTHONPATH*" + case "*" + echo "Appending PYTHONPATH" + set -gx PYTHONPATH "$PREFIX_PYTHONPATH:$PYTHONPATH" + end +end + +# Set PATH +if not contains $PREFIX_PATH $PATH + set -gx PATH $PREFIX_PATH $PATH +end + +# Set MANPATH +if not contains $PREFIX_MANPATH $MANPATH + if not set -q MANPATH + set -gx MANPATH $PREFIX_MANPATH + else + set -gx MANPATH $PREFIX_MANPATH $MANPATH + end +end + +set -gx ANSIBLE_LIBRARY $ANSIBLE_HOME/library + +if set -q argv + switch $argv + case '-q' '--quiet' + case '*' + echo "" + echo "Setting up Ansible to run out of checkout..." + echo "" + echo "PATH=$PATH" + echo "PYTHONPATH=$PYTHONPATH" + echo "ANSIBLE_LIBRARY=$ANSIBLE_LIBRARY" + echo "MANPATH=$MANPATH" + echo "" + + echo "Remember, you may wish to specify your host file with -i" + echo "" + echo "Done!" + echo "" + end +end diff --git a/v2/hacking/get_library.py b/v2/hacking/get_library.py new file mode 100755 index 0000000000..571183b688 --- /dev/null +++ b/v2/hacking/get_library.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +# (c) 2014, Will Thames <will@thames.id.au> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# + +import ansible.constants as C +import sys + +def main(): + print C.DEFAULT_MODULE_PATH + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/v2/hacking/module_formatter.py b/v2/hacking/module_formatter.py new file mode 100755 index 0000000000..04f098fc98 --- /dev/null +++ b/v2/hacking/module_formatter.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python +# (c) 2012, Jan-Piet Mens <jpmens () gmail.com> +# (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# + +import os +import glob +import sys +import yaml +import codecs +import json +import ast +import re +import optparse +import time +import datetime +import subprocess +import cgi +from jinja2 import Environment, FileSystemLoader + +import ansible.utils +import ansible.utils.module_docs as module_docs + +##################################################################################### +# constants and paths + +# if a module is added in a version of Ansible older than this, don't print the version added information +# in the module documentation because everyone is assumed to be running something newer than this already. +TO_OLD_TO_BE_NOTABLE = 1.0 + +# Get parent directory of the directory this script lives in +MODULEDIR=os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, 'lib', 'ansible', 'modules' +)) + +# The name of the DOCUMENTATION template +EXAMPLE_YAML=os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml' +)) + +_ITALIC = re.compile(r"I\(([^)]+)\)") +_BOLD = re.compile(r"B\(([^)]+)\)") +_MODULE = re.compile(r"M\(([^)]+)\)") +_URL = re.compile(r"U\(([^)]+)\)") +_CONST = re.compile(r"C\(([^)]+)\)") + +DEPRECATED = " (D)" +NOTCORE = " (E)" +##################################################################################### + +def rst_ify(text): + ''' convert symbols like I(this is in italics) to valid restructured text ''' + + t = _ITALIC.sub(r'*' + r"\1" + r"*", text) + t = _BOLD.sub(r'**' + r"\1" + r"**", t) + t = _MODULE.sub(r'``' + r"\1" + r"``", t) + t = _URL.sub(r"\1", t) + t = _CONST.sub(r'``' + r"\1" + r"``", t) + + return t + +##################################################################################### + +def html_ify(text): + ''' convert symbols like I(this is in italics) to valid HTML ''' + + t = cgi.escape(text) + t = _ITALIC.sub("<em>" + r"\1" + "</em>", t) + t = _BOLD.sub("<b>" + r"\1" + "</b>", t) + t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t) + t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t) + t = _CONST.sub("<code>" + r"\1" + "</code>", t) + + return t + + +##################################################################################### + +def rst_fmt(text, fmt): + ''' helper for Jinja2 to do format strings ''' + + return fmt % (text) + +##################################################################################### + +def rst_xline(width, char="="): + ''' return a restructured text line of a given length ''' + + return char * width + +##################################################################################### + +def write_data(text, options, outputname, module): + ''' dumps module output to a file or the screen, as requested ''' + + if options.output_dir is not None: + fname = os.path.join(options.output_dir, outputname % module) + fname = fname.replace(".py","") + f = open(fname, 'w') + f.write(text.encode('utf-8')) + f.close() + else: + print text + +##################################################################################### + + +def list_modules(module_dir, depth=0): + ''' returns a hash of categories, each category being a hash of module names to file paths ''' + + categories = dict(all=dict(),_aliases=dict()) + if depth <= 3: # limit # of subdirs + + files = glob.glob("%s/*" % module_dir) + for d in files: + + category = os.path.splitext(os.path.basename(d))[0] + if os.path.isdir(d): + + res = list_modules(d, depth + 1) + for key in res.keys(): + if key in categories: + categories[key] = ansible.utils.merge_hash(categories[key], res[key]) + res.pop(key, None) + + if depth < 2: + categories.update(res) + else: + category = module_dir.split("/")[-1] + if not category in categories: + categories[category] = res + else: + categories[category].update(res) + else: + module = category + category = os.path.basename(module_dir) + if not d.endswith(".py") or d.endswith('__init__.py'): + # windows powershell modules have documentation stubs in python docstring + # format (they are not executed) so skip the ps1 format files + continue + elif module.startswith("_") and os.path.islink(d): + source = os.path.splitext(os.path.basename(os.path.realpath(d)))[0] + module = module.replace("_","",1) + if not d in categories['_aliases']: + categories['_aliases'][source] = [module] + else: + categories['_aliases'][source].update(module) + continue + + if not category in categories: + categories[category] = {} + categories[category][module] = d + categories['all'][module] = d + + return categories + +##################################################################################### + +def generate_parser(): + ''' generate an optparse parser ''' + + p = optparse.OptionParser( + version='%prog 1.0', + usage='usage: %prog [options] arg1 arg2', + description='Generate module documentation from metadata', + ) + + p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number") + p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path") + p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates") + p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type") + p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose") + p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files") + p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules") + p.add_option('-V', action='version', help='Show version number and exit') + return p + +##################################################################################### + +def jinja2_environment(template_dir, typ): + + env = Environment(loader=FileSystemLoader(template_dir), + variable_start_string="@{", + variable_end_string="}@", + trim_blocks=True, + ) + env.globals['xline'] = rst_xline + + if typ == 'rst': + env.filters['convert_symbols_to_format'] = rst_ify + env.filters['html_ify'] = html_ify + env.filters['fmt'] = rst_fmt + env.filters['xline'] = rst_xline + template = env.get_template('rst.j2') + outputname = "%s_module.rst" + else: + raise Exception("unknown module format type: %s" % typ) + + return env, template, outputname + +##################################################################################### + +def process_module(module, options, env, template, outputname, module_map, aliases): + + fname = module_map[module] + if isinstance(fname, dict): + return "SKIPPED" + + basename = os.path.basename(fname) + deprecated = False + + # ignore files with extensions + if not basename.endswith(".py"): + return + elif module.startswith("_"): + if os.path.islink(fname): + return # ignore, its an alias + deprecated = True + module = module.replace("_","",1) + + print "rendering: %s" % module + + # use ansible core library to parse out doc metadata YAML and plaintext examples + doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose) + + # crash if module is missing documentation and not explicitly hidden from docs index + if doc is None: + if module in ansible.utils.module_docs.BLACKLIST_MODULES: + return "SKIPPED" + else: + sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.exit(1) + + if deprecated and 'deprecated' not in doc: + sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.exit(1) + + if "/core/" in fname: + doc['core'] = True + else: + doc['core'] = False + + if module in aliases: + doc['aliases'] = aliases[module] + + all_keys = [] + + if not 'version_added' in doc: + sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module) + sys.exit(1) + + added = 0 + if doc['version_added'] == 'historical': + del doc['version_added'] + else: + added = doc['version_added'] + + # don't show version added information if it's too old to be called out + if added: + added_tokens = str(added).split(".") + added = added_tokens[0] + "." + added_tokens[1] + added_float = float(added) + if added and added_float < TO_OLD_TO_BE_NOTABLE: + del doc['version_added'] + + for (k,v) in doc['options'].iteritems(): + all_keys.append(k) + + all_keys = sorted(all_keys) + + doc['option_keys'] = all_keys + doc['filename'] = fname + doc['docuri'] = doc['module'].replace('_', '-') + doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') + doc['ansible_version'] = options.ansible_version + doc['plainexamples'] = examples #plain text + + # here is where we build the table of contents... + + text = template.render(doc) + write_data(text, options, outputname, module) + return doc['short_description'] + +##################################################################################### + +def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases): + modstring = module + modname = module + if module in deprecated: + modstring = modstring + DEPRECATED + modname = "_" + module + elif module not in core: + modstring = modstring + NOTCORE + + result = process_module(modname, options, env, template, outputname, module_map, aliases) + + if result != "SKIPPED": + category_file.write(" %s - %s <%s_module>\n" % (modstring, result, module)) + +def process_category(category, categories, options, env, template, outputname): + + module_map = categories[category] + + aliases = {} + if '_aliases' in categories: + aliases = categories['_aliases'] + + category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category) + category_file = open(category_file_path, "w") + print "*** recording category %s in %s ***" % (category, category_file_path) + + # TODO: start a new category file + + category = category.replace("_"," ") + category = category.title() + + modules = [] + deprecated = [] + core = [] + for module in module_map.keys(): + + if isinstance(module_map[module], dict): + for mod in module_map[module].keys(): + if mod.startswith("_"): + mod = mod.replace("_","",1) + deprecated.append(mod) + elif '/core/' in module_map[module][mod]: + core.append(mod) + else: + if module.startswith("_"): + module = module.replace("_","",1) + deprecated.append(module) + elif '/core/' in module_map[module]: + core.append(module) + + modules.append(module) + + modules.sort() + + category_header = "%s Modules" % (category.title()) + underscores = "`" * len(category_header) + + category_file.write("""\ +%s +%s + +.. toctree:: :maxdepth: 1 + +""" % (category_header, underscores)) + sections = [] + for module in modules: + if module in module_map and isinstance(module_map[module], dict): + sections.append(module) + continue + else: + print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases) + + sections.sort() + for section in sections: + category_file.write("\n%s\n%s\n\n" % (section.replace("_"," ").title(),'-' * len(section))) + category_file.write(".. toctree:: :maxdepth: 1\n\n") + + section_modules = module_map[section].keys() + section_modules.sort() + #for module in module_map[section]: + for module in section_modules: + print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases) + + category_file.write("""\n\n +.. note:: + - %s: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged. The module documentation details page may explain more about this rationale. + - %s: This marks a module as 'extras', which means it ships with ansible but may be a newer module and possibly (but not neccessarily) less activity maintained than 'core' modules. + - Tickets filed on modules are filed to different repos than those on the main open source project. Core module tickets should be filed at `ansible/ansible-modules-core on GitHub <http://github.com/ansible/ansible-modules-core>`_, extras tickets to `ansible/ansible-modules-extras on GitHub <http://github.com/ansible/ansible-modules-extras>`_ +""" % (DEPRECATED, NOTCORE)) + category_file.close() + + # TODO: end a new category file + +##################################################################################### + +def validate_options(options): + ''' validate option parser options ''' + + if not options.module_dir: + print >>sys.stderr, "--module-dir is required" + sys.exit(1) + if not os.path.exists(options.module_dir): + print >>sys.stderr, "--module-dir does not exist: %s" % options.module_dir + sys.exit(1) + if not options.template_dir: + print "--template-dir must be specified" + sys.exit(1) + +##################################################################################### + +def main(): + + p = generate_parser() + + (options, args) = p.parse_args() + validate_options(options) + + env, template, outputname = jinja2_environment(options.template_dir, options.type) + + categories = list_modules(options.module_dir) + last_category = None + category_names = categories.keys() + category_names.sort() + + category_list_path = os.path.join(options.output_dir, "modules_by_category.rst") + category_list_file = open(category_list_path, "w") + category_list_file.write("Module Index\n") + category_list_file.write("============\n") + category_list_file.write("\n\n") + category_list_file.write(".. toctree::\n") + category_list_file.write(" :maxdepth: 1\n\n") + + for category in category_names: + if category.startswith("_"): + continue + category_list_file.write(" list_of_%s_modules\n" % category) + process_category(category, categories, options, env, template, outputname) + + category_list_file.close() + +if __name__ == '__main__': + main() diff --git a/v2/hacking/templates/rst.j2 b/v2/hacking/templates/rst.j2 new file mode 100644 index 0000000000..232d97a731 --- /dev/null +++ b/v2/hacking/templates/rst.j2 @@ -0,0 +1,153 @@ +.. _@{ module }@: + +{% if short_description %} +{% set title = module + ' - ' + short_description|convert_symbols_to_format %} +{% else %} +{% set title = module %} +{% endif %} +{% set title_len = title|length %} + +@{ title }@ +@{ '+' * title_len }@ + +.. contents:: + :local: + :depth: 1 + +{# ------------------------------------------ + # + # Please note: this looks like a core dump + # but it isn't one. + # + --------------------------------------------#} + +{% if aliases is defined -%} +Aliases: @{ ','.join(aliases) }@ +{% endif %} + +{% if deprecated is defined -%} +DEPRECATED +---------- + +@{ deprecated }@ +{% endif %} + +Synopsis +-------- + +{% if version_added is defined -%} +.. versionadded:: @{ version_added }@ +{% endif %} + +{% for desc in description -%} +@{ desc | convert_symbols_to_format }@ +{% endfor %} + +{% if options -%} +Options +------- + +.. raw:: html + + <table border=1 cellpadding=4> + <tr> + <th class="head">parameter</th> + <th class="head">required</th> + <th class="head">default</th> + <th class="head">choices</th> + <th class="head">comments</th> + </tr> + {% for k in option_keys %} + {% set v = options[k] %} + <tr> + <td>@{ k }@</td> + <td>{% if v.get('required', False) %}yes{% else %}no{% endif %}</td> + <td>{% if v['default'] %}@{ v['default'] }@{% endif %}</td> + {% if v.get('type', 'not_bool') == 'bool' %} + <td><ul><li>yes</li><li>no</li></ul></td> + {% else %} + <td><ul>{% for choice in v.get('choices',[]) -%}<li>@{ choice }@</li>{% endfor -%}</ul></td> + {% endif %} + <td>{% for desc in v.description -%}@{ desc | html_ify }@{% endfor -%}{% if v['version_added'] %} (added in Ansible @{v['version_added']}@){% endif %}</td> + </tr> + {% endfor %} + </table> +{% endif %} + +{% if requirements %} +{% for req in requirements %} + +.. note:: Requires @{ req | convert_symbols_to_format }@ + +{% endfor %} +{% endif %} + +{% if examples or plainexamples %} +Examples +-------- + +.. raw:: html + +{% for example in examples %} + {% if example['description'] %}<p>@{ example['description'] | html_ify }@</p>{% endif %} + <p> + <pre> +@{ example['code'] | escape | indent(4, True) }@ + </pre> + </p> +{% endfor %} + <br/> + +{% if plainexamples %} + +:: + +@{ plainexamples | indent(4, True) }@ +{% endif %} +{% endif %} + +{% if notes %} +{% for note in notes %} +.. note:: @{ note | convert_symbols_to_format }@ +{% endfor %} +{% endif %} + + +{% if not deprecated %} + {% if core %} + +This is a Core Module +--------------------- + +This source of this module is hosted on GitHub in the `ansible-modules-core <http://github.com/ansible/ansible-modules-core>`_ repo. + +If you believe you have found a bug in this module, and are already running the latest stable or development version of Ansible, first look in the `issue tracker at github.com/ansible/ansible-modules-core <http://github.com/ansible/ansible-modules-core>`_ to see if a bug has already been filed. If not, we would be grateful if you would file one. + +Should you have a question rather than a bug report, inquries are welcome on the `ansible-project google group <https://groups.google.com/forum/#!forum/ansible-project>`_ or on Ansible's "#ansible" channel, located on irc.freenode.net. Development oriented topics should instead use the similar `ansible-devel google group <https://groups.google.com/forum/#!forum/ansible-project>`_. + +Documentation updates for this module can also be edited directly by submitting a pull request to the module source code, just look for the "DOCUMENTATION" block in the source tree. + +This is a "core" ansible module, which means it will receive slightly higher priority for all requests than those in the "extras" repos. + + {% else %} + +This is an Extras Module +------------------------ + +This source of this module is hosted on GitHub in the `ansible-modules-extras <http://github.com/ansible/ansible-modules-extras>`_ repo. + +If you believe you have found a bug in this module, and are already running the latest stable or development version of Ansible, first look in the `issue tracker at github.com/ansible/ansible-modules-extras <http://github.com/ansible/ansible-modules-extras>`_ to see if a bug has already been filed. If not, we would be grateful if you would file one. + +Should you have a question rather than a bug report, inquries are welcome on the `ansible-project google group <https://groups.google.com/forum/#!forum/ansible-project>` or on Ansible's "#ansible" channel, located on irc.freenode.net. Development oriented topics should instead use the similar `ansible-devel google group <https://groups.google.com/forum/#!forum/ansible-project>`_. + +Documentation updates for this module can also be edited directly by submitting a pull request to the module source code, just look for the "DOCUMENTATION" block in the source tree. + +Note that this module is designated a "extras" module. Non-core modules are still fully usable, but may receive slightly lower response rates for issues and pull requests. +Popular "extras" modules may be promoted to core modules over time. + + {% endif %} +{% endif %} + +For help in developing on modules, should you be so inclined, please read :doc:`community`, :doc:`developing_test_pr` and :doc:`developing_modules`. + + diff --git a/v2/hacking/test-module b/v2/hacking/test-module new file mode 100755 index 0000000000..b6fe1f5cdb --- /dev/null +++ b/v2/hacking/test-module @@ -0,0 +1,193 @@ +#!/usr/bin/env python + +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +# + +# this script is for testing modules without running through the +# entire guts of ansible, and is very helpful for when developing +# modules +# +# example: +# test-module -m ../library/commands/command -a "/bin/sleep 3" +# test-module -m ../library/system/service -a "name=httpd ensure=restarted" +# test-module -m ../library/system/service -a "name=httpd ensure=restarted" --debugger /usr/bin/pdb +# test-modulr -m ../library/file/lineinfile -a "dest=/etc/exports line='/srv/home hostname1(rw,sync)'" --check + +import sys +import base64 +import os +import subprocess +import traceback +import optparse +import ansible.utils as utils +import ansible.module_common as module_common +import ansible.constants as C + +try: + import json +except ImportError: + import simplejson as json + +def parse(): + """parse command line + + :return : (options, args)""" + parser = optparse.OptionParser() + + parser.usage = "%prog -[options] (-h for help)" + + parser.add_option('-m', '--module-path', dest='module_path', + help="REQUIRED: full path of module source to execute") + parser.add_option('-a', '--args', dest='module_args', default="", + help="module argument string") + parser.add_option('-D', '--debugger', dest='debugger', + help="path to python debugger (e.g. /usr/bin/pdb)") + parser.add_option('-I', '--interpreter', dest='interpreter', + help="path to interpeter to use for this module (e.g. ansible_python_interpreter=/usr/bin/python)", + metavar='INTERPRETER_TYPE=INTERPRETER_PATH') + parser.add_option('-c', '--check', dest='check', action='store_true', + help="run the module in check mode") + options, args = parser.parse_args() + if not options.module_path: + parser.print_help() + sys.exit(1) + else: + return options, args + +def write_argsfile(argstring, json=False): + """ Write args to a file for old-style module's use. """ + argspath = os.path.expanduser("~/.ansible_test_module_arguments") + argsfile = open(argspath, 'w') + if json: + args = utils.parse_kv(argstring) + argstring = utils.jsonify(args) + argsfile.write(argstring) + argsfile.close() + return argspath + +def boilerplate_module(modfile, args, interpreter, check): + """ simulate what ansible does with new style modules """ + + #module_fh = open(modfile) + #module_data = module_fh.read() + #module_fh.close() + + replacer = module_common.ModuleReplacer() + + #included_boilerplate = module_data.find(module_common.REPLACER) != -1 or module_data.find("import ansible.module_utils") != -1 + + complex_args = {} + if args.startswith("@"): + # Argument is a YAML file (JSON is a subset of YAML) + complex_args = utils.combine_vars(complex_args, utils.parse_yaml_from_file(args[1:])) + args='' + elif args.startswith("{"): + # Argument is a YAML document (not a file) + complex_args = utils.combine_vars(complex_args, utils.parse_yaml(args)) + args='' + + inject = {} + if interpreter: + if '=' not in interpreter: + print 'interpeter must by in the form of ansible_python_interpreter=/usr/bin/python' + sys.exit(1) + interpreter_type, interpreter_path = interpreter.split('=') + if not interpreter_type.startswith('ansible_'): + interpreter_type = 'ansible_%s' % interpreter_type + if not interpreter_type.endswith('_interpreter'): + interpreter_type = '%s_interpreter' % interpreter_type + inject[interpreter_type] = interpreter_path + + if check: + complex_args['CHECKMODE'] = True + + (module_data, module_style, shebang) = replacer.modify_module( + modfile, + complex_args, + args, + inject + ) + + modfile2_path = os.path.expanduser("~/.ansible_module_generated") + print "* including generated source, if any, saving to: %s" % modfile2_path + print "* this may offset any line numbers in tracebacks/debuggers!" + modfile2 = open(modfile2_path, 'w') + modfile2.write(module_data) + modfile2.close() + modfile = modfile2_path + + return (modfile2_path, module_style) + +def runtest( modfile, argspath): + """Test run a module, piping it's output for reporting.""" + + os.system("chmod +x %s" % modfile) + + invoke = "%s" % (modfile) + if argspath is not None: + invoke = "%s %s" % (modfile, argspath) + + cmd = subprocess.Popen(invoke, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out, err) = cmd.communicate() + + try: + print "***********************************" + print "RAW OUTPUT" + print out + print err + results = utils.parse_json(out) + except: + print "***********************************" + print "INVALID OUTPUT FORMAT" + print out + traceback.print_exc() + sys.exit(1) + + print "***********************************" + print "PARSED OUTPUT" + print utils.jsonify(results,format=True) + +def rundebug(debugger, modfile, argspath): + """Run interactively with console debugger.""" + + if argspath is not None: + subprocess.call("%s %s %s" % (debugger, modfile, argspath), shell=True) + else: + subprocess.call("%s %s" % (debugger, modfile), shell=True) + +def main(): + + options, args = parse() + (modfile, module_style) = boilerplate_module(options.module_path, options.module_args, options.interpreter, options.check) + + argspath=None + if module_style != 'new': + if module_style == 'non_native_want_json': + argspath = write_argsfile(options.module_args, json=True) + elif module_style == 'old': + argspath = write_argsfile(options.module_args, json=False) + else: + raise Exception("internal error, unexpected module style: %s" % module_style) + if options.debugger: + rundebug(options.debugger, modfile, argspath) + else: + runtest(modfile, argspath) + +if __name__ == "__main__": + main() + diff --git a/v2/samples/README.md b/v2/samples/README.md new file mode 100644 index 0000000000..ed6af3a0c5 --- /dev/null +++ b/v2/samples/README.md @@ -0,0 +1 @@ +This is a small set of samples used for testing the v2 code. diff --git a/v2/samples/ignore_errors.yml b/v2/samples/ignore_errors.yml new file mode 100644 index 0000000000..5845d049c4 --- /dev/null +++ b/v2/samples/ignore_errors.yml @@ -0,0 +1,10 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - fail: + ignore_errors: yes + - debug: msg="you should still see this" + - fail: + - debug: msg="you should NOT see this" + diff --git a/v2/samples/inv_lg b/v2/samples/inv_lg new file mode 100644 index 0000000000..c99596a6ba --- /dev/null +++ b/v2/samples/inv_lg @@ -0,0 +1,2540 @@ +127.0.0.1 +127.0.0.2 +127.0.0.3 +127.0.0.4 +127.0.0.5 +127.0.0.6 +127.0.0.7 +127.0.0.8 +127.0.0.9 +127.0.0.10 +127.0.0.11 +127.0.0.12 +127.0.0.13 +127.0.0.14 +127.0.0.15 +127.0.0.16 +127.0.0.17 +127.0.0.18 +127.0.0.19 +127.0.0.20 +127.0.0.21 +127.0.0.22 +127.0.0.23 +127.0.0.24 +127.0.0.25 +127.0.0.26 +127.0.0.27 +127.0.0.28 +127.0.0.29 +127.0.0.30 +127.0.0.31 +127.0.0.32 +127.0.0.33 +127.0.0.34 +127.0.0.35 +127.0.0.36 +127.0.0.37 +127.0.0.38 +127.0.0.39 +127.0.0.40 +127.0.0.41 +127.0.0.42 +127.0.0.43 +127.0.0.44 +127.0.0.45 +127.0.0.46 +127.0.0.47 +127.0.0.48 +127.0.0.49 +127.0.0.50 +127.0.0.51 +127.0.0.52 +127.0.0.53 +127.0.0.54 +127.0.0.55 +127.0.0.56 +127.0.0.57 +127.0.0.58 +127.0.0.59 +127.0.0.60 +127.0.0.61 +127.0.0.62 +127.0.0.63 +127.0.0.64 +127.0.0.65 +127.0.0.66 +127.0.0.67 +127.0.0.68 +127.0.0.69 +127.0.0.70 +127.0.0.71 +127.0.0.72 +127.0.0.73 +127.0.0.74 +127.0.0.75 +127.0.0.76 +127.0.0.77 +127.0.0.78 +127.0.0.79 +127.0.0.80 +127.0.0.81 +127.0.0.82 +127.0.0.83 +127.0.0.84 +127.0.0.85 +127.0.0.86 +127.0.0.87 +127.0.0.88 +127.0.0.89 +127.0.0.90 +127.0.0.91 +127.0.0.92 +127.0.0.93 +127.0.0.94 +127.0.0.95 +127.0.0.96 +127.0.0.97 +127.0.0.98 +127.0.0.99 +127.0.0.100 +127.0.0.101 +127.0.0.102 +127.0.0.103 +127.0.0.104 +127.0.0.105 +127.0.0.106 +127.0.0.107 +127.0.0.108 +127.0.0.109 +127.0.0.110 +127.0.0.111 +127.0.0.112 +127.0.0.113 +127.0.0.114 +127.0.0.115 +127.0.0.116 +127.0.0.117 +127.0.0.118 +127.0.0.119 +127.0.0.120 +127.0.0.121 +127.0.0.122 +127.0.0.123 +127.0.0.124 +127.0.0.125 +127.0.0.126 +127.0.0.127 +127.0.0.128 +127.0.0.129 +127.0.0.130 +127.0.0.131 +127.0.0.132 +127.0.0.133 +127.0.0.134 +127.0.0.135 +127.0.0.136 +127.0.0.137 +127.0.0.138 +127.0.0.139 +127.0.0.140 +127.0.0.141 +127.0.0.142 +127.0.0.143 +127.0.0.144 +127.0.0.145 +127.0.0.146 +127.0.0.147 +127.0.0.148 +127.0.0.149 +127.0.0.150 +127.0.0.151 +127.0.0.152 +127.0.0.153 +127.0.0.154 +127.0.0.155 +127.0.0.156 +127.0.0.157 +127.0.0.158 +127.0.0.159 +127.0.0.160 +127.0.0.161 +127.0.0.162 +127.0.0.163 +127.0.0.164 +127.0.0.165 +127.0.0.166 +127.0.0.167 +127.0.0.168 +127.0.0.169 +127.0.0.170 +127.0.0.171 +127.0.0.172 +127.0.0.173 +127.0.0.174 +127.0.0.175 +127.0.0.176 +127.0.0.177 +127.0.0.178 +127.0.0.179 +127.0.0.180 +127.0.0.181 +127.0.0.182 +127.0.0.183 +127.0.0.184 +127.0.0.185 +127.0.0.186 +127.0.0.187 +127.0.0.188 +127.0.0.189 +127.0.0.190 +127.0.0.191 +127.0.0.192 +127.0.0.193 +127.0.0.194 +127.0.0.195 +127.0.0.196 +127.0.0.197 +127.0.0.198 +127.0.0.199 +127.0.0.200 +127.0.0.201 +127.0.0.202 +127.0.0.203 +127.0.0.204 +127.0.0.205 +127.0.0.206 +127.0.0.207 +127.0.0.208 +127.0.0.209 +127.0.0.210 +127.0.0.211 +127.0.0.212 +127.0.0.213 +127.0.0.214 +127.0.0.215 +127.0.0.216 +127.0.0.217 +127.0.0.218 +127.0.0.219 +127.0.0.220 +127.0.0.221 +127.0.0.222 +127.0.0.223 +127.0.0.224 +127.0.0.225 +127.0.0.226 +127.0.0.227 +127.0.0.228 +127.0.0.229 +127.0.0.230 +127.0.0.231 +127.0.0.232 +127.0.0.233 +127.0.0.234 +127.0.0.235 +127.0.0.236 +127.0.0.237 +127.0.0.238 +127.0.0.239 +127.0.0.240 +127.0.0.241 +127.0.0.242 +127.0.0.243 +127.0.0.244 +127.0.0.245 +127.0.0.246 +127.0.0.247 +127.0.0.248 +127.0.0.249 +127.0.0.250 +127.0.0.251 +127.0.0.252 +127.0.0.253 +127.0.0.254 +127.0.1.1 +127.0.1.2 +127.0.1.3 +127.0.1.4 +127.0.1.5 +127.0.1.6 +127.0.1.7 +127.0.1.8 +127.0.1.9 +127.0.1.10 +127.0.1.11 +127.0.1.12 +127.0.1.13 +127.0.1.14 +127.0.1.15 +127.0.1.16 +127.0.1.17 +127.0.1.18 +127.0.1.19 +127.0.1.20 +127.0.1.21 +127.0.1.22 +127.0.1.23 +127.0.1.24 +127.0.1.25 +127.0.1.26 +127.0.1.27 +127.0.1.28 +127.0.1.29 +127.0.1.30 +127.0.1.31 +127.0.1.32 +127.0.1.33 +127.0.1.34 +127.0.1.35 +127.0.1.36 +127.0.1.37 +127.0.1.38 +127.0.1.39 +127.0.1.40 +127.0.1.41 +127.0.1.42 +127.0.1.43 +127.0.1.44 +127.0.1.45 +127.0.1.46 +127.0.1.47 +127.0.1.48 +127.0.1.49 +127.0.1.50 +127.0.1.51 +127.0.1.52 +127.0.1.53 +127.0.1.54 +127.0.1.55 +127.0.1.56 +127.0.1.57 +127.0.1.58 +127.0.1.59 +127.0.1.60 +127.0.1.61 +127.0.1.62 +127.0.1.63 +127.0.1.64 +127.0.1.65 +127.0.1.66 +127.0.1.67 +127.0.1.68 +127.0.1.69 +127.0.1.70 +127.0.1.71 +127.0.1.72 +127.0.1.73 +127.0.1.74 +127.0.1.75 +127.0.1.76 +127.0.1.77 +127.0.1.78 +127.0.1.79 +127.0.1.80 +127.0.1.81 +127.0.1.82 +127.0.1.83 +127.0.1.84 +127.0.1.85 +127.0.1.86 +127.0.1.87 +127.0.1.88 +127.0.1.89 +127.0.1.90 +127.0.1.91 +127.0.1.92 +127.0.1.93 +127.0.1.94 +127.0.1.95 +127.0.1.96 +127.0.1.97 +127.0.1.98 +127.0.1.99 +127.0.1.100 +127.0.1.101 +127.0.1.102 +127.0.1.103 +127.0.1.104 +127.0.1.105 +127.0.1.106 +127.0.1.107 +127.0.1.108 +127.0.1.109 +127.0.1.110 +127.0.1.111 +127.0.1.112 +127.0.1.113 +127.0.1.114 +127.0.1.115 +127.0.1.116 +127.0.1.117 +127.0.1.118 +127.0.1.119 +127.0.1.120 +127.0.1.121 +127.0.1.122 +127.0.1.123 +127.0.1.124 +127.0.1.125 +127.0.1.126 +127.0.1.127 +127.0.1.128 +127.0.1.129 +127.0.1.130 +127.0.1.131 +127.0.1.132 +127.0.1.133 +127.0.1.134 +127.0.1.135 +127.0.1.136 +127.0.1.137 +127.0.1.138 +127.0.1.139 +127.0.1.140 +127.0.1.141 +127.0.1.142 +127.0.1.143 +127.0.1.144 +127.0.1.145 +127.0.1.146 +127.0.1.147 +127.0.1.148 +127.0.1.149 +127.0.1.150 +127.0.1.151 +127.0.1.152 +127.0.1.153 +127.0.1.154 +127.0.1.155 +127.0.1.156 +127.0.1.157 +127.0.1.158 +127.0.1.159 +127.0.1.160 +127.0.1.161 +127.0.1.162 +127.0.1.163 +127.0.1.164 +127.0.1.165 +127.0.1.166 +127.0.1.167 +127.0.1.168 +127.0.1.169 +127.0.1.170 +127.0.1.171 +127.0.1.172 +127.0.1.173 +127.0.1.174 +127.0.1.175 +127.0.1.176 +127.0.1.177 +127.0.1.178 +127.0.1.179 +127.0.1.180 +127.0.1.181 +127.0.1.182 +127.0.1.183 +127.0.1.184 +127.0.1.185 +127.0.1.186 +127.0.1.187 +127.0.1.188 +127.0.1.189 +127.0.1.190 +127.0.1.191 +127.0.1.192 +127.0.1.193 +127.0.1.194 +127.0.1.195 +127.0.1.196 +127.0.1.197 +127.0.1.198 +127.0.1.199 +127.0.1.200 +127.0.1.201 +127.0.1.202 +127.0.1.203 +127.0.1.204 +127.0.1.205 +127.0.1.206 +127.0.1.207 +127.0.1.208 +127.0.1.209 +127.0.1.210 +127.0.1.211 +127.0.1.212 +127.0.1.213 +127.0.1.214 +127.0.1.215 +127.0.1.216 +127.0.1.217 +127.0.1.218 +127.0.1.219 +127.0.1.220 +127.0.1.221 +127.0.1.222 +127.0.1.223 +127.0.1.224 +127.0.1.225 +127.0.1.226 +127.0.1.227 +127.0.1.228 +127.0.1.229 +127.0.1.230 +127.0.1.231 +127.0.1.232 +127.0.1.233 +127.0.1.234 +127.0.1.235 +127.0.1.236 +127.0.1.237 +127.0.1.238 +127.0.1.239 +127.0.1.240 +127.0.1.241 +127.0.1.242 +127.0.1.243 +127.0.1.244 +127.0.1.245 +127.0.1.246 +127.0.1.247 +127.0.1.248 +127.0.1.249 +127.0.1.250 +127.0.1.251 +127.0.1.252 +127.0.1.253 +127.0.1.254 +127.0.2.1 +127.0.2.2 +127.0.2.3 +127.0.2.4 +127.0.2.5 +127.0.2.6 +127.0.2.7 +127.0.2.8 +127.0.2.9 +127.0.2.10 +127.0.2.11 +127.0.2.12 +127.0.2.13 +127.0.2.14 +127.0.2.15 +127.0.2.16 +127.0.2.17 +127.0.2.18 +127.0.2.19 +127.0.2.20 +127.0.2.21 +127.0.2.22 +127.0.2.23 +127.0.2.24 +127.0.2.25 +127.0.2.26 +127.0.2.27 +127.0.2.28 +127.0.2.29 +127.0.2.30 +127.0.2.31 +127.0.2.32 +127.0.2.33 +127.0.2.34 +127.0.2.35 +127.0.2.36 +127.0.2.37 +127.0.2.38 +127.0.2.39 +127.0.2.40 +127.0.2.41 +127.0.2.42 +127.0.2.43 +127.0.2.44 +127.0.2.45 +127.0.2.46 +127.0.2.47 +127.0.2.48 +127.0.2.49 +127.0.2.50 +127.0.2.51 +127.0.2.52 +127.0.2.53 +127.0.2.54 +127.0.2.55 +127.0.2.56 +127.0.2.57 +127.0.2.58 +127.0.2.59 +127.0.2.60 +127.0.2.61 +127.0.2.62 +127.0.2.63 +127.0.2.64 +127.0.2.65 +127.0.2.66 +127.0.2.67 +127.0.2.68 +127.0.2.69 +127.0.2.70 +127.0.2.71 +127.0.2.72 +127.0.2.73 +127.0.2.74 +127.0.2.75 +127.0.2.76 +127.0.2.77 +127.0.2.78 +127.0.2.79 +127.0.2.80 +127.0.2.81 +127.0.2.82 +127.0.2.83 +127.0.2.84 +127.0.2.85 +127.0.2.86 +127.0.2.87 +127.0.2.88 +127.0.2.89 +127.0.2.90 +127.0.2.91 +127.0.2.92 +127.0.2.93 +127.0.2.94 +127.0.2.95 +127.0.2.96 +127.0.2.97 +127.0.2.98 +127.0.2.99 +127.0.2.100 +127.0.2.101 +127.0.2.102 +127.0.2.103 +127.0.2.104 +127.0.2.105 +127.0.2.106 +127.0.2.107 +127.0.2.108 +127.0.2.109 +127.0.2.110 +127.0.2.111 +127.0.2.112 +127.0.2.113 +127.0.2.114 +127.0.2.115 +127.0.2.116 +127.0.2.117 +127.0.2.118 +127.0.2.119 +127.0.2.120 +127.0.2.121 +127.0.2.122 +127.0.2.123 +127.0.2.124 +127.0.2.125 +127.0.2.126 +127.0.2.127 +127.0.2.128 +127.0.2.129 +127.0.2.130 +127.0.2.131 +127.0.2.132 +127.0.2.133 +127.0.2.134 +127.0.2.135 +127.0.2.136 +127.0.2.137 +127.0.2.138 +127.0.2.139 +127.0.2.140 +127.0.2.141 +127.0.2.142 +127.0.2.143 +127.0.2.144 +127.0.2.145 +127.0.2.146 +127.0.2.147 +127.0.2.148 +127.0.2.149 +127.0.2.150 +127.0.2.151 +127.0.2.152 +127.0.2.153 +127.0.2.154 +127.0.2.155 +127.0.2.156 +127.0.2.157 +127.0.2.158 +127.0.2.159 +127.0.2.160 +127.0.2.161 +127.0.2.162 +127.0.2.163 +127.0.2.164 +127.0.2.165 +127.0.2.166 +127.0.2.167 +127.0.2.168 +127.0.2.169 +127.0.2.170 +127.0.2.171 +127.0.2.172 +127.0.2.173 +127.0.2.174 +127.0.2.175 +127.0.2.176 +127.0.2.177 +127.0.2.178 +127.0.2.179 +127.0.2.180 +127.0.2.181 +127.0.2.182 +127.0.2.183 +127.0.2.184 +127.0.2.185 +127.0.2.186 +127.0.2.187 +127.0.2.188 +127.0.2.189 +127.0.2.190 +127.0.2.191 +127.0.2.192 +127.0.2.193 +127.0.2.194 +127.0.2.195 +127.0.2.196 +127.0.2.197 +127.0.2.198 +127.0.2.199 +127.0.2.200 +127.0.2.201 +127.0.2.202 +127.0.2.203 +127.0.2.204 +127.0.2.205 +127.0.2.206 +127.0.2.207 +127.0.2.208 +127.0.2.209 +127.0.2.210 +127.0.2.211 +127.0.2.212 +127.0.2.213 +127.0.2.214 +127.0.2.215 +127.0.2.216 +127.0.2.217 +127.0.2.218 +127.0.2.219 +127.0.2.220 +127.0.2.221 +127.0.2.222 +127.0.2.223 +127.0.2.224 +127.0.2.225 +127.0.2.226 +127.0.2.227 +127.0.2.228 +127.0.2.229 +127.0.2.230 +127.0.2.231 +127.0.2.232 +127.0.2.233 +127.0.2.234 +127.0.2.235 +127.0.2.236 +127.0.2.237 +127.0.2.238 +127.0.2.239 +127.0.2.240 +127.0.2.241 +127.0.2.242 +127.0.2.243 +127.0.2.244 +127.0.2.245 +127.0.2.246 +127.0.2.247 +127.0.2.248 +127.0.2.249 +127.0.2.250 +127.0.2.251 +127.0.2.252 +127.0.2.253 +127.0.2.254 +127.0.3.1 +127.0.3.2 +127.0.3.3 +127.0.3.4 +127.0.3.5 +127.0.3.6 +127.0.3.7 +127.0.3.8 +127.0.3.9 +127.0.3.10 +127.0.3.11 +127.0.3.12 +127.0.3.13 +127.0.3.14 +127.0.3.15 +127.0.3.16 +127.0.3.17 +127.0.3.18 +127.0.3.19 +127.0.3.20 +127.0.3.21 +127.0.3.22 +127.0.3.23 +127.0.3.24 +127.0.3.25 +127.0.3.26 +127.0.3.27 +127.0.3.28 +127.0.3.29 +127.0.3.30 +127.0.3.31 +127.0.3.32 +127.0.3.33 +127.0.3.34 +127.0.3.35 +127.0.3.36 +127.0.3.37 +127.0.3.38 +127.0.3.39 +127.0.3.40 +127.0.3.41 +127.0.3.42 +127.0.3.43 +127.0.3.44 +127.0.3.45 +127.0.3.46 +127.0.3.47 +127.0.3.48 +127.0.3.49 +127.0.3.50 +127.0.3.51 +127.0.3.52 +127.0.3.53 +127.0.3.54 +127.0.3.55 +127.0.3.56 +127.0.3.57 +127.0.3.58 +127.0.3.59 +127.0.3.60 +127.0.3.61 +127.0.3.62 +127.0.3.63 +127.0.3.64 +127.0.3.65 +127.0.3.66 +127.0.3.67 +127.0.3.68 +127.0.3.69 +127.0.3.70 +127.0.3.71 +127.0.3.72 +127.0.3.73 +127.0.3.74 +127.0.3.75 +127.0.3.76 +127.0.3.77 +127.0.3.78 +127.0.3.79 +127.0.3.80 +127.0.3.81 +127.0.3.82 +127.0.3.83 +127.0.3.84 +127.0.3.85 +127.0.3.86 +127.0.3.87 +127.0.3.88 +127.0.3.89 +127.0.3.90 +127.0.3.91 +127.0.3.92 +127.0.3.93 +127.0.3.94 +127.0.3.95 +127.0.3.96 +127.0.3.97 +127.0.3.98 +127.0.3.99 +127.0.3.100 +127.0.3.101 +127.0.3.102 +127.0.3.103 +127.0.3.104 +127.0.3.105 +127.0.3.106 +127.0.3.107 +127.0.3.108 +127.0.3.109 +127.0.3.110 +127.0.3.111 +127.0.3.112 +127.0.3.113 +127.0.3.114 +127.0.3.115 +127.0.3.116 +127.0.3.117 +127.0.3.118 +127.0.3.119 +127.0.3.120 +127.0.3.121 +127.0.3.122 +127.0.3.123 +127.0.3.124 +127.0.3.125 +127.0.3.126 +127.0.3.127 +127.0.3.128 +127.0.3.129 +127.0.3.130 +127.0.3.131 +127.0.3.132 +127.0.3.133 +127.0.3.134 +127.0.3.135 +127.0.3.136 +127.0.3.137 +127.0.3.138 +127.0.3.139 +127.0.3.140 +127.0.3.141 +127.0.3.142 +127.0.3.143 +127.0.3.144 +127.0.3.145 +127.0.3.146 +127.0.3.147 +127.0.3.148 +127.0.3.149 +127.0.3.150 +127.0.3.151 +127.0.3.152 +127.0.3.153 +127.0.3.154 +127.0.3.155 +127.0.3.156 +127.0.3.157 +127.0.3.158 +127.0.3.159 +127.0.3.160 +127.0.3.161 +127.0.3.162 +127.0.3.163 +127.0.3.164 +127.0.3.165 +127.0.3.166 +127.0.3.167 +127.0.3.168 +127.0.3.169 +127.0.3.170 +127.0.3.171 +127.0.3.172 +127.0.3.173 +127.0.3.174 +127.0.3.175 +127.0.3.176 +127.0.3.177 +127.0.3.178 +127.0.3.179 +127.0.3.180 +127.0.3.181 +127.0.3.182 +127.0.3.183 +127.0.3.184 +127.0.3.185 +127.0.3.186 +127.0.3.187 +127.0.3.188 +127.0.3.189 +127.0.3.190 +127.0.3.191 +127.0.3.192 +127.0.3.193 +127.0.3.194 +127.0.3.195 +127.0.3.196 +127.0.3.197 +127.0.3.198 +127.0.3.199 +127.0.3.200 +127.0.3.201 +127.0.3.202 +127.0.3.203 +127.0.3.204 +127.0.3.205 +127.0.3.206 +127.0.3.207 +127.0.3.208 +127.0.3.209 +127.0.3.210 +127.0.3.211 +127.0.3.212 +127.0.3.213 +127.0.3.214 +127.0.3.215 +127.0.3.216 +127.0.3.217 +127.0.3.218 +127.0.3.219 +127.0.3.220 +127.0.3.221 +127.0.3.222 +127.0.3.223 +127.0.3.224 +127.0.3.225 +127.0.3.226 +127.0.3.227 +127.0.3.228 +127.0.3.229 +127.0.3.230 +127.0.3.231 +127.0.3.232 +127.0.3.233 +127.0.3.234 +127.0.3.235 +127.0.3.236 +127.0.3.237 +127.0.3.238 +127.0.3.239 +127.0.3.240 +127.0.3.241 +127.0.3.242 +127.0.3.243 +127.0.3.244 +127.0.3.245 +127.0.3.246 +127.0.3.247 +127.0.3.248 +127.0.3.249 +127.0.3.250 +127.0.3.251 +127.0.3.252 +127.0.3.253 +127.0.3.254 +127.0.4.1 +127.0.4.2 +127.0.4.3 +127.0.4.4 +127.0.4.5 +127.0.4.6 +127.0.4.7 +127.0.4.8 +127.0.4.9 +127.0.4.10 +127.0.4.11 +127.0.4.12 +127.0.4.13 +127.0.4.14 +127.0.4.15 +127.0.4.16 +127.0.4.17 +127.0.4.18 +127.0.4.19 +127.0.4.20 +127.0.4.21 +127.0.4.22 +127.0.4.23 +127.0.4.24 +127.0.4.25 +127.0.4.26 +127.0.4.27 +127.0.4.28 +127.0.4.29 +127.0.4.30 +127.0.4.31 +127.0.4.32 +127.0.4.33 +127.0.4.34 +127.0.4.35 +127.0.4.36 +127.0.4.37 +127.0.4.38 +127.0.4.39 +127.0.4.40 +127.0.4.41 +127.0.4.42 +127.0.4.43 +127.0.4.44 +127.0.4.45 +127.0.4.46 +127.0.4.47 +127.0.4.48 +127.0.4.49 +127.0.4.50 +127.0.4.51 +127.0.4.52 +127.0.4.53 +127.0.4.54 +127.0.4.55 +127.0.4.56 +127.0.4.57 +127.0.4.58 +127.0.4.59 +127.0.4.60 +127.0.4.61 +127.0.4.62 +127.0.4.63 +127.0.4.64 +127.0.4.65 +127.0.4.66 +127.0.4.67 +127.0.4.68 +127.0.4.69 +127.0.4.70 +127.0.4.71 +127.0.4.72 +127.0.4.73 +127.0.4.74 +127.0.4.75 +127.0.4.76 +127.0.4.77 +127.0.4.78 +127.0.4.79 +127.0.4.80 +127.0.4.81 +127.0.4.82 +127.0.4.83 +127.0.4.84 +127.0.4.85 +127.0.4.86 +127.0.4.87 +127.0.4.88 +127.0.4.89 +127.0.4.90 +127.0.4.91 +127.0.4.92 +127.0.4.93 +127.0.4.94 +127.0.4.95 +127.0.4.96 +127.0.4.97 +127.0.4.98 +127.0.4.99 +127.0.4.100 +127.0.4.101 +127.0.4.102 +127.0.4.103 +127.0.4.104 +127.0.4.105 +127.0.4.106 +127.0.4.107 +127.0.4.108 +127.0.4.109 +127.0.4.110 +127.0.4.111 +127.0.4.112 +127.0.4.113 +127.0.4.114 +127.0.4.115 +127.0.4.116 +127.0.4.117 +127.0.4.118 +127.0.4.119 +127.0.4.120 +127.0.4.121 +127.0.4.122 +127.0.4.123 +127.0.4.124 +127.0.4.125 +127.0.4.126 +127.0.4.127 +127.0.4.128 +127.0.4.129 +127.0.4.130 +127.0.4.131 +127.0.4.132 +127.0.4.133 +127.0.4.134 +127.0.4.135 +127.0.4.136 +127.0.4.137 +127.0.4.138 +127.0.4.139 +127.0.4.140 +127.0.4.141 +127.0.4.142 +127.0.4.143 +127.0.4.144 +127.0.4.145 +127.0.4.146 +127.0.4.147 +127.0.4.148 +127.0.4.149 +127.0.4.150 +127.0.4.151 +127.0.4.152 +127.0.4.153 +127.0.4.154 +127.0.4.155 +127.0.4.156 +127.0.4.157 +127.0.4.158 +127.0.4.159 +127.0.4.160 +127.0.4.161 +127.0.4.162 +127.0.4.163 +127.0.4.164 +127.0.4.165 +127.0.4.166 +127.0.4.167 +127.0.4.168 +127.0.4.169 +127.0.4.170 +127.0.4.171 +127.0.4.172 +127.0.4.173 +127.0.4.174 +127.0.4.175 +127.0.4.176 +127.0.4.177 +127.0.4.178 +127.0.4.179 +127.0.4.180 +127.0.4.181 +127.0.4.182 +127.0.4.183 +127.0.4.184 +127.0.4.185 +127.0.4.186 +127.0.4.187 +127.0.4.188 +127.0.4.189 +127.0.4.190 +127.0.4.191 +127.0.4.192 +127.0.4.193 +127.0.4.194 +127.0.4.195 +127.0.4.196 +127.0.4.197 +127.0.4.198 +127.0.4.199 +127.0.4.200 +127.0.4.201 +127.0.4.202 +127.0.4.203 +127.0.4.204 +127.0.4.205 +127.0.4.206 +127.0.4.207 +127.0.4.208 +127.0.4.209 +127.0.4.210 +127.0.4.211 +127.0.4.212 +127.0.4.213 +127.0.4.214 +127.0.4.215 +127.0.4.216 +127.0.4.217 +127.0.4.218 +127.0.4.219 +127.0.4.220 +127.0.4.221 +127.0.4.222 +127.0.4.223 +127.0.4.224 +127.0.4.225 +127.0.4.226 +127.0.4.227 +127.0.4.228 +127.0.4.229 +127.0.4.230 +127.0.4.231 +127.0.4.232 +127.0.4.233 +127.0.4.234 +127.0.4.235 +127.0.4.236 +127.0.4.237 +127.0.4.238 +127.0.4.239 +127.0.4.240 +127.0.4.241 +127.0.4.242 +127.0.4.243 +127.0.4.244 +127.0.4.245 +127.0.4.246 +127.0.4.247 +127.0.4.248 +127.0.4.249 +127.0.4.250 +127.0.4.251 +127.0.4.252 +127.0.4.253 +127.0.4.254 +127.0.5.1 +127.0.5.2 +127.0.5.3 +127.0.5.4 +127.0.5.5 +127.0.5.6 +127.0.5.7 +127.0.5.8 +127.0.5.9 +127.0.5.10 +127.0.5.11 +127.0.5.12 +127.0.5.13 +127.0.5.14 +127.0.5.15 +127.0.5.16 +127.0.5.17 +127.0.5.18 +127.0.5.19 +127.0.5.20 +127.0.5.21 +127.0.5.22 +127.0.5.23 +127.0.5.24 +127.0.5.25 +127.0.5.26 +127.0.5.27 +127.0.5.28 +127.0.5.29 +127.0.5.30 +127.0.5.31 +127.0.5.32 +127.0.5.33 +127.0.5.34 +127.0.5.35 +127.0.5.36 +127.0.5.37 +127.0.5.38 +127.0.5.39 +127.0.5.40 +127.0.5.41 +127.0.5.42 +127.0.5.43 +127.0.5.44 +127.0.5.45 +127.0.5.46 +127.0.5.47 +127.0.5.48 +127.0.5.49 +127.0.5.50 +127.0.5.51 +127.0.5.52 +127.0.5.53 +127.0.5.54 +127.0.5.55 +127.0.5.56 +127.0.5.57 +127.0.5.58 +127.0.5.59 +127.0.5.60 +127.0.5.61 +127.0.5.62 +127.0.5.63 +127.0.5.64 +127.0.5.65 +127.0.5.66 +127.0.5.67 +127.0.5.68 +127.0.5.69 +127.0.5.70 +127.0.5.71 +127.0.5.72 +127.0.5.73 +127.0.5.74 +127.0.5.75 +127.0.5.76 +127.0.5.77 +127.0.5.78 +127.0.5.79 +127.0.5.80 +127.0.5.81 +127.0.5.82 +127.0.5.83 +127.0.5.84 +127.0.5.85 +127.0.5.86 +127.0.5.87 +127.0.5.88 +127.0.5.89 +127.0.5.90 +127.0.5.91 +127.0.5.92 +127.0.5.93 +127.0.5.94 +127.0.5.95 +127.0.5.96 +127.0.5.97 +127.0.5.98 +127.0.5.99 +127.0.5.100 +127.0.5.101 +127.0.5.102 +127.0.5.103 +127.0.5.104 +127.0.5.105 +127.0.5.106 +127.0.5.107 +127.0.5.108 +127.0.5.109 +127.0.5.110 +127.0.5.111 +127.0.5.112 +127.0.5.113 +127.0.5.114 +127.0.5.115 +127.0.5.116 +127.0.5.117 +127.0.5.118 +127.0.5.119 +127.0.5.120 +127.0.5.121 +127.0.5.122 +127.0.5.123 +127.0.5.124 +127.0.5.125 +127.0.5.126 +127.0.5.127 +127.0.5.128 +127.0.5.129 +127.0.5.130 +127.0.5.131 +127.0.5.132 +127.0.5.133 +127.0.5.134 +127.0.5.135 +127.0.5.136 +127.0.5.137 +127.0.5.138 +127.0.5.139 +127.0.5.140 +127.0.5.141 +127.0.5.142 +127.0.5.143 +127.0.5.144 +127.0.5.145 +127.0.5.146 +127.0.5.147 +127.0.5.148 +127.0.5.149 +127.0.5.150 +127.0.5.151 +127.0.5.152 +127.0.5.153 +127.0.5.154 +127.0.5.155 +127.0.5.156 +127.0.5.157 +127.0.5.158 +127.0.5.159 +127.0.5.160 +127.0.5.161 +127.0.5.162 +127.0.5.163 +127.0.5.164 +127.0.5.165 +127.0.5.166 +127.0.5.167 +127.0.5.168 +127.0.5.169 +127.0.5.170 +127.0.5.171 +127.0.5.172 +127.0.5.173 +127.0.5.174 +127.0.5.175 +127.0.5.176 +127.0.5.177 +127.0.5.178 +127.0.5.179 +127.0.5.180 +127.0.5.181 +127.0.5.182 +127.0.5.183 +127.0.5.184 +127.0.5.185 +127.0.5.186 +127.0.5.187 +127.0.5.188 +127.0.5.189 +127.0.5.190 +127.0.5.191 +127.0.5.192 +127.0.5.193 +127.0.5.194 +127.0.5.195 +127.0.5.196 +127.0.5.197 +127.0.5.198 +127.0.5.199 +127.0.5.200 +127.0.5.201 +127.0.5.202 +127.0.5.203 +127.0.5.204 +127.0.5.205 +127.0.5.206 +127.0.5.207 +127.0.5.208 +127.0.5.209 +127.0.5.210 +127.0.5.211 +127.0.5.212 +127.0.5.213 +127.0.5.214 +127.0.5.215 +127.0.5.216 +127.0.5.217 +127.0.5.218 +127.0.5.219 +127.0.5.220 +127.0.5.221 +127.0.5.222 +127.0.5.223 +127.0.5.224 +127.0.5.225 +127.0.5.226 +127.0.5.227 +127.0.5.228 +127.0.5.229 +127.0.5.230 +127.0.5.231 +127.0.5.232 +127.0.5.233 +127.0.5.234 +127.0.5.235 +127.0.5.236 +127.0.5.237 +127.0.5.238 +127.0.5.239 +127.0.5.240 +127.0.5.241 +127.0.5.242 +127.0.5.243 +127.0.5.244 +127.0.5.245 +127.0.5.246 +127.0.5.247 +127.0.5.248 +127.0.5.249 +127.0.5.250 +127.0.5.251 +127.0.5.252 +127.0.5.253 +127.0.5.254 +127.0.6.1 +127.0.6.2 +127.0.6.3 +127.0.6.4 +127.0.6.5 +127.0.6.6 +127.0.6.7 +127.0.6.8 +127.0.6.9 +127.0.6.10 +127.0.6.11 +127.0.6.12 +127.0.6.13 +127.0.6.14 +127.0.6.15 +127.0.6.16 +127.0.6.17 +127.0.6.18 +127.0.6.19 +127.0.6.20 +127.0.6.21 +127.0.6.22 +127.0.6.23 +127.0.6.24 +127.0.6.25 +127.0.6.26 +127.0.6.27 +127.0.6.28 +127.0.6.29 +127.0.6.30 +127.0.6.31 +127.0.6.32 +127.0.6.33 +127.0.6.34 +127.0.6.35 +127.0.6.36 +127.0.6.37 +127.0.6.38 +127.0.6.39 +127.0.6.40 +127.0.6.41 +127.0.6.42 +127.0.6.43 +127.0.6.44 +127.0.6.45 +127.0.6.46 +127.0.6.47 +127.0.6.48 +127.0.6.49 +127.0.6.50 +127.0.6.51 +127.0.6.52 +127.0.6.53 +127.0.6.54 +127.0.6.55 +127.0.6.56 +127.0.6.57 +127.0.6.58 +127.0.6.59 +127.0.6.60 +127.0.6.61 +127.0.6.62 +127.0.6.63 +127.0.6.64 +127.0.6.65 +127.0.6.66 +127.0.6.67 +127.0.6.68 +127.0.6.69 +127.0.6.70 +127.0.6.71 +127.0.6.72 +127.0.6.73 +127.0.6.74 +127.0.6.75 +127.0.6.76 +127.0.6.77 +127.0.6.78 +127.0.6.79 +127.0.6.80 +127.0.6.81 +127.0.6.82 +127.0.6.83 +127.0.6.84 +127.0.6.85 +127.0.6.86 +127.0.6.87 +127.0.6.88 +127.0.6.89 +127.0.6.90 +127.0.6.91 +127.0.6.92 +127.0.6.93 +127.0.6.94 +127.0.6.95 +127.0.6.96 +127.0.6.97 +127.0.6.98 +127.0.6.99 +127.0.6.100 +127.0.6.101 +127.0.6.102 +127.0.6.103 +127.0.6.104 +127.0.6.105 +127.0.6.106 +127.0.6.107 +127.0.6.108 +127.0.6.109 +127.0.6.110 +127.0.6.111 +127.0.6.112 +127.0.6.113 +127.0.6.114 +127.0.6.115 +127.0.6.116 +127.0.6.117 +127.0.6.118 +127.0.6.119 +127.0.6.120 +127.0.6.121 +127.0.6.122 +127.0.6.123 +127.0.6.124 +127.0.6.125 +127.0.6.126 +127.0.6.127 +127.0.6.128 +127.0.6.129 +127.0.6.130 +127.0.6.131 +127.0.6.132 +127.0.6.133 +127.0.6.134 +127.0.6.135 +127.0.6.136 +127.0.6.137 +127.0.6.138 +127.0.6.139 +127.0.6.140 +127.0.6.141 +127.0.6.142 +127.0.6.143 +127.0.6.144 +127.0.6.145 +127.0.6.146 +127.0.6.147 +127.0.6.148 +127.0.6.149 +127.0.6.150 +127.0.6.151 +127.0.6.152 +127.0.6.153 +127.0.6.154 +127.0.6.155 +127.0.6.156 +127.0.6.157 +127.0.6.158 +127.0.6.159 +127.0.6.160 +127.0.6.161 +127.0.6.162 +127.0.6.163 +127.0.6.164 +127.0.6.165 +127.0.6.166 +127.0.6.167 +127.0.6.168 +127.0.6.169 +127.0.6.170 +127.0.6.171 +127.0.6.172 +127.0.6.173 +127.0.6.174 +127.0.6.175 +127.0.6.176 +127.0.6.177 +127.0.6.178 +127.0.6.179 +127.0.6.180 +127.0.6.181 +127.0.6.182 +127.0.6.183 +127.0.6.184 +127.0.6.185 +127.0.6.186 +127.0.6.187 +127.0.6.188 +127.0.6.189 +127.0.6.190 +127.0.6.191 +127.0.6.192 +127.0.6.193 +127.0.6.194 +127.0.6.195 +127.0.6.196 +127.0.6.197 +127.0.6.198 +127.0.6.199 +127.0.6.200 +127.0.6.201 +127.0.6.202 +127.0.6.203 +127.0.6.204 +127.0.6.205 +127.0.6.206 +127.0.6.207 +127.0.6.208 +127.0.6.209 +127.0.6.210 +127.0.6.211 +127.0.6.212 +127.0.6.213 +127.0.6.214 +127.0.6.215 +127.0.6.216 +127.0.6.217 +127.0.6.218 +127.0.6.219 +127.0.6.220 +127.0.6.221 +127.0.6.222 +127.0.6.223 +127.0.6.224 +127.0.6.225 +127.0.6.226 +127.0.6.227 +127.0.6.228 +127.0.6.229 +127.0.6.230 +127.0.6.231 +127.0.6.232 +127.0.6.233 +127.0.6.234 +127.0.6.235 +127.0.6.236 +127.0.6.237 +127.0.6.238 +127.0.6.239 +127.0.6.240 +127.0.6.241 +127.0.6.242 +127.0.6.243 +127.0.6.244 +127.0.6.245 +127.0.6.246 +127.0.6.247 +127.0.6.248 +127.0.6.249 +127.0.6.250 +127.0.6.251 +127.0.6.252 +127.0.6.253 +127.0.6.254 +127.0.7.1 +127.0.7.2 +127.0.7.3 +127.0.7.4 +127.0.7.5 +127.0.7.6 +127.0.7.7 +127.0.7.8 +127.0.7.9 +127.0.7.10 +127.0.7.11 +127.0.7.12 +127.0.7.13 +127.0.7.14 +127.0.7.15 +127.0.7.16 +127.0.7.17 +127.0.7.18 +127.0.7.19 +127.0.7.20 +127.0.7.21 +127.0.7.22 +127.0.7.23 +127.0.7.24 +127.0.7.25 +127.0.7.26 +127.0.7.27 +127.0.7.28 +127.0.7.29 +127.0.7.30 +127.0.7.31 +127.0.7.32 +127.0.7.33 +127.0.7.34 +127.0.7.35 +127.0.7.36 +127.0.7.37 +127.0.7.38 +127.0.7.39 +127.0.7.40 +127.0.7.41 +127.0.7.42 +127.0.7.43 +127.0.7.44 +127.0.7.45 +127.0.7.46 +127.0.7.47 +127.0.7.48 +127.0.7.49 +127.0.7.50 +127.0.7.51 +127.0.7.52 +127.0.7.53 +127.0.7.54 +127.0.7.55 +127.0.7.56 +127.0.7.57 +127.0.7.58 +127.0.7.59 +127.0.7.60 +127.0.7.61 +127.0.7.62 +127.0.7.63 +127.0.7.64 +127.0.7.65 +127.0.7.66 +127.0.7.67 +127.0.7.68 +127.0.7.69 +127.0.7.70 +127.0.7.71 +127.0.7.72 +127.0.7.73 +127.0.7.74 +127.0.7.75 +127.0.7.76 +127.0.7.77 +127.0.7.78 +127.0.7.79 +127.0.7.80 +127.0.7.81 +127.0.7.82 +127.0.7.83 +127.0.7.84 +127.0.7.85 +127.0.7.86 +127.0.7.87 +127.0.7.88 +127.0.7.89 +127.0.7.90 +127.0.7.91 +127.0.7.92 +127.0.7.93 +127.0.7.94 +127.0.7.95 +127.0.7.96 +127.0.7.97 +127.0.7.98 +127.0.7.99 +127.0.7.100 +127.0.7.101 +127.0.7.102 +127.0.7.103 +127.0.7.104 +127.0.7.105 +127.0.7.106 +127.0.7.107 +127.0.7.108 +127.0.7.109 +127.0.7.110 +127.0.7.111 +127.0.7.112 +127.0.7.113 +127.0.7.114 +127.0.7.115 +127.0.7.116 +127.0.7.117 +127.0.7.118 +127.0.7.119 +127.0.7.120 +127.0.7.121 +127.0.7.122 +127.0.7.123 +127.0.7.124 +127.0.7.125 +127.0.7.126 +127.0.7.127 +127.0.7.128 +127.0.7.129 +127.0.7.130 +127.0.7.131 +127.0.7.132 +127.0.7.133 +127.0.7.134 +127.0.7.135 +127.0.7.136 +127.0.7.137 +127.0.7.138 +127.0.7.139 +127.0.7.140 +127.0.7.141 +127.0.7.142 +127.0.7.143 +127.0.7.144 +127.0.7.145 +127.0.7.146 +127.0.7.147 +127.0.7.148 +127.0.7.149 +127.0.7.150 +127.0.7.151 +127.0.7.152 +127.0.7.153 +127.0.7.154 +127.0.7.155 +127.0.7.156 +127.0.7.157 +127.0.7.158 +127.0.7.159 +127.0.7.160 +127.0.7.161 +127.0.7.162 +127.0.7.163 +127.0.7.164 +127.0.7.165 +127.0.7.166 +127.0.7.167 +127.0.7.168 +127.0.7.169 +127.0.7.170 +127.0.7.171 +127.0.7.172 +127.0.7.173 +127.0.7.174 +127.0.7.175 +127.0.7.176 +127.0.7.177 +127.0.7.178 +127.0.7.179 +127.0.7.180 +127.0.7.181 +127.0.7.182 +127.0.7.183 +127.0.7.184 +127.0.7.185 +127.0.7.186 +127.0.7.187 +127.0.7.188 +127.0.7.189 +127.0.7.190 +127.0.7.191 +127.0.7.192 +127.0.7.193 +127.0.7.194 +127.0.7.195 +127.0.7.196 +127.0.7.197 +127.0.7.198 +127.0.7.199 +127.0.7.200 +127.0.7.201 +127.0.7.202 +127.0.7.203 +127.0.7.204 +127.0.7.205 +127.0.7.206 +127.0.7.207 +127.0.7.208 +127.0.7.209 +127.0.7.210 +127.0.7.211 +127.0.7.212 +127.0.7.213 +127.0.7.214 +127.0.7.215 +127.0.7.216 +127.0.7.217 +127.0.7.218 +127.0.7.219 +127.0.7.220 +127.0.7.221 +127.0.7.222 +127.0.7.223 +127.0.7.224 +127.0.7.225 +127.0.7.226 +127.0.7.227 +127.0.7.228 +127.0.7.229 +127.0.7.230 +127.0.7.231 +127.0.7.232 +127.0.7.233 +127.0.7.234 +127.0.7.235 +127.0.7.236 +127.0.7.237 +127.0.7.238 +127.0.7.239 +127.0.7.240 +127.0.7.241 +127.0.7.242 +127.0.7.243 +127.0.7.244 +127.0.7.245 +127.0.7.246 +127.0.7.247 +127.0.7.248 +127.0.7.249 +127.0.7.250 +127.0.7.251 +127.0.7.252 +127.0.7.253 +127.0.7.254 +127.0.8.1 +127.0.8.2 +127.0.8.3 +127.0.8.4 +127.0.8.5 +127.0.8.6 +127.0.8.7 +127.0.8.8 +127.0.8.9 +127.0.8.10 +127.0.8.11 +127.0.8.12 +127.0.8.13 +127.0.8.14 +127.0.8.15 +127.0.8.16 +127.0.8.17 +127.0.8.18 +127.0.8.19 +127.0.8.20 +127.0.8.21 +127.0.8.22 +127.0.8.23 +127.0.8.24 +127.0.8.25 +127.0.8.26 +127.0.8.27 +127.0.8.28 +127.0.8.29 +127.0.8.30 +127.0.8.31 +127.0.8.32 +127.0.8.33 +127.0.8.34 +127.0.8.35 +127.0.8.36 +127.0.8.37 +127.0.8.38 +127.0.8.39 +127.0.8.40 +127.0.8.41 +127.0.8.42 +127.0.8.43 +127.0.8.44 +127.0.8.45 +127.0.8.46 +127.0.8.47 +127.0.8.48 +127.0.8.49 +127.0.8.50 +127.0.8.51 +127.0.8.52 +127.0.8.53 +127.0.8.54 +127.0.8.55 +127.0.8.56 +127.0.8.57 +127.0.8.58 +127.0.8.59 +127.0.8.60 +127.0.8.61 +127.0.8.62 +127.0.8.63 +127.0.8.64 +127.0.8.65 +127.0.8.66 +127.0.8.67 +127.0.8.68 +127.0.8.69 +127.0.8.70 +127.0.8.71 +127.0.8.72 +127.0.8.73 +127.0.8.74 +127.0.8.75 +127.0.8.76 +127.0.8.77 +127.0.8.78 +127.0.8.79 +127.0.8.80 +127.0.8.81 +127.0.8.82 +127.0.8.83 +127.0.8.84 +127.0.8.85 +127.0.8.86 +127.0.8.87 +127.0.8.88 +127.0.8.89 +127.0.8.90 +127.0.8.91 +127.0.8.92 +127.0.8.93 +127.0.8.94 +127.0.8.95 +127.0.8.96 +127.0.8.97 +127.0.8.98 +127.0.8.99 +127.0.8.100 +127.0.8.101 +127.0.8.102 +127.0.8.103 +127.0.8.104 +127.0.8.105 +127.0.8.106 +127.0.8.107 +127.0.8.108 +127.0.8.109 +127.0.8.110 +127.0.8.111 +127.0.8.112 +127.0.8.113 +127.0.8.114 +127.0.8.115 +127.0.8.116 +127.0.8.117 +127.0.8.118 +127.0.8.119 +127.0.8.120 +127.0.8.121 +127.0.8.122 +127.0.8.123 +127.0.8.124 +127.0.8.125 +127.0.8.126 +127.0.8.127 +127.0.8.128 +127.0.8.129 +127.0.8.130 +127.0.8.131 +127.0.8.132 +127.0.8.133 +127.0.8.134 +127.0.8.135 +127.0.8.136 +127.0.8.137 +127.0.8.138 +127.0.8.139 +127.0.8.140 +127.0.8.141 +127.0.8.142 +127.0.8.143 +127.0.8.144 +127.0.8.145 +127.0.8.146 +127.0.8.147 +127.0.8.148 +127.0.8.149 +127.0.8.150 +127.0.8.151 +127.0.8.152 +127.0.8.153 +127.0.8.154 +127.0.8.155 +127.0.8.156 +127.0.8.157 +127.0.8.158 +127.0.8.159 +127.0.8.160 +127.0.8.161 +127.0.8.162 +127.0.8.163 +127.0.8.164 +127.0.8.165 +127.0.8.166 +127.0.8.167 +127.0.8.168 +127.0.8.169 +127.0.8.170 +127.0.8.171 +127.0.8.172 +127.0.8.173 +127.0.8.174 +127.0.8.175 +127.0.8.176 +127.0.8.177 +127.0.8.178 +127.0.8.179 +127.0.8.180 +127.0.8.181 +127.0.8.182 +127.0.8.183 +127.0.8.184 +127.0.8.185 +127.0.8.186 +127.0.8.187 +127.0.8.188 +127.0.8.189 +127.0.8.190 +127.0.8.191 +127.0.8.192 +127.0.8.193 +127.0.8.194 +127.0.8.195 +127.0.8.196 +127.0.8.197 +127.0.8.198 +127.0.8.199 +127.0.8.200 +127.0.8.201 +127.0.8.202 +127.0.8.203 +127.0.8.204 +127.0.8.205 +127.0.8.206 +127.0.8.207 +127.0.8.208 +127.0.8.209 +127.0.8.210 +127.0.8.211 +127.0.8.212 +127.0.8.213 +127.0.8.214 +127.0.8.215 +127.0.8.216 +127.0.8.217 +127.0.8.218 +127.0.8.219 +127.0.8.220 +127.0.8.221 +127.0.8.222 +127.0.8.223 +127.0.8.224 +127.0.8.225 +127.0.8.226 +127.0.8.227 +127.0.8.228 +127.0.8.229 +127.0.8.230 +127.0.8.231 +127.0.8.232 +127.0.8.233 +127.0.8.234 +127.0.8.235 +127.0.8.236 +127.0.8.237 +127.0.8.238 +127.0.8.239 +127.0.8.240 +127.0.8.241 +127.0.8.242 +127.0.8.243 +127.0.8.244 +127.0.8.245 +127.0.8.246 +127.0.8.247 +127.0.8.248 +127.0.8.249 +127.0.8.250 +127.0.8.251 +127.0.8.252 +127.0.8.253 +127.0.8.254 +127.0.9.1 +127.0.9.2 +127.0.9.3 +127.0.9.4 +127.0.9.5 +127.0.9.6 +127.0.9.7 +127.0.9.8 +127.0.9.9 +127.0.9.10 +127.0.9.11 +127.0.9.12 +127.0.9.13 +127.0.9.14 +127.0.9.15 +127.0.9.16 +127.0.9.17 +127.0.9.18 +127.0.9.19 +127.0.9.20 +127.0.9.21 +127.0.9.22 +127.0.9.23 +127.0.9.24 +127.0.9.25 +127.0.9.26 +127.0.9.27 +127.0.9.28 +127.0.9.29 +127.0.9.30 +127.0.9.31 +127.0.9.32 +127.0.9.33 +127.0.9.34 +127.0.9.35 +127.0.9.36 +127.0.9.37 +127.0.9.38 +127.0.9.39 +127.0.9.40 +127.0.9.41 +127.0.9.42 +127.0.9.43 +127.0.9.44 +127.0.9.45 +127.0.9.46 +127.0.9.47 +127.0.9.48 +127.0.9.49 +127.0.9.50 +127.0.9.51 +127.0.9.52 +127.0.9.53 +127.0.9.54 +127.0.9.55 +127.0.9.56 +127.0.9.57 +127.0.9.58 +127.0.9.59 +127.0.9.60 +127.0.9.61 +127.0.9.62 +127.0.9.63 +127.0.9.64 +127.0.9.65 +127.0.9.66 +127.0.9.67 +127.0.9.68 +127.0.9.69 +127.0.9.70 +127.0.9.71 +127.0.9.72 +127.0.9.73 +127.0.9.74 +127.0.9.75 +127.0.9.76 +127.0.9.77 +127.0.9.78 +127.0.9.79 +127.0.9.80 +127.0.9.81 +127.0.9.82 +127.0.9.83 +127.0.9.84 +127.0.9.85 +127.0.9.86 +127.0.9.87 +127.0.9.88 +127.0.9.89 +127.0.9.90 +127.0.9.91 +127.0.9.92 +127.0.9.93 +127.0.9.94 +127.0.9.95 +127.0.9.96 +127.0.9.97 +127.0.9.98 +127.0.9.99 +127.0.9.100 +127.0.9.101 +127.0.9.102 +127.0.9.103 +127.0.9.104 +127.0.9.105 +127.0.9.106 +127.0.9.107 +127.0.9.108 +127.0.9.109 +127.0.9.110 +127.0.9.111 +127.0.9.112 +127.0.9.113 +127.0.9.114 +127.0.9.115 +127.0.9.116 +127.0.9.117 +127.0.9.118 +127.0.9.119 +127.0.9.120 +127.0.9.121 +127.0.9.122 +127.0.9.123 +127.0.9.124 +127.0.9.125 +127.0.9.126 +127.0.9.127 +127.0.9.128 +127.0.9.129 +127.0.9.130 +127.0.9.131 +127.0.9.132 +127.0.9.133 +127.0.9.134 +127.0.9.135 +127.0.9.136 +127.0.9.137 +127.0.9.138 +127.0.9.139 +127.0.9.140 +127.0.9.141 +127.0.9.142 +127.0.9.143 +127.0.9.144 +127.0.9.145 +127.0.9.146 +127.0.9.147 +127.0.9.148 +127.0.9.149 +127.0.9.150 +127.0.9.151 +127.0.9.152 +127.0.9.153 +127.0.9.154 +127.0.9.155 +127.0.9.156 +127.0.9.157 +127.0.9.158 +127.0.9.159 +127.0.9.160 +127.0.9.161 +127.0.9.162 +127.0.9.163 +127.0.9.164 +127.0.9.165 +127.0.9.166 +127.0.9.167 +127.0.9.168 +127.0.9.169 +127.0.9.170 +127.0.9.171 +127.0.9.172 +127.0.9.173 +127.0.9.174 +127.0.9.175 +127.0.9.176 +127.0.9.177 +127.0.9.178 +127.0.9.179 +127.0.9.180 +127.0.9.181 +127.0.9.182 +127.0.9.183 +127.0.9.184 +127.0.9.185 +127.0.9.186 +127.0.9.187 +127.0.9.188 +127.0.9.189 +127.0.9.190 +127.0.9.191 +127.0.9.192 +127.0.9.193 +127.0.9.194 +127.0.9.195 +127.0.9.196 +127.0.9.197 +127.0.9.198 +127.0.9.199 +127.0.9.200 +127.0.9.201 +127.0.9.202 +127.0.9.203 +127.0.9.204 +127.0.9.205 +127.0.9.206 +127.0.9.207 +127.0.9.208 +127.0.9.209 +127.0.9.210 +127.0.9.211 +127.0.9.212 +127.0.9.213 +127.0.9.214 +127.0.9.215 +127.0.9.216 +127.0.9.217 +127.0.9.218 +127.0.9.219 +127.0.9.220 +127.0.9.221 +127.0.9.222 +127.0.9.223 +127.0.9.224 +127.0.9.225 +127.0.9.226 +127.0.9.227 +127.0.9.228 +127.0.9.229 +127.0.9.230 +127.0.9.231 +127.0.9.232 +127.0.9.233 +127.0.9.234 +127.0.9.235 +127.0.9.236 +127.0.9.237 +127.0.9.238 +127.0.9.239 +127.0.9.240 +127.0.9.241 +127.0.9.242 +127.0.9.243 +127.0.9.244 +127.0.9.245 +127.0.9.246 +127.0.9.247 +127.0.9.248 +127.0.9.249 +127.0.9.250 +127.0.9.251 +127.0.9.252 +127.0.9.253 +127.0.9.254 diff --git a/v2/samples/inv_md b/v2/samples/inv_md new file mode 100644 index 0000000000..013bdc6966 --- /dev/null +++ b/v2/samples/inv_md @@ -0,0 +1,1270 @@ +127.0.0.1 +127.0.0.2 +127.0.0.3 +127.0.0.4 +127.0.0.5 +127.0.0.6 +127.0.0.7 +127.0.0.8 +127.0.0.9 +127.0.0.10 +127.0.0.11 +127.0.0.12 +127.0.0.13 +127.0.0.14 +127.0.0.15 +127.0.0.16 +127.0.0.17 +127.0.0.18 +127.0.0.19 +127.0.0.20 +127.0.0.21 +127.0.0.22 +127.0.0.23 +127.0.0.24 +127.0.0.25 +127.0.0.26 +127.0.0.27 +127.0.0.28 +127.0.0.29 +127.0.0.30 +127.0.0.31 +127.0.0.32 +127.0.0.33 +127.0.0.34 +127.0.0.35 +127.0.0.36 +127.0.0.37 +127.0.0.38 +127.0.0.39 +127.0.0.40 +127.0.0.41 +127.0.0.42 +127.0.0.43 +127.0.0.44 +127.0.0.45 +127.0.0.46 +127.0.0.47 +127.0.0.48 +127.0.0.49 +127.0.0.50 +127.0.0.51 +127.0.0.52 +127.0.0.53 +127.0.0.54 +127.0.0.55 +127.0.0.56 +127.0.0.57 +127.0.0.58 +127.0.0.59 +127.0.0.60 +127.0.0.61 +127.0.0.62 +127.0.0.63 +127.0.0.64 +127.0.0.65 +127.0.0.66 +127.0.0.67 +127.0.0.68 +127.0.0.69 +127.0.0.70 +127.0.0.71 +127.0.0.72 +127.0.0.73 +127.0.0.74 +127.0.0.75 +127.0.0.76 +127.0.0.77 +127.0.0.78 +127.0.0.79 +127.0.0.80 +127.0.0.81 +127.0.0.82 +127.0.0.83 +127.0.0.84 +127.0.0.85 +127.0.0.86 +127.0.0.87 +127.0.0.88 +127.0.0.89 +127.0.0.90 +127.0.0.91 +127.0.0.92 +127.0.0.93 +127.0.0.94 +127.0.0.95 +127.0.0.96 +127.0.0.97 +127.0.0.98 +127.0.0.99 +127.0.0.100 +127.0.0.101 +127.0.0.102 +127.0.0.103 +127.0.0.104 +127.0.0.105 +127.0.0.106 +127.0.0.107 +127.0.0.108 +127.0.0.109 +127.0.0.110 +127.0.0.111 +127.0.0.112 +127.0.0.113 +127.0.0.114 +127.0.0.115 +127.0.0.116 +127.0.0.117 +127.0.0.118 +127.0.0.119 +127.0.0.120 +127.0.0.121 +127.0.0.122 +127.0.0.123 +127.0.0.124 +127.0.0.125 +127.0.0.126 +127.0.0.127 +127.0.0.128 +127.0.0.129 +127.0.0.130 +127.0.0.131 +127.0.0.132 +127.0.0.133 +127.0.0.134 +127.0.0.135 +127.0.0.136 +127.0.0.137 +127.0.0.138 +127.0.0.139 +127.0.0.140 +127.0.0.141 +127.0.0.142 +127.0.0.143 +127.0.0.144 +127.0.0.145 +127.0.0.146 +127.0.0.147 +127.0.0.148 +127.0.0.149 +127.0.0.150 +127.0.0.151 +127.0.0.152 +127.0.0.153 +127.0.0.154 +127.0.0.155 +127.0.0.156 +127.0.0.157 +127.0.0.158 +127.0.0.159 +127.0.0.160 +127.0.0.161 +127.0.0.162 +127.0.0.163 +127.0.0.164 +127.0.0.165 +127.0.0.166 +127.0.0.167 +127.0.0.168 +127.0.0.169 +127.0.0.170 +127.0.0.171 +127.0.0.172 +127.0.0.173 +127.0.0.174 +127.0.0.175 +127.0.0.176 +127.0.0.177 +127.0.0.178 +127.0.0.179 +127.0.0.180 +127.0.0.181 +127.0.0.182 +127.0.0.183 +127.0.0.184 +127.0.0.185 +127.0.0.186 +127.0.0.187 +127.0.0.188 +127.0.0.189 +127.0.0.190 +127.0.0.191 +127.0.0.192 +127.0.0.193 +127.0.0.194 +127.0.0.195 +127.0.0.196 +127.0.0.197 +127.0.0.198 +127.0.0.199 +127.0.0.200 +127.0.0.201 +127.0.0.202 +127.0.0.203 +127.0.0.204 +127.0.0.205 +127.0.0.206 +127.0.0.207 +127.0.0.208 +127.0.0.209 +127.0.0.210 +127.0.0.211 +127.0.0.212 +127.0.0.213 +127.0.0.214 +127.0.0.215 +127.0.0.216 +127.0.0.217 +127.0.0.218 +127.0.0.219 +127.0.0.220 +127.0.0.221 +127.0.0.222 +127.0.0.223 +127.0.0.224 +127.0.0.225 +127.0.0.226 +127.0.0.227 +127.0.0.228 +127.0.0.229 +127.0.0.230 +127.0.0.231 +127.0.0.232 +127.0.0.233 +127.0.0.234 +127.0.0.235 +127.0.0.236 +127.0.0.237 +127.0.0.238 +127.0.0.239 +127.0.0.240 +127.0.0.241 +127.0.0.242 +127.0.0.243 +127.0.0.244 +127.0.0.245 +127.0.0.246 +127.0.0.247 +127.0.0.248 +127.0.0.249 +127.0.0.250 +127.0.0.251 +127.0.0.252 +127.0.0.253 +127.0.0.254 +127.0.1.1 +127.0.1.2 +127.0.1.3 +127.0.1.4 +127.0.1.5 +127.0.1.6 +127.0.1.7 +127.0.1.8 +127.0.1.9 +127.0.1.10 +127.0.1.11 +127.0.1.12 +127.0.1.13 +127.0.1.14 +127.0.1.15 +127.0.1.16 +127.0.1.17 +127.0.1.18 +127.0.1.19 +127.0.1.20 +127.0.1.21 +127.0.1.22 +127.0.1.23 +127.0.1.24 +127.0.1.25 +127.0.1.26 +127.0.1.27 +127.0.1.28 +127.0.1.29 +127.0.1.30 +127.0.1.31 +127.0.1.32 +127.0.1.33 +127.0.1.34 +127.0.1.35 +127.0.1.36 +127.0.1.37 +127.0.1.38 +127.0.1.39 +127.0.1.40 +127.0.1.41 +127.0.1.42 +127.0.1.43 +127.0.1.44 +127.0.1.45 +127.0.1.46 +127.0.1.47 +127.0.1.48 +127.0.1.49 +127.0.1.50 +127.0.1.51 +127.0.1.52 +127.0.1.53 +127.0.1.54 +127.0.1.55 +127.0.1.56 +127.0.1.57 +127.0.1.58 +127.0.1.59 +127.0.1.60 +127.0.1.61 +127.0.1.62 +127.0.1.63 +127.0.1.64 +127.0.1.65 +127.0.1.66 +127.0.1.67 +127.0.1.68 +127.0.1.69 +127.0.1.70 +127.0.1.71 +127.0.1.72 +127.0.1.73 +127.0.1.74 +127.0.1.75 +127.0.1.76 +127.0.1.77 +127.0.1.78 +127.0.1.79 +127.0.1.80 +127.0.1.81 +127.0.1.82 +127.0.1.83 +127.0.1.84 +127.0.1.85 +127.0.1.86 +127.0.1.87 +127.0.1.88 +127.0.1.89 +127.0.1.90 +127.0.1.91 +127.0.1.92 +127.0.1.93 +127.0.1.94 +127.0.1.95 +127.0.1.96 +127.0.1.97 +127.0.1.98 +127.0.1.99 +127.0.1.100 +127.0.1.101 +127.0.1.102 +127.0.1.103 +127.0.1.104 +127.0.1.105 +127.0.1.106 +127.0.1.107 +127.0.1.108 +127.0.1.109 +127.0.1.110 +127.0.1.111 +127.0.1.112 +127.0.1.113 +127.0.1.114 +127.0.1.115 +127.0.1.116 +127.0.1.117 +127.0.1.118 +127.0.1.119 +127.0.1.120 +127.0.1.121 +127.0.1.122 +127.0.1.123 +127.0.1.124 +127.0.1.125 +127.0.1.126 +127.0.1.127 +127.0.1.128 +127.0.1.129 +127.0.1.130 +127.0.1.131 +127.0.1.132 +127.0.1.133 +127.0.1.134 +127.0.1.135 +127.0.1.136 +127.0.1.137 +127.0.1.138 +127.0.1.139 +127.0.1.140 +127.0.1.141 +127.0.1.142 +127.0.1.143 +127.0.1.144 +127.0.1.145 +127.0.1.146 +127.0.1.147 +127.0.1.148 +127.0.1.149 +127.0.1.150 +127.0.1.151 +127.0.1.152 +127.0.1.153 +127.0.1.154 +127.0.1.155 +127.0.1.156 +127.0.1.157 +127.0.1.158 +127.0.1.159 +127.0.1.160 +127.0.1.161 +127.0.1.162 +127.0.1.163 +127.0.1.164 +127.0.1.165 +127.0.1.166 +127.0.1.167 +127.0.1.168 +127.0.1.169 +127.0.1.170 +127.0.1.171 +127.0.1.172 +127.0.1.173 +127.0.1.174 +127.0.1.175 +127.0.1.176 +127.0.1.177 +127.0.1.178 +127.0.1.179 +127.0.1.180 +127.0.1.181 +127.0.1.182 +127.0.1.183 +127.0.1.184 +127.0.1.185 +127.0.1.186 +127.0.1.187 +127.0.1.188 +127.0.1.189 +127.0.1.190 +127.0.1.191 +127.0.1.192 +127.0.1.193 +127.0.1.194 +127.0.1.195 +127.0.1.196 +127.0.1.197 +127.0.1.198 +127.0.1.199 +127.0.1.200 +127.0.1.201 +127.0.1.202 +127.0.1.203 +127.0.1.204 +127.0.1.205 +127.0.1.206 +127.0.1.207 +127.0.1.208 +127.0.1.209 +127.0.1.210 +127.0.1.211 +127.0.1.212 +127.0.1.213 +127.0.1.214 +127.0.1.215 +127.0.1.216 +127.0.1.217 +127.0.1.218 +127.0.1.219 +127.0.1.220 +127.0.1.221 +127.0.1.222 +127.0.1.223 +127.0.1.224 +127.0.1.225 +127.0.1.226 +127.0.1.227 +127.0.1.228 +127.0.1.229 +127.0.1.230 +127.0.1.231 +127.0.1.232 +127.0.1.233 +127.0.1.234 +127.0.1.235 +127.0.1.236 +127.0.1.237 +127.0.1.238 +127.0.1.239 +127.0.1.240 +127.0.1.241 +127.0.1.242 +127.0.1.243 +127.0.1.244 +127.0.1.245 +127.0.1.246 +127.0.1.247 +127.0.1.248 +127.0.1.249 +127.0.1.250 +127.0.1.251 +127.0.1.252 +127.0.1.253 +127.0.1.254 +127.0.2.1 +127.0.2.2 +127.0.2.3 +127.0.2.4 +127.0.2.5 +127.0.2.6 +127.0.2.7 +127.0.2.8 +127.0.2.9 +127.0.2.10 +127.0.2.11 +127.0.2.12 +127.0.2.13 +127.0.2.14 +127.0.2.15 +127.0.2.16 +127.0.2.17 +127.0.2.18 +127.0.2.19 +127.0.2.20 +127.0.2.21 +127.0.2.22 +127.0.2.23 +127.0.2.24 +127.0.2.25 +127.0.2.26 +127.0.2.27 +127.0.2.28 +127.0.2.29 +127.0.2.30 +127.0.2.31 +127.0.2.32 +127.0.2.33 +127.0.2.34 +127.0.2.35 +127.0.2.36 +127.0.2.37 +127.0.2.38 +127.0.2.39 +127.0.2.40 +127.0.2.41 +127.0.2.42 +127.0.2.43 +127.0.2.44 +127.0.2.45 +127.0.2.46 +127.0.2.47 +127.0.2.48 +127.0.2.49 +127.0.2.50 +127.0.2.51 +127.0.2.52 +127.0.2.53 +127.0.2.54 +127.0.2.55 +127.0.2.56 +127.0.2.57 +127.0.2.58 +127.0.2.59 +127.0.2.60 +127.0.2.61 +127.0.2.62 +127.0.2.63 +127.0.2.64 +127.0.2.65 +127.0.2.66 +127.0.2.67 +127.0.2.68 +127.0.2.69 +127.0.2.70 +127.0.2.71 +127.0.2.72 +127.0.2.73 +127.0.2.74 +127.0.2.75 +127.0.2.76 +127.0.2.77 +127.0.2.78 +127.0.2.79 +127.0.2.80 +127.0.2.81 +127.0.2.82 +127.0.2.83 +127.0.2.84 +127.0.2.85 +127.0.2.86 +127.0.2.87 +127.0.2.88 +127.0.2.89 +127.0.2.90 +127.0.2.91 +127.0.2.92 +127.0.2.93 +127.0.2.94 +127.0.2.95 +127.0.2.96 +127.0.2.97 +127.0.2.98 +127.0.2.99 +127.0.2.100 +127.0.2.101 +127.0.2.102 +127.0.2.103 +127.0.2.104 +127.0.2.105 +127.0.2.106 +127.0.2.107 +127.0.2.108 +127.0.2.109 +127.0.2.110 +127.0.2.111 +127.0.2.112 +127.0.2.113 +127.0.2.114 +127.0.2.115 +127.0.2.116 +127.0.2.117 +127.0.2.118 +127.0.2.119 +127.0.2.120 +127.0.2.121 +127.0.2.122 +127.0.2.123 +127.0.2.124 +127.0.2.125 +127.0.2.126 +127.0.2.127 +127.0.2.128 +127.0.2.129 +127.0.2.130 +127.0.2.131 +127.0.2.132 +127.0.2.133 +127.0.2.134 +127.0.2.135 +127.0.2.136 +127.0.2.137 +127.0.2.138 +127.0.2.139 +127.0.2.140 +127.0.2.141 +127.0.2.142 +127.0.2.143 +127.0.2.144 +127.0.2.145 +127.0.2.146 +127.0.2.147 +127.0.2.148 +127.0.2.149 +127.0.2.150 +127.0.2.151 +127.0.2.152 +127.0.2.153 +127.0.2.154 +127.0.2.155 +127.0.2.156 +127.0.2.157 +127.0.2.158 +127.0.2.159 +127.0.2.160 +127.0.2.161 +127.0.2.162 +127.0.2.163 +127.0.2.164 +127.0.2.165 +127.0.2.166 +127.0.2.167 +127.0.2.168 +127.0.2.169 +127.0.2.170 +127.0.2.171 +127.0.2.172 +127.0.2.173 +127.0.2.174 +127.0.2.175 +127.0.2.176 +127.0.2.177 +127.0.2.178 +127.0.2.179 +127.0.2.180 +127.0.2.181 +127.0.2.182 +127.0.2.183 +127.0.2.184 +127.0.2.185 +127.0.2.186 +127.0.2.187 +127.0.2.188 +127.0.2.189 +127.0.2.190 +127.0.2.191 +127.0.2.192 +127.0.2.193 +127.0.2.194 +127.0.2.195 +127.0.2.196 +127.0.2.197 +127.0.2.198 +127.0.2.199 +127.0.2.200 +127.0.2.201 +127.0.2.202 +127.0.2.203 +127.0.2.204 +127.0.2.205 +127.0.2.206 +127.0.2.207 +127.0.2.208 +127.0.2.209 +127.0.2.210 +127.0.2.211 +127.0.2.212 +127.0.2.213 +127.0.2.214 +127.0.2.215 +127.0.2.216 +127.0.2.217 +127.0.2.218 +127.0.2.219 +127.0.2.220 +127.0.2.221 +127.0.2.222 +127.0.2.223 +127.0.2.224 +127.0.2.225 +127.0.2.226 +127.0.2.227 +127.0.2.228 +127.0.2.229 +127.0.2.230 +127.0.2.231 +127.0.2.232 +127.0.2.233 +127.0.2.234 +127.0.2.235 +127.0.2.236 +127.0.2.237 +127.0.2.238 +127.0.2.239 +127.0.2.240 +127.0.2.241 +127.0.2.242 +127.0.2.243 +127.0.2.244 +127.0.2.245 +127.0.2.246 +127.0.2.247 +127.0.2.248 +127.0.2.249 +127.0.2.250 +127.0.2.251 +127.0.2.252 +127.0.2.253 +127.0.2.254 +127.0.3.1 +127.0.3.2 +127.0.3.3 +127.0.3.4 +127.0.3.5 +127.0.3.6 +127.0.3.7 +127.0.3.8 +127.0.3.9 +127.0.3.10 +127.0.3.11 +127.0.3.12 +127.0.3.13 +127.0.3.14 +127.0.3.15 +127.0.3.16 +127.0.3.17 +127.0.3.18 +127.0.3.19 +127.0.3.20 +127.0.3.21 +127.0.3.22 +127.0.3.23 +127.0.3.24 +127.0.3.25 +127.0.3.26 +127.0.3.27 +127.0.3.28 +127.0.3.29 +127.0.3.30 +127.0.3.31 +127.0.3.32 +127.0.3.33 +127.0.3.34 +127.0.3.35 +127.0.3.36 +127.0.3.37 +127.0.3.38 +127.0.3.39 +127.0.3.40 +127.0.3.41 +127.0.3.42 +127.0.3.43 +127.0.3.44 +127.0.3.45 +127.0.3.46 +127.0.3.47 +127.0.3.48 +127.0.3.49 +127.0.3.50 +127.0.3.51 +127.0.3.52 +127.0.3.53 +127.0.3.54 +127.0.3.55 +127.0.3.56 +127.0.3.57 +127.0.3.58 +127.0.3.59 +127.0.3.60 +127.0.3.61 +127.0.3.62 +127.0.3.63 +127.0.3.64 +127.0.3.65 +127.0.3.66 +127.0.3.67 +127.0.3.68 +127.0.3.69 +127.0.3.70 +127.0.3.71 +127.0.3.72 +127.0.3.73 +127.0.3.74 +127.0.3.75 +127.0.3.76 +127.0.3.77 +127.0.3.78 +127.0.3.79 +127.0.3.80 +127.0.3.81 +127.0.3.82 +127.0.3.83 +127.0.3.84 +127.0.3.85 +127.0.3.86 +127.0.3.87 +127.0.3.88 +127.0.3.89 +127.0.3.90 +127.0.3.91 +127.0.3.92 +127.0.3.93 +127.0.3.94 +127.0.3.95 +127.0.3.96 +127.0.3.97 +127.0.3.98 +127.0.3.99 +127.0.3.100 +127.0.3.101 +127.0.3.102 +127.0.3.103 +127.0.3.104 +127.0.3.105 +127.0.3.106 +127.0.3.107 +127.0.3.108 +127.0.3.109 +127.0.3.110 +127.0.3.111 +127.0.3.112 +127.0.3.113 +127.0.3.114 +127.0.3.115 +127.0.3.116 +127.0.3.117 +127.0.3.118 +127.0.3.119 +127.0.3.120 +127.0.3.121 +127.0.3.122 +127.0.3.123 +127.0.3.124 +127.0.3.125 +127.0.3.126 +127.0.3.127 +127.0.3.128 +127.0.3.129 +127.0.3.130 +127.0.3.131 +127.0.3.132 +127.0.3.133 +127.0.3.134 +127.0.3.135 +127.0.3.136 +127.0.3.137 +127.0.3.138 +127.0.3.139 +127.0.3.140 +127.0.3.141 +127.0.3.142 +127.0.3.143 +127.0.3.144 +127.0.3.145 +127.0.3.146 +127.0.3.147 +127.0.3.148 +127.0.3.149 +127.0.3.150 +127.0.3.151 +127.0.3.152 +127.0.3.153 +127.0.3.154 +127.0.3.155 +127.0.3.156 +127.0.3.157 +127.0.3.158 +127.0.3.159 +127.0.3.160 +127.0.3.161 +127.0.3.162 +127.0.3.163 +127.0.3.164 +127.0.3.165 +127.0.3.166 +127.0.3.167 +127.0.3.168 +127.0.3.169 +127.0.3.170 +127.0.3.171 +127.0.3.172 +127.0.3.173 +127.0.3.174 +127.0.3.175 +127.0.3.176 +127.0.3.177 +127.0.3.178 +127.0.3.179 +127.0.3.180 +127.0.3.181 +127.0.3.182 +127.0.3.183 +127.0.3.184 +127.0.3.185 +127.0.3.186 +127.0.3.187 +127.0.3.188 +127.0.3.189 +127.0.3.190 +127.0.3.191 +127.0.3.192 +127.0.3.193 +127.0.3.194 +127.0.3.195 +127.0.3.196 +127.0.3.197 +127.0.3.198 +127.0.3.199 +127.0.3.200 +127.0.3.201 +127.0.3.202 +127.0.3.203 +127.0.3.204 +127.0.3.205 +127.0.3.206 +127.0.3.207 +127.0.3.208 +127.0.3.209 +127.0.3.210 +127.0.3.211 +127.0.3.212 +127.0.3.213 +127.0.3.214 +127.0.3.215 +127.0.3.216 +127.0.3.217 +127.0.3.218 +127.0.3.219 +127.0.3.220 +127.0.3.221 +127.0.3.222 +127.0.3.223 +127.0.3.224 +127.0.3.225 +127.0.3.226 +127.0.3.227 +127.0.3.228 +127.0.3.229 +127.0.3.230 +127.0.3.231 +127.0.3.232 +127.0.3.233 +127.0.3.234 +127.0.3.235 +127.0.3.236 +127.0.3.237 +127.0.3.238 +127.0.3.239 +127.0.3.240 +127.0.3.241 +127.0.3.242 +127.0.3.243 +127.0.3.244 +127.0.3.245 +127.0.3.246 +127.0.3.247 +127.0.3.248 +127.0.3.249 +127.0.3.250 +127.0.3.251 +127.0.3.252 +127.0.3.253 +127.0.3.254 +127.0.4.1 +127.0.4.2 +127.0.4.3 +127.0.4.4 +127.0.4.5 +127.0.4.6 +127.0.4.7 +127.0.4.8 +127.0.4.9 +127.0.4.10 +127.0.4.11 +127.0.4.12 +127.0.4.13 +127.0.4.14 +127.0.4.15 +127.0.4.16 +127.0.4.17 +127.0.4.18 +127.0.4.19 +127.0.4.20 +127.0.4.21 +127.0.4.22 +127.0.4.23 +127.0.4.24 +127.0.4.25 +127.0.4.26 +127.0.4.27 +127.0.4.28 +127.0.4.29 +127.0.4.30 +127.0.4.31 +127.0.4.32 +127.0.4.33 +127.0.4.34 +127.0.4.35 +127.0.4.36 +127.0.4.37 +127.0.4.38 +127.0.4.39 +127.0.4.40 +127.0.4.41 +127.0.4.42 +127.0.4.43 +127.0.4.44 +127.0.4.45 +127.0.4.46 +127.0.4.47 +127.0.4.48 +127.0.4.49 +127.0.4.50 +127.0.4.51 +127.0.4.52 +127.0.4.53 +127.0.4.54 +127.0.4.55 +127.0.4.56 +127.0.4.57 +127.0.4.58 +127.0.4.59 +127.0.4.60 +127.0.4.61 +127.0.4.62 +127.0.4.63 +127.0.4.64 +127.0.4.65 +127.0.4.66 +127.0.4.67 +127.0.4.68 +127.0.4.69 +127.0.4.70 +127.0.4.71 +127.0.4.72 +127.0.4.73 +127.0.4.74 +127.0.4.75 +127.0.4.76 +127.0.4.77 +127.0.4.78 +127.0.4.79 +127.0.4.80 +127.0.4.81 +127.0.4.82 +127.0.4.83 +127.0.4.84 +127.0.4.85 +127.0.4.86 +127.0.4.87 +127.0.4.88 +127.0.4.89 +127.0.4.90 +127.0.4.91 +127.0.4.92 +127.0.4.93 +127.0.4.94 +127.0.4.95 +127.0.4.96 +127.0.4.97 +127.0.4.98 +127.0.4.99 +127.0.4.100 +127.0.4.101 +127.0.4.102 +127.0.4.103 +127.0.4.104 +127.0.4.105 +127.0.4.106 +127.0.4.107 +127.0.4.108 +127.0.4.109 +127.0.4.110 +127.0.4.111 +127.0.4.112 +127.0.4.113 +127.0.4.114 +127.0.4.115 +127.0.4.116 +127.0.4.117 +127.0.4.118 +127.0.4.119 +127.0.4.120 +127.0.4.121 +127.0.4.122 +127.0.4.123 +127.0.4.124 +127.0.4.125 +127.0.4.126 +127.0.4.127 +127.0.4.128 +127.0.4.129 +127.0.4.130 +127.0.4.131 +127.0.4.132 +127.0.4.133 +127.0.4.134 +127.0.4.135 +127.0.4.136 +127.0.4.137 +127.0.4.138 +127.0.4.139 +127.0.4.140 +127.0.4.141 +127.0.4.142 +127.0.4.143 +127.0.4.144 +127.0.4.145 +127.0.4.146 +127.0.4.147 +127.0.4.148 +127.0.4.149 +127.0.4.150 +127.0.4.151 +127.0.4.152 +127.0.4.153 +127.0.4.154 +127.0.4.155 +127.0.4.156 +127.0.4.157 +127.0.4.158 +127.0.4.159 +127.0.4.160 +127.0.4.161 +127.0.4.162 +127.0.4.163 +127.0.4.164 +127.0.4.165 +127.0.4.166 +127.0.4.167 +127.0.4.168 +127.0.4.169 +127.0.4.170 +127.0.4.171 +127.0.4.172 +127.0.4.173 +127.0.4.174 +127.0.4.175 +127.0.4.176 +127.0.4.177 +127.0.4.178 +127.0.4.179 +127.0.4.180 +127.0.4.181 +127.0.4.182 +127.0.4.183 +127.0.4.184 +127.0.4.185 +127.0.4.186 +127.0.4.187 +127.0.4.188 +127.0.4.189 +127.0.4.190 +127.0.4.191 +127.0.4.192 +127.0.4.193 +127.0.4.194 +127.0.4.195 +127.0.4.196 +127.0.4.197 +127.0.4.198 +127.0.4.199 +127.0.4.200 +127.0.4.201 +127.0.4.202 +127.0.4.203 +127.0.4.204 +127.0.4.205 +127.0.4.206 +127.0.4.207 +127.0.4.208 +127.0.4.209 +127.0.4.210 +127.0.4.211 +127.0.4.212 +127.0.4.213 +127.0.4.214 +127.0.4.215 +127.0.4.216 +127.0.4.217 +127.0.4.218 +127.0.4.219 +127.0.4.220 +127.0.4.221 +127.0.4.222 +127.0.4.223 +127.0.4.224 +127.0.4.225 +127.0.4.226 +127.0.4.227 +127.0.4.228 +127.0.4.229 +127.0.4.230 +127.0.4.231 +127.0.4.232 +127.0.4.233 +127.0.4.234 +127.0.4.235 +127.0.4.236 +127.0.4.237 +127.0.4.238 +127.0.4.239 +127.0.4.240 +127.0.4.241 +127.0.4.242 +127.0.4.243 +127.0.4.244 +127.0.4.245 +127.0.4.246 +127.0.4.247 +127.0.4.248 +127.0.4.249 +127.0.4.250 +127.0.4.251 +127.0.4.252 +127.0.4.253 +127.0.4.254 diff --git a/v2/samples/inv_sm b/v2/samples/inv_sm new file mode 100644 index 0000000000..dafa73d870 --- /dev/null +++ b/v2/samples/inv_sm @@ -0,0 +1,254 @@ +127.0.0.1 +127.0.0.2 +127.0.0.3 +127.0.0.4 +127.0.0.5 +127.0.0.6 +127.0.0.7 +127.0.0.8 +127.0.0.9 +127.0.0.10 +127.0.0.11 +127.0.0.12 +127.0.0.13 +127.0.0.14 +127.0.0.15 +127.0.0.16 +127.0.0.17 +127.0.0.18 +127.0.0.19 +127.0.0.20 +127.0.0.21 +127.0.0.22 +127.0.0.23 +127.0.0.24 +127.0.0.25 +127.0.0.26 +127.0.0.27 +127.0.0.28 +127.0.0.29 +127.0.0.30 +127.0.0.31 +127.0.0.32 +127.0.0.33 +127.0.0.34 +127.0.0.35 +127.0.0.36 +127.0.0.37 +127.0.0.38 +127.0.0.39 +127.0.0.40 +127.0.0.41 +127.0.0.42 +127.0.0.43 +127.0.0.44 +127.0.0.45 +127.0.0.46 +127.0.0.47 +127.0.0.48 +127.0.0.49 +127.0.0.50 +127.0.0.51 +127.0.0.52 +127.0.0.53 +127.0.0.54 +127.0.0.55 +127.0.0.56 +127.0.0.57 +127.0.0.58 +127.0.0.59 +127.0.0.60 +127.0.0.61 +127.0.0.62 +127.0.0.63 +127.0.0.64 +127.0.0.65 +127.0.0.66 +127.0.0.67 +127.0.0.68 +127.0.0.69 +127.0.0.70 +127.0.0.71 +127.0.0.72 +127.0.0.73 +127.0.0.74 +127.0.0.75 +127.0.0.76 +127.0.0.77 +127.0.0.78 +127.0.0.79 +127.0.0.80 +127.0.0.81 +127.0.0.82 +127.0.0.83 +127.0.0.84 +127.0.0.85 +127.0.0.86 +127.0.0.87 +127.0.0.88 +127.0.0.89 +127.0.0.90 +127.0.0.91 +127.0.0.92 +127.0.0.93 +127.0.0.94 +127.0.0.95 +127.0.0.96 +127.0.0.97 +127.0.0.98 +127.0.0.99 +127.0.0.100 +127.0.0.101 +127.0.0.102 +127.0.0.103 +127.0.0.104 +127.0.0.105 +127.0.0.106 +127.0.0.107 +127.0.0.108 +127.0.0.109 +127.0.0.110 +127.0.0.111 +127.0.0.112 +127.0.0.113 +127.0.0.114 +127.0.0.115 +127.0.0.116 +127.0.0.117 +127.0.0.118 +127.0.0.119 +127.0.0.120 +127.0.0.121 +127.0.0.122 +127.0.0.123 +127.0.0.124 +127.0.0.125 +127.0.0.126 +127.0.0.127 +127.0.0.128 +127.0.0.129 +127.0.0.130 +127.0.0.131 +127.0.0.132 +127.0.0.133 +127.0.0.134 +127.0.0.135 +127.0.0.136 +127.0.0.137 +127.0.0.138 +127.0.0.139 +127.0.0.140 +127.0.0.141 +127.0.0.142 +127.0.0.143 +127.0.0.144 +127.0.0.145 +127.0.0.146 +127.0.0.147 +127.0.0.148 +127.0.0.149 +127.0.0.150 +127.0.0.151 +127.0.0.152 +127.0.0.153 +127.0.0.154 +127.0.0.155 +127.0.0.156 +127.0.0.157 +127.0.0.158 +127.0.0.159 +127.0.0.160 +127.0.0.161 +127.0.0.162 +127.0.0.163 +127.0.0.164 +127.0.0.165 +127.0.0.166 +127.0.0.167 +127.0.0.168 +127.0.0.169 +127.0.0.170 +127.0.0.171 +127.0.0.172 +127.0.0.173 +127.0.0.174 +127.0.0.175 +127.0.0.176 +127.0.0.177 +127.0.0.178 +127.0.0.179 +127.0.0.180 +127.0.0.181 +127.0.0.182 +127.0.0.183 +127.0.0.184 +127.0.0.185 +127.0.0.186 +127.0.0.187 +127.0.0.188 +127.0.0.189 +127.0.0.190 +127.0.0.191 +127.0.0.192 +127.0.0.193 +127.0.0.194 +127.0.0.195 +127.0.0.196 +127.0.0.197 +127.0.0.198 +127.0.0.199 +127.0.0.200 +127.0.0.201 +127.0.0.202 +127.0.0.203 +127.0.0.204 +127.0.0.205 +127.0.0.206 +127.0.0.207 +127.0.0.208 +127.0.0.209 +127.0.0.210 +127.0.0.211 +127.0.0.212 +127.0.0.213 +127.0.0.214 +127.0.0.215 +127.0.0.216 +127.0.0.217 +127.0.0.218 +127.0.0.219 +127.0.0.220 +127.0.0.221 +127.0.0.222 +127.0.0.223 +127.0.0.224 +127.0.0.225 +127.0.0.226 +127.0.0.227 +127.0.0.228 +127.0.0.229 +127.0.0.230 +127.0.0.231 +127.0.0.232 +127.0.0.233 +127.0.0.234 +127.0.0.235 +127.0.0.236 +127.0.0.237 +127.0.0.238 +127.0.0.239 +127.0.0.240 +127.0.0.241 +127.0.0.242 +127.0.0.243 +127.0.0.244 +127.0.0.245 +127.0.0.246 +127.0.0.247 +127.0.0.248 +127.0.0.249 +127.0.0.250 +127.0.0.251 +127.0.0.252 +127.0.0.253 +127.0.0.254 diff --git a/v2/samples/lookup_file.yml b/v2/samples/lookup_file.yml new file mode 100644 index 0000000000..15cec9d294 --- /dev/null +++ b/v2/samples/lookup_file.yml @@ -0,0 +1,5 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="the pubkey is {{lookup('file', '~/.ssh/id_rsa.pub')}}" diff --git a/v2/samples/lookup_password.yml b/v2/samples/lookup_password.yml new file mode 100644 index 0000000000..07bc71358b --- /dev/null +++ b/v2/samples/lookup_password.yml @@ -0,0 +1,7 @@ +- hosts: localhost + gather_facts: no + #vars: + # my_password: "{{ lookup('password', '/tmp/test_lookup_password length=15') }}" + tasks: + #- debug: msg="the password is {{my_password}}" + - debug: msg="the password is {{ lookup('password', '/tmp/test_lookup_password length=15') }}" diff --git a/v2/samples/lookup_pipe.py b/v2/samples/lookup_pipe.py new file mode 100644 index 0000000000..4430c76fc5 --- /dev/null +++ b/v2/samples/lookup_pipe.py @@ -0,0 +1,4 @@ +- hosts: localhost + gather_facts: no + tasks: + - debug: msg="the date is {{ lookup('pipe', 'date') }}" diff --git a/v2/samples/lookup_template.yml b/v2/samples/lookup_template.yml new file mode 100644 index 0000000000..8fdd981b9f --- /dev/null +++ b/v2/samples/lookup_template.yml @@ -0,0 +1,7 @@ +- hosts: localhost + gather_facts: no + vars: + my_var: "Bazinga!" + tasks: + - debug: msg="the rendered template is {{ lookup('template', 'template.j2') }}" + diff --git a/v2/samples/multi.py b/v2/samples/multi.py new file mode 100644 index 0000000000..ca4c8b68f7 --- /dev/null +++ b/v2/samples/multi.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +import time +import Queue +import traceback +from multiprocessing import Process, Manager, Pipe, RLock + +from ansible.playbook.play import Play +from ansible.playbook.task import Task +from ansible.utils.debug import debug + +NUM_WORKERS = 50 +NUM_HOSTS = 2500 +NUM_TASKS = 1 + +class Foo: + def __init__(self, i, j): + self._foo = "FOO_%05d_%05d" % (i, j) + + def __repr__(self): + return self._foo + + def __getstate__(self): + debug("pickling %s" % self._foo) + return dict(foo=self._foo) + + def __setstate__(self, data): + debug("unpickling...") + self._foo = data.get('foo', "BAD PICKLE!") + debug("unpickled %s" % self._foo) + +def results(pipe, workers): + cur_worker = 0 + def _read_worker_result(cur_worker): + result = None + starting_point = cur_worker + while True: + (worker_prc, main_pipe, res_pipe) = workers[cur_worker] + cur_worker += 1 + if cur_worker >= len(workers): + cur_worker = 0 + + if res_pipe[1].poll(0.01): + debug("worker %d has data to read" % cur_worker) + result = res_pipe[1].recv() + debug("got a result from worker %d: %s" % (cur_worker, result)) + break + + if cur_worker == starting_point: + break + + return (result, cur_worker) + + while True: + result = None + try: + (result, cur_worker) = _read_worker_result(cur_worker) + if result is None: + time.sleep(0.01) + continue + pipe.send(result) + except (IOError, EOFError, KeyboardInterrupt), e: + debug("got a breaking error: %s" % e) + break + except Exception, e: + debug("EXCEPTION DURING RESULTS PROCESSING: %s" % e) + traceback.print_exc() + break + +def worker(main_pipe, res_pipe): + while True: + foo = None + try: + if main_pipe.poll(0.01): + foo = main_pipe.recv() + time.sleep(0.07) + res_pipe.send(foo) + else: + time.sleep(0.01) + except (IOError, EOFError, KeyboardInterrupt), e: + debug("got a breaking error: %s" % e) + break + except Exception, e: + debug("EXCEPTION DURING WORKER PROCESSING: %s" % e) + traceback.print_exc() + break + +workers = [] +for i in range(NUM_WORKERS): + (main_p1, main_p2) = Pipe() + (res_p1, res_p2) = Pipe() + worker_p = Process(target=worker, args=(main_p2, res_p1)) + worker_p.start() + workers.append((worker_p, (main_p1, main_p2), (res_p1, res_p2))) + +in_p, out_p = Pipe() +res_p = Process(target=results, args=(in_p, workers)) +res_p.start() + +def send_data(obj): + global cur_worker + global workers + global pending_results + + (w_proc, main_pipe, res_pipe) = workers[cur_worker] + cur_worker += 1 + if cur_worker >= len(workers): + cur_worker = 0 + + pending_results += 1 + main_pipe[0].send(obj) + +def _process_pending_results(): + global out_p + global pending_results + + try: + #p_lock.acquire() + while out_p.poll(0.01): + result = out_p.recv() + debug("got final result: %s" % (result,)) + pending_results -= 1 + finally: + #p_lock.release() + pass + +def _wait_on_pending_results(): + global pending_results + while pending_results > 0: + debug("waiting for pending results (%d left)" % pending_results) + _process_pending_results() + time.sleep(0.01) + + +debug("starting") +cur_worker = 0 +pending_results = 0 + +sample_play = Play() +for i in range(NUM_TASKS): + for j in range(NUM_HOSTS): + debug("queuing %d, %d" % (i, j)) + send_data(Task().load(dict(name="task %d %d" % (i,j), ping=""), sample_play)) + debug("done queuing %d, %d" % (i, j)) + _process_pending_results() + debug("waiting for the results to drain...") + _wait_on_pending_results() + +in_p.close() +out_p.close() +res_p.terminate() + +for (w_p, main_pipe, res_pipe) in workers: + res_pipe[1].close() + res_pipe[0].close() + main_pipe[1].close() + main_pipe[0].close() + w_p.terminate() + +debug("done") diff --git a/v2/samples/multi_queues.py b/v2/samples/multi_queues.py new file mode 100644 index 0000000000..8eb8036607 --- /dev/null +++ b/v2/samples/multi_queues.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +import sys +import time +import Queue +import traceback +import multiprocessing + +from ansible.inventory import Inventory +from ansible.inventory.host import Host +from ansible.playbook.play import Play +from ansible.playbook.task import Task +from ansible.executor.connection_info import ConnectionInformation +from ansible.executor.task_executor import TaskExecutor +from ansible.executor.task_result import TaskResult +from ansible.parsing import DataLoader +from ansible.vars import VariableManager + +from ansible.utils.debug import debug + +NUM_WORKERS = 20 +NUM_HOSTS = 1778 +NUM_TASKS = 1 + +def results(final_q, workers): + cur_worker = 0 + def _read_worker_result(cur_worker): + result = None + starting_point = cur_worker + while True: + (worker_prc, main_q, res_q) = workers[cur_worker] + cur_worker += 1 + if cur_worker >= len(workers): + cur_worker = 0 + + try: + if not res_q.empty(): + debug("worker %d has data to read" % cur_worker) + result = res_q.get() + debug("got a result from worker %d: %s" % (cur_worker, result)) + break + except: + pass + + if cur_worker == starting_point: + break + + return (result, cur_worker) + + while True: + result = None + try: + (result, cur_worker) = _read_worker_result(cur_worker) + if result is None: + time.sleep(0.01) + continue + final_q.put(result, block=False) + except (IOError, EOFError, KeyboardInterrupt), e: + debug("got a breaking error: %s" % e) + break + except Exception, e: + debug("EXCEPTION DURING RESULTS PROCESSING: %s" % e) + traceback.print_exc() + break + +def worker(main_q, res_q, loader): + while True: + task = None + try: + if not main_q.empty(): + (host, task, task_vars, conn_info) = main_q.get(block=False) + executor_result = TaskExecutor(host, task, task_vars, conn_info, loader).run() + debug("executor result: %s" % executor_result) + task_result = TaskResult(host, task, executor_result) + res_q.put(task_result) + else: + time.sleep(0.01) + except Queue.Empty: + pass + except (IOError, EOFError, KeyboardInterrupt), e: + debug("got a breaking error: %s" % e) + break + except Exception, e: + debug("EXCEPTION DURING WORKER PROCESSING: %s" % e) + traceback.print_exc() + break + +loader = DataLoader() + +workers = [] +for i in range(NUM_WORKERS): + main_q = multiprocessing.Queue() + res_q = multiprocessing.Queue() + worker_p = multiprocessing.Process(target=worker, args=(main_q, res_q, loader)) + worker_p.start() + workers.append((worker_p, main_q, res_q)) + +res_q = multiprocessing.Queue() +res_p = multiprocessing.Process(target=results, args=(res_q, workers)) +res_p.start() + +def send_data(obj): + global cur_worker + global workers + global pending_results + + (w_proc, main_q, wrkr_q) = workers[cur_worker] + cur_worker += 1 + if cur_worker >= len(workers): + cur_worker = 0 + + pending_results += 1 + main_q.put(obj, block=False) + +def _process_pending_results(): + global res_q + global pending_results + + while not res_q.empty(): + try: + result = res_q.get(block=False) + debug("got final result: %s" % (result,)) + pending_results -= 1 + except Queue.Empty: + pass + +def _wait_on_pending_results(): + global pending_results + while pending_results > 0: + debug("waiting for pending results (%d left)" % pending_results) + _process_pending_results() + time.sleep(0.01) + + +debug("starting") +cur_worker = 0 +pending_results = 0 + + +var_manager = VariableManager() + +debug("loading inventory") +inventory = Inventory(host_list='/tmp/med_inventory', loader=loader, variable_manager=var_manager) +hosts = inventory.get_hosts()[:] +debug("done loading inventory") + +ci = ConnectionInformation() +ci.connection = 'local' + +for i in range(NUM_TASKS): + #for j in range(NUM_HOSTS): + for h in hosts: + debug("queuing %s %d" % (h, i)) + #h = Host(name="host%06d" % j) + t = Task().load(dict(name="task %d" % (i,), debug="msg='hello from %s, %d'" % (h,i))) + #t = Task().load(dict(name="task %d" % (i,), ping="")) + #task_vars = var_manager.get_vars(loader=loader, host=h, task=t) + task_vars = dict() + new_t = t.copy() + new_t.post_validate(task_vars) + send_data((h, t, task_vars, ci)) + debug("done queuing %s %d" % (h, i)) + _process_pending_results() + debug("waiting for the results to drain...") + _wait_on_pending_results() + +res_q.close() +res_p.terminate() + +for (w_p, main_q, wrkr_q) in workers: + main_q.close() + wrkr_q.close() + w_p.terminate() + +debug("done") diff --git a/v2/samples/roles/test_role/tasks/main.yml b/v2/samples/roles/test_role/tasks/main.yml new file mode 100644 index 0000000000..ea0160bc16 --- /dev/null +++ b/v2/samples/roles/test_role/tasks/main.yml @@ -0,0 +1 @@ +- debug: msg="here we are in the role, foo={{foo}}" diff --git a/v2/samples/src b/v2/samples/src new file mode 100644 index 0000000000..5fc24f20db --- /dev/null +++ b/v2/samples/src @@ -0,0 +1,5 @@ +num_retries: 2 +frag 1 +frag 2 +frag 3 +vars_file_var: "this is in a vars file" diff --git a/v2/samples/template.j2 b/v2/samples/template.j2 new file mode 100644 index 0000000000..b564862cc7 --- /dev/null +++ b/v2/samples/template.j2 @@ -0,0 +1 @@ +the variable is {{my_var}} diff --git a/v2/samples/test_big_debug.yml b/v2/samples/test_big_debug.yml new file mode 100644 index 0000000000..09770f7eb4 --- /dev/null +++ b/v2/samples/test_big_debug.yml @@ -0,0 +1,4 @@ +- hosts: all + gather_facts: no + tasks: + - debug: msg="hi" diff --git a/v2/samples/test_big_ping.yml b/v2/samples/test_big_ping.yml new file mode 100644 index 0000000000..7f275cb8d5 --- /dev/null +++ b/v2/samples/test_big_ping.yml @@ -0,0 +1,5 @@ +- hosts: all + gather_facts: no + tasks: + #- debug: msg="hi" + - ping: diff --git a/v2/samples/test_fact_gather.yml b/v2/samples/test_fact_gather.yml new file mode 100644 index 0000000000..adf7f5df36 --- /dev/null +++ b/v2/samples/test_fact_gather.yml @@ -0,0 +1,7 @@ +- hosts: localhost + tasks: + - debug: msg="this is play 1" + +- hosts: localhost + tasks: + - debug: msg="this is play 2, facts should not have gathered" diff --git a/v2/samples/test_pb.yml b/v2/samples/test_pb.yml new file mode 100644 index 0000000000..3912d4566b --- /dev/null +++ b/v2/samples/test_pb.yml @@ -0,0 +1,70 @@ +# will use linear strategy by default +- hosts: + - "{{hosts|default('all')}}" + #- ubuntu1404 + #- awxlocal + connection: ssh + #gather_facts: false + #strategy: free + #serial: 3 + vars: + play_var: foo + test_dict: + a: 1 + b: 2 + vars_files: + - testing/vars.yml + tasks: + - block: + - debug: var=ansible_nodename + when: ansible_nodename == "ubuntu1404" + - block: + - debug: msg="in block for {{inventory_hostname}} ({{ansible_nodename}}), group_var is {{group_var}}, host var is {{host_var}}" + notify: foo + - debug: msg="test dictionary is {{test_dict}}" + when: asdf is defined + - command: hostname + register: hostname_result + - debug: msg="registered result is {{hostname_result.stdout}}" + - command: whoami + sudo: true + sudo_user: testing + - assemble: src=./testing/ dest=/tmp/output.txt remote_src=no + - copy: content="hello world\n" dest=/tmp/copy_content.out mode=600 + - command: /bin/false + retries: "{{num_retries|default(5)}}" + delay: 1 + - debug: msg="you shouldn't see me" + rescue: + - debug: msg="this is the rescue" + - command: /bin/false + - debug: msg="you should not see this rescue message" + always: + - debug: msg="this is the always block, it should always be seen" + - command: /bin/false + - debug: msg="you should not see this always message" + + #- debug: msg="linear task 01" + #- debug: msg="linear task 02" + #- debug: msg="linear task 03" + # with_items: + # - a + # - b + # - c + + handlers: + - name: foo + debug: msg="this is the foo handler" + - name: bar + debug: msg="this is the bar handler, you should not see this" + +#- hosts: all +# connection: local +# strategy: free +# tasks: +# - ping: +# - command: /bin/false +# - debug: msg="free task 01" +# - debug: msg="free task 02" +# - debug: msg="free task 03" + diff --git a/v2/samples/test_role.yml b/v2/samples/test_role.yml new file mode 100644 index 0000000000..114fd5a489 --- /dev/null +++ b/v2/samples/test_role.yml @@ -0,0 +1,8 @@ +- hosts: ubuntu1404 + gather_facts: no + vars: + foo: "BAD!!" + roles: + - { role: test_role, foo: bar } + tasks: + - debug: msg="done" diff --git a/v2/samples/testing/extra_vars.yml b/v2/samples/testing/extra_vars.yml new file mode 100644 index 0000000000..1f1cdb46f8 --- /dev/null +++ b/v2/samples/testing/extra_vars.yml @@ -0,0 +1 @@ +num_retries: 2 diff --git a/v2/samples/testing/frag1 b/v2/samples/testing/frag1 new file mode 100644 index 0000000000..a9575ae04b --- /dev/null +++ b/v2/samples/testing/frag1 @@ -0,0 +1 @@ +frag 1 diff --git a/v2/samples/testing/frag2 b/v2/samples/testing/frag2 new file mode 100644 index 0000000000..7a15a34848 --- /dev/null +++ b/v2/samples/testing/frag2 @@ -0,0 +1 @@ +frag 2 diff --git a/v2/samples/testing/frag3 b/v2/samples/testing/frag3 new file mode 100644 index 0000000000..d79a92e92c --- /dev/null +++ b/v2/samples/testing/frag3 @@ -0,0 +1 @@ +frag 3 diff --git a/v2/samples/testing/vars.yml b/v2/samples/testing/vars.yml new file mode 100644 index 0000000000..d7323611b4 --- /dev/null +++ b/v2/samples/testing/vars.yml @@ -0,0 +1 @@ +vars_file_var: "this is in a vars file" diff --git a/v2/samples/with_dict.yml b/v2/samples/with_dict.yml new file mode 100644 index 0000000000..59aa3da16e --- /dev/null +++ b/v2/samples/with_dict.yml @@ -0,0 +1,15 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + users: + alice: + name: Alice Appleworth + telephone: 123-456-7890 + bob: + name: Bob Bananarama + telephone: 987-654-3210 + tasks: + - name: Print phone records + debug: msg="User {{ item.key }} is {{ item.value.name }} ({{ item.value.telephone }})" + with_dict: users diff --git a/v2/samples/with_env.yml b/v2/samples/with_env.yml new file mode 100644 index 0000000000..856df20867 --- /dev/null +++ b/v2/samples/with_env.yml @@ -0,0 +1,5 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="{{ lookup('env','HOME') }} is an environment variable" diff --git a/v2/samples/with_fileglob.yml b/v2/samples/with_fileglob.yml new file mode 100644 index 0000000000..f955ee2132 --- /dev/null +++ b/v2/samples/with_fileglob.yml @@ -0,0 +1,7 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="file is {{item}}" + with_fileglob: + - "*.yml" diff --git a/v2/samples/with_first_found.yml b/v2/samples/with_first_found.yml new file mode 100644 index 0000000000..e64b36cb50 --- /dev/null +++ b/v2/samples/with_first_found.yml @@ -0,0 +1,10 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="file is {{item}}" + with_first_found: + - /etc/foo + - /etc/bar + - /etc/passwd + - /etc/shadow diff --git a/v2/samples/with_flattened.yml b/v2/samples/with_flattened.yml new file mode 100644 index 0000000000..b5d2876ace --- /dev/null +++ b/v2/samples/with_flattened.yml @@ -0,0 +1,13 @@ +- hosts: localhost + connection: local + gather_facts: + vars: + list_a: + - ['foo', 'bar'] + list_b: + - [['bam', 'baz']] + tasks: + - debug: msg="item is {{item}}" + with_flattened: + - list_a + - list_b diff --git a/v2/samples/with_indexed_items.yml b/v2/samples/with_indexed_items.yml new file mode 100644 index 0000000000..de8fdf1888 --- /dev/null +++ b/v2/samples/with_indexed_items.yml @@ -0,0 +1,11 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + some_list: + - a + - b + - c + tasks: + - debug: msg="at array position {{ item.0 }} there is a value {{ item.1 }}" + with_indexed_items: some_list diff --git a/v2/samples/with_items.yml b/v2/samples/with_items.yml new file mode 100644 index 0000000000..c486cf686e --- /dev/null +++ b/v2/samples/with_items.yml @@ -0,0 +1,11 @@ +- hosts: localhost + connection: local + vars: + my_list: + - a + - b + - c + gather_facts: no + tasks: + - debug: msg="item is {{item}}" + with_items: my_list diff --git a/v2/samples/with_lines.yml b/v2/samples/with_lines.yml new file mode 100644 index 0000000000..ab00491028 --- /dev/null +++ b/v2/samples/with_lines.yml @@ -0,0 +1,6 @@ +- hosts: localhost + gather_facts: no + tasks: + - debug: msg="line is {{item}}" + with_lines: + - "cat /etc/hosts" diff --git a/v2/samples/with_nested.yml b/v2/samples/with_nested.yml new file mode 100644 index 0000000000..aa295554fd --- /dev/null +++ b/v2/samples/with_nested.yml @@ -0,0 +1,13 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + users: + - foo + - bar + - bam + tasks: + - debug: msg="item.0={{ item[0] }} item.1={{ item[1] }}" + with_nested: + - users + - [ 'clientdb', 'employeedb', 'providerdb' ] diff --git a/v2/samples/with_random_choice.yml b/v2/samples/with_random_choice.yml new file mode 100644 index 0000000000..4ad4fc1a35 --- /dev/null +++ b/v2/samples/with_random_choice.yml @@ -0,0 +1,10 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg={{ item }} + with_random_choice: + - "go through the door" + - "drink from the goblet" + - "press the red button" + - "do nothing" diff --git a/v2/samples/with_sequence.yml b/v2/samples/with_sequence.yml new file mode 100644 index 0000000000..f25e9d24b3 --- /dev/null +++ b/v2/samples/with_sequence.yml @@ -0,0 +1,13 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + + - debug: msg="name={{ item }} state=present groups=evens" + with_sequence: start=0 end=32 format=testuser%02x + + - debug: msg="dest=/var/stuff/{{ item }} state=directory" + with_sequence: start=4 end=16 stride=2 + + - debug: msg="name=group{{ item }} state=present" + with_sequence: count=4 diff --git a/v2/samples/with_subelements.yml b/v2/samples/with_subelements.yml new file mode 100644 index 0000000000..95d0dda67c --- /dev/null +++ b/v2/samples/with_subelements.yml @@ -0,0 +1,18 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + users: + - name: alice + authorized: + - /tmp/alice/onekey.pub + - /tmp/alice/twokey.pub + - name: bob + authorized: + - /tmp/bob/id_rsa.pub + + tasks: + - debug: msg="user={{ item.0.name }} key='{{ item.1 }}'" + with_subelements: + - users + - authorized diff --git a/v2/samples/with_together.yml b/v2/samples/with_together.yml new file mode 100644 index 0000000000..073b801033 --- /dev/null +++ b/v2/samples/with_together.yml @@ -0,0 +1,11 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + alpha: [ 'a', 'b', 'c', 'd' ] + numbers: [ 1, 2, 3, 4 ] + tasks: + - debug: msg="{{ item.0 }} and {{ item.1 }}" + with_together: + - alpha + - numbers diff --git a/v2/test/errors/test_errors.py b/v2/test/errors/test_errors.py index 30ff411128..3e8e0dd7ba 100644 --- a/v2/test/errors/test_errors.py +++ b/v2/test/errors/test_errors.py @@ -39,8 +39,8 @@ class TestErrors(unittest.TestCase): def test_basic_error(self): e = AnsibleError(self.message) - self.assertEqual(e.message, self.message) - self.assertEqual(e.__repr__(), self.message) + self.assertEqual(e.message, 'ERROR! ' + self.message) + self.assertEqual(e.__repr__(), 'ERROR! ' + self.message) @patch.object(AnsibleError, '_get_error_lines_from_file') def test_error_with_object(self, mock_method): @@ -51,7 +51,7 @@ class TestErrors(unittest.TestCase): mock_method.return_value = ('this is line 1\n', '') e = AnsibleError(self.message, self.obj) - self.assertEqual(e.message, "This is the error message\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nthis is line 1\n^\n") + self.assertEqual(e.message, "ERROR! This is the error message\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\nthis is line 1\n^ here\n") def test_get_error_lines_from_file(self): m = mock_open() @@ -63,12 +63,12 @@ class TestErrors(unittest.TestCase): self.obj._line_number = 1 self.obj._column_number = 1 e = AnsibleError(self.message, self.obj) - self.assertEqual(e.message, "This is the error message\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nthis is line 1\n^\n") + self.assertEqual(e.message, "ERROR! This is the error message\n\nThe error appears to have been in 'foo.yml': line 1, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\nthis is line 1\n^ here\n") # this line will not be found, as it is out of the index range self.obj._data_source = 'foo.yml' self.obj._line_number = 2 self.obj._column_number = 1 e = AnsibleError(self.message, self.obj) - self.assertEqual(e.message, "This is the error message\n\nThe error appears to have been in 'foo.yml': line 2, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\n(specified line no longer in file, maybe it changed?)") + self.assertEqual(e.message, "ERROR! This is the error message\n\nThe error appears to have been in 'foo.yml': line 2, column 1, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\n(specified line no longer in file, maybe it changed?)") diff --git a/v2/test/executor/test_playbook_iterator.py b/v2/test/executor/test_play_iterator.py index 96db014fd6..47c0352b25 100644 --- a/v2/test/executor/test_playbook_iterator.py +++ b/v2/test/executor/test_play_iterator.py @@ -23,12 +23,12 @@ from ansible.compat.tests import unittest from ansible.compat.tests.mock import patch, MagicMock from ansible.errors import AnsibleError, AnsibleParserError -from ansible.executor.playbook_iterator import PlaybookIterator +from ansible.executor.play_iterator import PlayIterator from ansible.playbook import Playbook from test.mock.loader import DictDataLoader -class TestPlaybookIterator(unittest.TestCase): +class TestPlayIterator(unittest.TestCase): def setUp(self): pass @@ -36,10 +36,11 @@ class TestPlaybookIterator(unittest.TestCase): def tearDown(self): pass - def test_playbook_iterator(self): + def test_play_iterator(self): fake_loader = DictDataLoader({ "test_play.yml": """ - hosts: all + gather_facts: false roles: - test_role pre_tasks: @@ -64,8 +65,9 @@ class TestPlaybookIterator(unittest.TestCase): inventory = MagicMock() inventory.get_hosts.return_value = hosts + inventory.filter_hosts.return_value = hosts - itr = PlaybookIterator(inventory, None, p) + itr = PlayIterator(inventory, p._entries[0]) task = itr.get_next_task_for_host(hosts[0]) print(task) self.assertIsNotNone(task) diff --git a/v2/test/mock/loader.py b/v2/test/mock/loader.py index 89dbfeea62..b79dfa509d 100644 --- a/v2/test/mock/loader.py +++ b/v2/test/mock/loader.py @@ -21,7 +21,7 @@ __metaclass__ = type import os -from ansible.parsing.yaml import DataLoader +from ansible.parsing import DataLoader class DictDataLoader(DataLoader): diff --git a/v2/test/parsing/yaml/test_data_loader.py b/v2/test/parsing/test_data_loader.py index 166a60ee5e..370046dbf3 100644 --- a/v2/test/parsing/yaml/test_data_loader.py +++ b/v2/test/parsing/test_data_loader.py @@ -25,7 +25,7 @@ from ansible.compat.tests import unittest from ansible.compat.tests.mock import patch from ansible.errors import AnsibleParserError -from ansible.parsing.yaml import DataLoader +from ansible.parsing import DataLoader from ansible.parsing.yaml.objects import AnsibleMapping class TestDataLoader(unittest.TestCase): diff --git a/v2/test/playbook/test_task.py b/v2/test/playbook/test_task.py index 0af53c9117..b2160e0dd2 100644 --- a/v2/test/playbook/test_task.py +++ b/v2/test/playbook/test_task.py @@ -60,7 +60,6 @@ class TestTask(unittest.TestCase): def test_load_task_kv_form(self): t = Task.load(kv_shell_task) - print("task action is %s" % t.action) self.assertEqual(t.action, 'command') self.assertEqual(t.args, dict(_raw_params='echo hi', _uses_shell=True)) |