path: root/storage/netapp/netapp_e_volume.py

#!/usr/bin/python

# (c) 2016, NetApp, Inc
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

from ansible.module_utils.api import basic_auth_argument_spec

DOCUMENTATION = '''
---
module: netapp_e_volume
version_added: "2.2"
short_description: Manage storage volumes (standard and thin)
description:
    - Create or remove volumes (standard and thin) for NetApp E/EF-series storage arrays.
options:
  api_username:
      required: true
      description:
      - The username to authenticate with the SANtricity WebServices Proxy or embedded REST API.
  api_password:
      required: true
      description:
      - The password to authenticate with the SANtricity WebServices Proxy or embedded REST API.
  api_url:
      required: true
      description:
      - The url to the SANtricity WebServices Proxy or embedded REST API.
      example:
      - https://prod-1.wahoo.acme.com/devmgr/v2
  validate_certs:
      required: false
      default: true
      description:
      - Should https certificates be validated?
  ssid:
    required: true
    description:
    - The ID of the array to manage (as configured on the web services proxy).
  state:
    required: true
    description:
    - Whether the specified volume should exist or not.
    choices: ['present', 'absent']
  name:
    required: true
    description:
    - The name of the volume to manage
  storage_pool_name:
    required: true
    description:
    - "Required only when requested state is 'present'.  The name of the storage pool the volume should exist on."
  size_unit:
    description:
    - The unit used to interpret the size parameter
    choices: ['bytes', 'b', 'kb', 'mb', 'gb', 'tb', 'pb', 'eb', 'zb', 'yb']
    default: 'gb'
  size:
    required: true
    description:
    - "Required only when state = 'present'.  The size of the volume in (size_unit)."
  segment_size_kb:
    description:
    - The segment size of the new volume
    default: 512
  thin_provision:
    description:
    - Whether the volume should be thin provisioned.  Thin volumes can only be created on disk pools (raidDiskPool).
    default: False
    choices: ['yes','no','true','false']
  thin_volume_repo_size:
    description:
    - Initial size of the thin volume repository volume (in size_unit)
    required: True
  thin_volume_max_repo_size:
    description:
    - Maximum size that the thin volume repository volume will automatically expand to
    default: same as size (in size_unit)
  ssd_cache_enabled:
    description:
    - Whether an existing SSD cache should be enabled on the volume (fails if no SSD cache defined)
    default: None (ignores existing SSD cache setting)
    choices: ['yes','no','true','false']
  data_assurance_enabled:
    description:
    - If data assurance should be enabled for the volume
    default: false

# TODO: doc thin volume parameters

author: Kevin Hulquest (@hulquest)

'''
EXAMPLES = '''
    - name: No thin volume
      netapp_e_volume:
        ssid: "{{ ssid }}"
        name: NewThinVolumeByAnsible
        state: absent
        log_path: /tmp/volume.log
        api_url: "{{ netapp_api_url }}"
        api_username: "{{ netapp_api_username }}"
        api_password: "{{ netapp_api_password }}"
        validate_certs: "{{ netapp_api_validate_certs }}"
      when: check_volume


    - name: No fat volume
      netapp_e_volume:
        ssid: "{{ ssid }}"
        name: NewVolumeByAnsible
        state: absent
        log_path: /tmp/volume.log
        api_url: "{{ netapp_api_url }}"
        api_username: "{{ netapp_api_username }}"
        api_password: "{{ netapp_api_password }}"
        validate_certs: "{{ netapp_api_validate_certs }}"
      when: check_volume
'''
RETURN = '''
---
msg: "Standard volume [workload_vol_1] has been created."
msg: "Thin volume [workload_thin_vol] has been created."
msg: "Volume [workload_vol_1] has been expanded."
msg: "Volume [workload_vol_1] has been deleted."
msg: "Volume [workload_vol_1] did not exist."
msg: "Volume [workload_vol_1] already exists."
'''

import json
import logging
import time
from traceback import format_exc

from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.pycompat24 import get_exception
from ansible.module_utils.urls import open_url
from ansible.module_utils.six.moves.urllib.error import HTTPError


def request(url, data=None, headers=None, method='GET', use_proxy=True,
            force=False, last_mod_time=None, timeout=10, validate_certs=True,
            url_username=None, url_password=None, http_agent=None, force_basic_auth=True, ignore_errors=False):
    try:
        r = open_url(url=url, data=data, headers=headers, method=method, use_proxy=use_proxy,
                     force=force, last_mod_time=last_mod_time, timeout=timeout, validate_certs=validate_certs,
                     url_username=url_username, url_password=url_password, http_agent=http_agent,
                     force_basic_auth=force_basic_auth)
    except HTTPError:
        err = get_exception()
        r = err.fp

    try:
        raw_data = r.read()
        if raw_data:
            data = json.loads(raw_data)
        else:
            raw_data is None
    except:
        if ignore_errors:
            pass
        else:
            raise Exception(raw_data)

    resp_code = r.getcode()

    if resp_code >= 400 and not ignore_errors:
        raise Exception(resp_code, data)
    else:
        return resp_code, data


def ifilter(predicate, iterable):
    # python 2, 3 generic filtering.
    if predicate is None:
        predicate = bool
    for x in iterable:
        if predicate(x):
            yield x


class NetAppESeriesVolume(object):
    def __init__(self):
        self._size_unit_map = dict(
            bytes=1,
            b=1,
            kb=1024,
            mb=1024 ** 2,
            gb=1024 ** 3,
            tb=1024 ** 4,
            pb=1024 ** 5,
            eb=1024 ** 6,
            zb=1024 ** 7,
            yb=1024 ** 8
        )

        self._post_headers = dict(Accept="application/json")
        self._post_headers['Content-Type'] = 'application/json'

        argument_spec = basic_auth_argument_spec()
        argument_spec.update(dict(
            state=dict(required=True, choices=['present', 'absent']),
            ssid=dict(required=True, type='str'),
            name=dict(required=True, type='str'),
            storage_pool_name=dict(type='str'),
            size_unit=dict(default='gb', choices=['bytes', 'b', 'kb', 'mb', 'gb', 'tb', 'pb', 'eb', 'zb', 'yb'],
                           type='str'),
            size=dict(type='int'),
            segment_size_kb=dict(default=128, choices=[8, 16, 32, 64, 128, 256, 512], type='int'),
            ssd_cache_enabled=dict(type='bool'),  # no default, leave existing setting alone
            data_assurance_enabled=dict(default=False, type='bool'),
            thin_provision=dict(default=False, type='bool'),
            thin_volume_repo_size=dict(type='int'),
            thin_volume_max_repo_size=dict(type='int'),
            # TODO: add cache, owning controller support, thin expansion policy, etc
            log_path=dict(type='str'),
            api_url=dict(type='str'),
            api_username=dict(type='str'),
            api_password=dict(type='str'),
            validate_certs=dict(type='bool'),
        ))

        self.module = AnsibleModule(argument_spec=argument_spec,
                                    required_if=[
                                        ('state', 'present', ['storage_pool_name', 'size']),
                                        ('thin_provision', 'true', ['thin_volume_repo_size'])
                                    ],
                                    supports_check_mode=True)
        p = self.module.params

        log_path = p['log_path']

        # logging setup
        self._logger = logging.getLogger(self.__class__.__name__)
        self.debug = self._logger.debug

        if log_path:
            logging.basicConfig(level=logging.DEBUG, filename=log_path)

        self.state = p['state']
        self.ssid = p['ssid']
        self.name = p['name']
        self.storage_pool_name = p['storage_pool_name']
        self.size_unit = p['size_unit']
        self.size = p['size']
        self.segment_size_kb = p['segment_size_kb']
        self.ssd_cache_enabled = p['ssd_cache_enabled']
        self.data_assurance_enabled = p['data_assurance_enabled']
        self.thin_provision = p['thin_provision']
        self.thin_volume_repo_size = p['thin_volume_repo_size']
        self.thin_volume_max_repo_size = p['thin_volume_max_repo_size']

        if not self.thin_volume_max_repo_size:
            self.thin_volume_max_repo_size = self.size

        self.validate_certs = p['validate_certs']

        try:
            self.api_usr = p['api_username']
            self.api_pwd = p['api_password']
            self.api_url = p['api_url']
        except KeyError:
            self.module.fail_json(msg="You must pass in api_username "
                                      "and api_password and api_url to the module.")

    def get_volume(self, volume_name):
        self.debug('fetching volumes')
        # fetch the list of volume objects and look for one with a matching name (we'll need to merge volumes and thin-volumes)
        try:
            (rc, volumes) = request(self.api_url + "/storage-systems/%s/volumes" % (self.ssid),
                                    headers=dict(Accept="application/json"), url_username=self.api_usr,
                                    url_password=self.api_pwd, validate_certs=self.validate_certs)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to obtain list of standard/thick volumes.  Array Id [%s]. Error[%s]." % (self.ssid,
                                                                                                     str(err)))

        try:
            self.debug('fetching thin-volumes')
            (rc, thinvols) = request(self.api_url + "/storage-systems/%s/thin-volumes" % (self.ssid),
                                     headers=dict(Accept="application/json"), url_username=self.api_usr,
                                     url_password=self.api_pwd, validate_certs=self.validate_certs)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to obtain list of thin volumes.  Array Id [%s]. Error[%s]." % (self.ssid, str(err)))

        volumes.extend(thinvols)

        self.debug("searching for volume '%s'" % volume_name)
        volume_detail = next(ifilter(lambda a: a['name'] == volume_name, volumes), None)

        if volume_detail:
            self.debug('found')
        else:
            self.debug('not found')

        return volume_detail

    def get_storage_pool(self, storage_pool_name):
        self.debug("fetching storage pools")
        # map the storage pool name to its id
        try:
            (rc, resp) = request(self.api_url + "/storage-systems/%s/storage-pools" % (self.ssid),
                                 headers=dict(Accept="application/json"), url_username=self.api_usr,
                                 url_password=self.api_pwd, validate_certs=self.validate_certs)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to obtain list of storage pools.  Array Id [%s]. Error[%s]." % (self.ssid, str(err)))

        self.debug("searching for storage pool '%s'" % storage_pool_name)
        pool_detail = next(ifilter(lambda a: a['name'] == storage_pool_name, resp), None)

        if pool_detail:
            self.debug('found')
        else:
            self.debug('not found')

        return pool_detail

    def create_volume(self, pool_id, name, size_unit, size, segment_size_kb, data_assurance_enabled):
        volume_add_req = dict(
            name=name,
            poolId=pool_id,
            sizeUnit=size_unit,
            size=size,
            segSize=segment_size_kb,
            dataAssuranceEnabled=data_assurance_enabled,
        )

        self.debug("creating volume '%s'" % name)
        try:
            (rc, resp) = request(self.api_url + "/storage-systems/%s/volumes" % (self.ssid),
                                 data=json.dumps(volume_add_req), headers=self._post_headers, method='POST',
                                 url_username=self.api_usr, url_password=self.api_pwd,
                                 validate_certs=self.validate_certs,
                                 timeout=120)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to create volume.  Volume [%s].  Array Id [%s]. Error[%s]." % (self.name, self.ssid,
                                                                                           str(err)))

    def create_thin_volume(self, pool_id, name, size_unit, size, thin_volume_repo_size,
                           thin_volume_max_repo_size, data_assurance_enabled):
        thin_volume_add_req = dict(
            name=name,
            poolId=pool_id,
            sizeUnit=size_unit,
            virtualSize=size,
            repositorySize=thin_volume_repo_size,
            maximumRepositorySize=thin_volume_max_repo_size,
            dataAssuranceEnabled=data_assurance_enabled,
        )

        self.debug("creating thin-volume '%s'" % name)
        try:
            (rc, resp) = request(self.api_url + "/storage-systems/%s/thin-volumes" % (self.ssid),
                                 data=json.dumps(thin_volume_add_req), headers=self._post_headers, method='POST',
                                 url_username=self.api_usr, url_password=self.api_pwd,
                                 validate_certs=self.validate_certs,
                                 timeout=120)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to create thin volume.  Volume [%s].  Array Id [%s]. Error[%s]." % (self.name,
                                                                                                self.ssid,
                                                                                                str(err)))

    def delete_volume(self):
        # delete the volume
        self.debug("deleting volume '%s'" % self.volume_detail['name'])
        try:
            (rc, resp) = request(
                self.api_url + "/storage-systems/%s/%s/%s" % (self.ssid, self.volume_resource_name,
                                                              self.volume_detail['id']),
                method='DELETE', url_username=self.api_usr, url_password=self.api_pwd,
                validate_certs=self.validate_certs, timeout=120)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to delete volume.  Volume [%s].  Array Id [%s]. Error[%s]." % (self.name, self.ssid,
                                                                                           str(err)))

    @property
    def volume_resource_name(self):
        if self.volume_detail['thinProvisioned']:
            return 'thin-volumes'
        else:
            return 'volumes'

    @property
    def volume_properties_changed(self):
        return self.volume_ssdcache_setting_changed  # or with other props here when extended

        # TODO: add support for r/w cache settings, owning controller, scan settings, expansion policy, growth alert threshold

    @property
    def volume_ssdcache_setting_changed(self):
        # None means ignore existing setting
        if self.ssd_cache_enabled is not None and self.ssd_cache_enabled != self.volume_detail['flashCached']:
            self.debug("flash cache setting changed")
            return True

    def update_volume_properties(self):
        update_volume_req = dict()

        # conditionally add values so we ignore unspecified props
        if self.volume_ssdcache_setting_changed:
            update_volume_req['flashCache'] = self.ssd_cache_enabled

        self.debug("updating volume properties...")
        try:
            (rc, resp) = request(
                self.api_url + "/storage-systems/%s/%s/%s/" % (self.ssid, self.volume_resource_name,
                                                               self.volume_detail['id']),
                data=json.dumps(update_volume_req), headers=self._post_headers, method='POST',
                url_username=self.api_usr, url_password=self.api_pwd, validate_certs=self.validate_certs,
                timeout=120)
        except Exception:
            err = get_exception()
            self.module.fail_json(
                msg="Failed to update volume properties.  Volume [%s].  Array Id [%s]. Error[%s]." % (self.name,
                                                                                                      self.ssid,
                                                                                                      str(err)))

    @property
    def volume_needs_expansion(self):
        current_size_bytes = int(self.volume_detail['capacity'])
        requested_size_bytes = self.size * self._size_unit_map[self.size_unit]

        # TODO: check requested/current repo volume size for thin-volumes as well

        # TODO: do we need to build any kind of slop factor in here?
        return requested_size_bytes > current_size_bytes

    def expand_volume(self):
        is_thin = self.volume_detail['thinProvisioned']
        if is_thin:
            # TODO: support manual repo expansion as well
            self.debug('expanding thin volume')
            thin_volume_expand_req = dict(
                newVirtualSize=self.size,
                sizeUnit=self.size_unit
            )
            try:
                (rc, resp) = request(self.api_url + "/storage-systems/%s/thin-volumes/%s/expand" % (self.ssid,
                                                                                                    self.volume_detail[
                                                                                                        'id']),
                                     data=json.dumps(thin_volume_expand_req), headers=self._post_headers, method='POST',
                                     url_username=self.api_usr, url_password=self.api_pwd,
                                     validate_certs=self.validate_certs, timeout=120)
            except Exception:
                err = get_exception()
                self.module.fail_json(
                    msg="Failed to expand thin volume.  Volume [%s].  Array Id [%s]. Error[%s]." % (self.name,
                                                                                                    self.ssid,
                                                                                                    str(err)))

                # TODO: check return code
        else:
            self.debug('expanding volume')
            volume_expand_req = dict(
                expansionSize=self.size,
                sizeUnit=self.size_unit
            )
            try:
                (rc, resp) = request(
                    self.api_url + "/storage-systems/%s/volumes/%s/expand" % (self.ssid,
                                                                              self.volume_detail['id']),
                    data=json.dumps(volume_expand_req), headers=self._post_headers, method='POST',
                    url_username=self.api_usr, url_password=self.api_pwd, validate_certs=self.validate_certs,
                    timeout=120)
            except Exception:
                err = get_exception()
                self.module.fail_json(
                    msg="Failed to expand volume.  Volume [%s].  Array Id [%s]. Error[%s]." % (self.name,
                                                                                               self.ssid,
                                                                                               str(err)))

            self.debug('polling for completion...')

            while True:
                try:
                    (rc, resp) = request(self.api_url + "/storage-systems/%s/volumes/%s/expand" % (self.ssid,
                                                                                                   self.volume_detail[
                                                                                                       'id']),
                                         method='GET', url_username=self.api_usr, url_password=self.api_pwd,
                                         validate_certs=self.validate_certs)
                except Exception:
                    err = get_exception()
                    self.module.fail_json(
                        msg="Failed to get volume expansion progress.  Volume [%s].  Array Id [%s]. Error[%s]." % (
                            self.name, self.ssid, str(err)))

                action = resp['action']
                percent_complete = resp['percentComplete']

                self.debug('expand action %s, %s complete...' % (action, percent_complete))

                if action == 'none':
                    self.debug('expand complete')
                    break
                else:
                    time.sleep(5)

    def apply(self):
        changed = False
        volume_exists = False
        msg = None

        self.volume_detail = self.get_volume(self.name)

        if self.volume_detail:
            volume_exists = True

            if self.state == 'absent':
                self.debug("CHANGED: volume exists, but requested state is 'absent'")
                changed = True
            elif self.state == 'present':
                # check requested volume size, see if expansion is necessary
                if self.volume_needs_expansion:
                    self.debug(
                        "CHANGED: requested volume size %s%s is larger than current size %sb" % (self.size,
                                                                                                 self.size_unit,
                                                                                                 self.volume_detail[
                                                                                                     'capacity']))
                    changed = True

                if self.volume_properties_changed:
                    self.debug("CHANGED: one or more volume properties have changed")
                    changed = True

        else:
            if self.state == 'present':
                self.debug("CHANGED: volume does not exist, but requested state is 'present'")
                changed = True

        if changed:
            if self.module.check_mode:
                self.debug('skipping changes due to check mode')
            else:
                if self.state == 'present':
                    if not volume_exists:
                        pool_detail = self.get_storage_pool(self.storage_pool_name)

                        if not pool_detail:
                            self.module.fail_json(msg='Requested storage pool (%s) not found' % self.storage_pool_name)

                        if self.thin_provision and not pool_detail['diskPool']:
                            self.module.fail_json(
                                msg='Thin provisioned volumes can only be located on disk pools (not volume groups)')

                        pool_id = pool_detail['id']

                        if not self.thin_provision:
                            self.create_volume(pool_id, self.name, self.size_unit, self.size, self.segment_size_kb,
                                               self.data_assurance_enabled)
                            msg = "Standard volume [%s] has been created." % (self.name)

                        else:
                            self.create_thin_volume(pool_id, self.name, self.size_unit, self.size,
                                                    self.thin_volume_repo_size, self.thin_volume_max_repo_size,
                                                    self.data_assurance_enabled)
                            msg = "Thin volume [%s] has been created." % (self.name)

                    else:  # volume exists but differs, modify...
                        if self.volume_needs_expansion:
                            self.expand_volume()
                            msg = "Volume [%s] has been expanded." % (self.name)

                    # this stuff always needs to run on present (since props can't be set on creation)
                    if self.volume_properties_changed:
                        self.update_volume_properties()
                        msg = "Properties of volume [%s] has been updated." % (self.name)

                elif self.state == 'absent':
                    self.delete_volume()
                    msg = "Volume [%s] has been deleted." % (self.name)
        else:
            self.debug("exiting with no changes")
            if self.state == 'absent':
                msg = "Volume [%s] did not exist." % (self.name)
            else:
                msg = "Volume [%s] already exists." % (self.name)

        self.module.exit_json(msg=msg, changed=changed)


def main():
    v = NetAppESeriesVolume()

    try:
        v.apply()
    except Exception:
        e = get_exception()
        v.debug("Exception in apply(): \n%s" % format_exc(e))
        v.module.fail_json(msg="Module failed. Error [%s]." % (str(e)))


if __name__ == '__main__':
    main()