Add ceph object storage meters

Implemented pollster classes to get the basic meters from ceph
object storage (i.e radosgw) and added corresponding unittests.

DocImpact

Co-Authored-By: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>

Implements: blueprint ceph-ceilometer-integration

Change-Id: I0c71b88621e503863f153912b9c70371b194ed2c

2 files changed, 284 insertions, 0 deletions

diff --git a/ceilometer/objectstore/rgw.py b/ceilometer/objectstore/rgw.py
new file mode 100644
index 00000000..ef4506fd
--- /dev/null
+++ b/ceilometer/objectstore/rgw.py
@@ -0,0 +1,212 @@
+#
+# Copyright 2015 Reliance Jio Infocomm Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Common code for working with ceph object stores
+"""
+
+from keystoneclient import exceptions
+from oslo_config import cfg
+from oslo_utils import timeutils
+import six.moves.urllib.parse as urlparse
+
+from ceilometer.agent import plugin_base
+from ceilometer.i18n import _
+from ceilometer.objectstore.rgw_client import RGWAdminClient as rgwclient
+from ceilometer.openstack.common import log
+from ceilometer import sample
+
+LOG = log.getLogger(__name__)
+
+SERVICE_OPTS = [
+    cfg.StrOpt('radosgw',
+               default='object-store',
+               help='Radosgw service type.'),
+]
+
+CREDENTIAL_OPTS = [
+    cfg.StrOpt('access_key',
+               secret=True,
+               help='Access key for Radosgw Admin.'),
+    cfg.StrOpt('secret_key',
+               secret=True,
+               help='Secret key for Radosgw Admin.')
+]
+
+cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
+cfg.CONF.register_opts(CREDENTIAL_OPTS, group='rgw_admin_credentials')
+cfg.CONF.import_group('rgw_admin_credentials', 'ceilometer.service')
+
+
+class _Base(plugin_base.PollsterBase):
+    METHOD = 'bucket'
+    _ENDPOINT = None
+
+    def __init__(self):
+        self.access_key = cfg.CONF.rgw_admin_credentials.access_key
+        self.secret = cfg.CONF.rgw_admin_credentials.secret_key
+
+    @property
+    def default_discovery(self):
+        return 'tenant'
+
+    @property
+    def CACHE_KEY_METHOD(self):
+        return 'rgw.get_%s' % self.METHOD
+
+    @staticmethod
+    def _get_endpoint(ksclient):
+        # we store the endpoint as a base class attribute, so keystone is
+        # only ever called once, also we assume that in a single deployment
+        # we may be only deploying `radosgw` or `swift` as the object-store
+        if _Base._ENDPOINT is None:
+            try:
+                conf = cfg.CONF.service_credentials
+                rgw_url = ksclient.service_catalog.url_for(
+                    service_type=cfg.CONF.service_types.radosgw,
+                    endpoint_type=conf.os_endpoint_type)
+                _Base._ENDPOINT = urlparse.urljoin(rgw_url, '/admin')
+            except exceptions.EndpointNotFound:
+                LOG.debug(_("Radosgw endpoint not found"))
+        return _Base._ENDPOINT
+
+    def _iter_accounts(self, ksclient, cache, tenants):
+        if self.CACHE_KEY_METHOD not in cache:
+            cache[self.CACHE_KEY_METHOD] = list(self._get_account_info(
+                ksclient, tenants))
+        return iter(cache[self.CACHE_KEY_METHOD])
+
+    def _get_account_info(self, ksclient, tenants):
+        endpoint = self._get_endpoint(ksclient)
+        if not endpoint:
+            raise StopIteration()
+
+        rgw_client = rgwclient(endpoint, self.access_key, self.secret)
+        for t in tenants:
+            api_method = 'get_%s' % self.METHOD
+            yield t.id, getattr(rgw_client, api_method)(t.id)
+
+
+class ContainersObjectsPollster(_Base):
+    """Get info about object counts in a container using RGW Admin APIs."""
+
+    def get_samples(self, manager, cache, resources):
+        for tenant, bucket_info in self._iter_accounts(manager.keystone,
+                                                       cache, resources):
+            for it in bucket_info['buckets']:
+                yield sample.Sample(
+                    name='radosgw.containers.objects',
+                    type=sample.TYPE_GAUGE,
+                    volume=int(it.num_objects),
+                    unit='object',
+                    user_id=None,
+                    project_id=tenant,
+                    resource_id=tenant + '/' + it.name,
+                    timestamp=timeutils.isotime(),
+                    resource_metadata=None,
+                )
+
+
+class ContainersSizePollster(_Base):
+    """Get info about object sizes in a container using RGW Admin APIs."""
+
+    def get_samples(self, manager, cache, resources):
+        for tenant, bucket_info in self._iter_accounts(manager.keystone,
+                                                       cache, resources):
+            for it in bucket_info['buckets']:
+                    yield sample.Sample(
+                        name='radosgw.containers.objects.size',
+                        type=sample.TYPE_GAUGE,
+                        volume=int(it.size * 1024),
+                        unit='B',
+                        user_id=None,
+                        project_id=tenant,
+                        resource_id=tenant + '/' + it.name,
+                        timestamp=timeutils.isotime(),
+                        resource_metadata=None,
+                    )
+
+
+class ObjectsSizePollster(_Base):
+    """Iterate over all accounts, using keystone."""
+
+    def get_samples(self, manager, cache, resources):
+        for tenant, bucket_info in self._iter_accounts(manager.keystone,
+                                                       cache, resources):
+            yield sample.Sample(
+                name='radosgw.objects.size',
+                type=sample.TYPE_GAUGE,
+                volume=int(bucket_info['size'] * 1024),
+                unit='B',
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+                )
+
+
+class ObjectsPollster(_Base):
+    """Iterate over all accounts, using keystone."""
+
+    def get_samples(self, manager, cache, resources):
+        for tenant, bucket_info in self._iter_accounts(manager.keystone,
+                                                       cache, resources):
+            yield sample.Sample(
+                name='radosgw.objects',
+                type=sample.TYPE_GAUGE,
+                volume=int(bucket_info['num_objects']),
+                unit='object',
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+                )
+
+
+class ObjectsContainersPollster(_Base):
+    def get_samples(self, manager, cache, resources):
+        for tenant, bucket_info in self._iter_accounts(manager.keystone,
+                                                       cache, resources):
+            yield sample.Sample(
+                name='radosgw.objects.containers',
+                type=sample.TYPE_GAUGE,
+                volume=int(bucket_info['num_buckets']),
+                unit='object',
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+                )
+
+
+class UsagePollster(_Base):
+
+    METHOD = 'usage'
+
+    def get_samples(self, manager, cache, resources):
+        for tenant, usage in self._iter_accounts(manager.keystone,
+                                                 cache, resources):
+            yield sample.Sample(
+                name='radosgw.api.request',
+                type=sample.TYPE_GAUGE,
+                volume=int(usage),
+                unit='request',
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+                )
diff --git a/ceilometer/objectstore/rgw_client.py b/ceilometer/objectstore/rgw_client.py
new file mode 100644
index 00000000..2a3d1d7f
--- /dev/null
+++ b/ceilometer/objectstore/rgw_client.py
@@ -0,0 +1,72 @@
+#
+# Copyright 2015 Reliance Jio Infocomm Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from collections import namedtuple
+
+from awsauth import S3Auth
+import requests
+import six.moves.urllib.parse as urlparse
+
+from ceilometer.i18n import _
+
+
+class RGWAdminAPIFailed(Exception):
+    pass
+
+
+class RGWAdminClient(object):
+    Bucket = namedtuple('Bucket', 'name, num_objects, size')
+
+    def __init__(self, endpoint, access_key, secret_key):
+        self.access_key = access_key
+        self.secret = secret_key
+        self.endpoint = endpoint
+        self.hostname = urlparse.urlparse(endpoint).netloc
+
+    def _make_request(self, path, req_params):
+        uri = "{0}/{1}".format(self.endpoint, path)
+        r = requests.get(uri, params=req_params,
+                         auth=S3Auth(self.access_key, self.secret,
+                                     self.hostname)
+                         )
+
+        if r.status_code != 200:
+            raise RGWAdminAPIFailed(
+                _('RGW AdminOps API returned %(status)s %(reason)s') %
+                {'status': r.status_code, 'reason': r.reason})
+
+        return r.json()
+
+    def get_bucket(self, tenant_id):
+        path = "bucket"
+        req_params = {"uid": tenant_id, "stats": "true"}
+        json_data = self._make_request(path, req_params)
+        stats = {'num_buckets': 0, 'buckets': [], 'size': 0, 'num_objects': 0}
+        stats['num_buckets'] = len(json_data)
+        for it in json_data:
+            for k, v in it["usage"].items():
+                stats['num_objects'] += v["num_objects"]
+                stats['size'] += v["size_kb"]
+                stats['buckets'].append(self.Bucket(it["bucket"],
+                                        v["num_objects"], v["size_kb"]))
+        return stats
+
+    def get_usage(self, tenant_id):
+        path = "usage"
+        req_params = {"uid": tenant_id}
+        json_data = self._make_request(path, req_params)
+        usage_data = json_data["summary"]
+        return sum((it["total"]["ops"] for it in usage_data))