Add user/project names to polled samples

Project and user names would be first fetched from cache, if not found,
they will be requested from keystone and then cached. Using cache will
significanlty reduce the number of calls made to keystone.

If ceilometer is configured with no caching backend, the results
will always be fetched by querying request to keystone.

A new config option, `tenant_name_discovery` is introduced
to operate this feature. This feature is optional and is disabled by default.

No attempts to identify names will be made if uuids are found to be `None`.

Signed-off-by: Yadnesh Kulkarni <ykulkarn@redhat.com>
Change-Id: Iee5dbf09a1fd3ac571746fc66d2683eb8e6a1b27

5 files changed, 132 insertions, 2 deletions

diff --git a/ceilometer/cache_utils.py b/ceilometer/cache_utils.py
new file mode 100644
index 00000000..55a9e263
--- /dev/null
+++ b/ceilometer/cache_utils.py
@@ -0,0 +1,53 @@
+#
+# Copyright 2022 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Simple wrapper for oslo_cache."""
+
+
+from oslo_cache import core as cache
+
+
+class CacheClient(object):
+    def __init__(self, region):
+        self.region = region
+
+    def get(self, key):
+        value = self.region.get(key)
+        if value == cache.NO_VALUE:
+            return None
+        return value
+
+    def set(self, key, value):
+        return self.region.set(key, value)
+
+    def delete(self, key):
+        return self.region.delete(key)
+
+
+def get_client(conf, expiration_time=0):
+    cache.configure(conf)
+    if conf.cache.enabled:
+        return CacheClient(_get_default_cache_region(
+            conf,
+            expiration_time=expiration_time
+        ))
+
+
+def _get_default_cache_region(conf, expiration_time):
+    region = cache.create_region()
+    if expiration_time != 0:
+        conf.cache.expiration_time = expiration_time
+    cache.configure_cache_region(conf, region)
+    return region
diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py
index 6b9289d1..e5776a3e 100644
--- a/ceilometer/polling/manager.py
+++ b/ceilometer/polling/manager.py
@@ -35,6 +35,7 @@ from tooz import coordination
 from urllib import parse as urlparse
 
 from ceilometer import agent
+from ceilometer import cache_utils
 from ceilometer import declarative
 from ceilometer import keystone_client
 from ceilometer import messaging
@@ -45,6 +46,8 @@ from ceilometer import utils
 
 LOG = log.getLogger(__name__)
 
+CACHE_DURATION = 3600
+
 POLLING_OPTS = [
     cfg.StrOpt('cfg_file',
                default="polling.yaml",
@@ -64,7 +67,18 @@ POLLING_OPTS = [
     cfg.MultiStrOpt('pollsters_definitions_dirs',
                     default=["/etc/ceilometer/pollsters.d"],
                     help="List of directories with YAML files used "
-                         "to created pollsters.")
+                         "to created pollsters."),
+    cfg.BoolOpt('tenant_name_discovery',
+                default=False,
+                help="Identify project and user names from polled samples"
+                     "By default, collecting these values is disabled due"
+                     "to the fact that it could overwhelm keystone service"
+                     "with lots of continuous requests depending upon the"
+                     "number of projects, users and samples polled from"
+                     "the environment. While using this feature, it is"
+                     "recommended that ceilometer be configured with a"
+                     "caching backend to reduce the number of calls"
+                     "made to keystone"),
 ]
 
 
@@ -138,11 +152,39 @@ class PollingTask(object):
 
         self._telemetry_secret = self.manager.conf.publisher.telemetry_secret
 
+        self.ks_client = self.manager.keystone
+
+        self.cache_client = cache_utils.get_client(
+            self.manager.conf,
+            expiration_time=CACHE_DURATION
+        )
+
     def add(self, pollster, source):
         self.pollster_matches[source.name].add(pollster)
         key = Resources.key(source.name, pollster)
         self.resources[key].setup(source)
 
+    def resolve_uuid_from_cache(self, attr, uuid):
+        if self.cache_client:
+            name = self.cache_client.get(uuid)
+            if name:
+                return name
+            name = self.resolve_uuid_from_keystone(attr, uuid)
+            self.cache_client.set(uuid, name)
+            return name
+
+        # Retrieve project and user names from Keystone only
+        # if ceilometer doesn't have a caching backend
+        return self.resolve_uuid_from_keystone(attr, uuid)
+
+    def resolve_uuid_from_keystone(self, attr, uuid):
+        try:
+            return getattr(self.ks_client, attr).get(uuid).name
+        except AttributeError as e:
+            LOG.warning("Found '%s' while resolving uuid %s to name", e, uuid)
+        except ka_exceptions.NotFound as e:
+            LOG.warning(e.message)
+
     def poll_and_notify(self):
         """Polling sample and notify."""
         cache = {}
@@ -194,6 +236,25 @@ class PollingTask(object):
                     for sample in samples:
                         # Note(yuywz): Unify the timestamp of polled samples
                         sample.set_timestamp(polling_timestamp)
+
+                        if self.manager.conf.tenant_name_discovery:
+
+                            # Try to resolve project UUIDs from cache first,
+                            # and then keystone
+                            if sample.project_id:
+                                sample.project_name = \
+                                    self.resolve_uuid_from_cache(
+                                        "projects", sample.project_id
+                                    )
+
+                            # Try to resolve user UUIDs from cache first,
+                            # and then keystone
+                            if sample.user_id:
+                                sample.user_name = \
+                                    self.resolve_uuid_from_cache(
+                                        "users", sample.user_id
+                                    )
+
                         sample_dict = (
                             publisher_utils.meter_message_from_counter(
                                 sample, self._telemetry_secret
diff --git a/ceilometer/publisher/utils.py b/ceilometer/publisher/utils.py
index 75df2b70..0d1e7be0 100644
--- a/ceilometer/publisher/utils.py
+++ b/ceilometer/publisher/utils.py
@@ -126,7 +126,9 @@ def meter_message_from_counter(sample, secret):
            'counter_unit': sample.unit,
            'counter_volume': sample.volume,
            'user_id': sample.user_id,
+           'user_name': sample.user_name,
            'project_id': sample.project_id,
+           'project_name': sample.project_name,
            'resource_id': sample.resource_id,
            'timestamp': sample.timestamp,
            'resource_metadata': sample.resource_metadata,
diff --git a/ceilometer/sample.py b/ceilometer/sample.py
index c86caa35..536b561d 100644
--- a/ceilometer/sample.py
+++ b/ceilometer/sample.py
@@ -94,13 +94,16 @@ class Sample(object):
 
     def __init__(self, name, type, unit, volume, user_id, project_id,
                  resource_id, timestamp=None, resource_metadata=None,
-                 source=None, id=None, monotonic_time=None):
+                 source=None, id=None, monotonic_time=None,
+                 user_name=None, project_name=None):
         self.name = name
         self.type = type
         self.unit = unit
         self.volume = volume
         self.user_id = user_id
+        self.user_name = user_name
         self.project_id = project_id
+        self.project_name = project_name
         self.resource_id = resource_id
         self.timestamp = timestamp
         self.resource_metadata = resource_metadata or {}
diff --git a/ceilometer/tests/unit/polling/test_manager.py b/ceilometer/tests/unit/polling/test_manager.py
index e805b9be..963e841c 100644
--- a/ceilometer/tests/unit/polling/test_manager.py
+++ b/ceilometer/tests/unit/polling/test_manager.py
@@ -378,6 +378,17 @@ class TestPollingAgent(BaseAgent):
         super(TestPollingAgent, self).setUp()
         self.mgr = self.create_manager()
         self.mgr.extensions = self.create_extension_list()
+        ks_client = mock.Mock(auth_token='fake_token')
+        ks_client.projects.get.return_value = mock.Mock(
+            name='admin', id='4465ecd1438b4d23a866cf8447387a7b'
+        )
+        ks_client.users.get.return_value = mock.Mock(
+            name='admin', id='c0c935468e654d5a8baae1a08adf4dfb'
+        )
+        self.useFixture(fixtures.MockPatch(
+            'ceilometer.keystone_client.get_client',
+            return_value=ks_client))
+        self.ks_client = ks_client
         self.setup_polling()
 
     @mock.patch('ceilometer.polling.manager.PollingManager')