NoUniqueMatch: ClientException on Gnocchi publisher

Ceilometer can ignore/discard measurements that come from possible
VMs that can be used to host Gnocchi. The assumption is that one
can use OpenStack itself to host the VMs that are used to run Gnocchi.
The configuration is called `filter_project`. This configuration can
be used in `event_pipeline.yaml` and `pipeline.yaml` configuration files.
However, when this config is not used, it has a default project name
for Gnocchi as `service`, as we can see in the following code snippet:

```
def __init__(self, conf, parsed_url):
  super(GnocchiPublisher, self).__init__(conf, parsed_url)
  # TODO(jd) allow to override Gnocchi endpoint via the host in the URL
  options = urlparse.parse_qs(parsed_url.query)
  self.filter_project = options.get('filter_project', ['service'])[-1]
```

Which means that if somebody creates a project called `service`, this project would not push measurements to Gnocchi.
This configuration is then used by the following code:
```
def gnocchi_project_id(self):
       if self._gnocchi_project_id is not None:
           return self._gnocchi_project_id
       with self._gnocchi_project_id_lock:
           if self._gnocchi_project_id is None:
               try:
                   project = self._ks_client.projects.find(
                       name=self.filter_project,
                       domain=self.filter_domain)
               except ka_exceptions.NotFound:
                   LOG.warning('filtered project not found in keystone,'
                               ' ignoring the filter_project '
                               'option')
                   self.filter_project = None
                   return None
               except Exception:
                   LOG.exception('fail to retrieve filtered project ')
                   raise
               self._gnocchi_project_id = project.id
               LOG.debug("filtered project found: %s",
                         self._gnocchi_project_id)
           return self._gnocchi_project_id
```

Basically, this method will look for the project ID of the project
name that is configured in `filter_project` option. If it does not
find any project, it returns None, and it changes the value of
`filter_project` to None as well. Before this `gnocchi_project_id`
method/property is called, there is a verification if
`filter_project` is None. Therefore, it is assumed that when we
set the value of `filter_project` to None, this method
(`gnocchi_project_id`) would not be called anymore.
However, that is not taking into account concurrency parallel
executions.

In the code, we can see `with self._gnocchi_project_id_lock:`
statement, which seems to execute locking in the execution flow.
However, that will not always be the case because multiple
concurrent calls can be queued in that part of the code, and
when the first one finishes setting the `filter_project` to None,
the others will execute with this variable as None, which will cause
Keystone command `self._ks_client.projects.find` to find/list all
projects. That command was designed to list/find only one project;
therefore, when it finds more than one project, it throws an error.
That is the cause for the exception we were seeing from time to time
in the log files.

Change-Id: I3b4ac918015b2fd3fbe24047c3eb13419f580b27

2 files changed, 42 insertions, 12 deletions

diff --git a/ceilometer/publisher/gnocchi.py b/ceilometer/publisher/gnocchi.py
index 19f186c7..79a659e1 100644
--- a/ceilometer/publisher/gnocchi.py
+++ b/ceilometer/publisher/gnocchi.py
@@ -280,22 +280,31 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
             return self._gnocchi_project_id
         with self._gnocchi_project_id_lock:
             if self._gnocchi_project_id is None:
+                if not self.filter_project:
+                    LOG.debug(
+                        "Multiple executions were locked on "
+                        "self._gnocchi_project_id_lock`. This execution "
+                        "should no call `_internal_gnocchi_project_discovery` "
+                        "as `self.filter_project` is None.")
+                    return None
                 try:
                     project = self._ks_client.projects.find(
                         name=self.filter_project,
                         domain=self.filter_domain)
                 except ka_exceptions.NotFound:
-                    LOG.warning('project %s not found in keystone,'
-                                ' ignoring the filter_project '
-                                'option', self.filter_project)
+                    LOG.warning('Filtered project [%s] not found in keystone, '
+                                'ignoring the filter_project option' %
+                                self.filter_project)
+
                     self.filter_project = None
                     return None
                 except Exception:
-                    LOG.exception('fail to retrieve filtered project ')
+                    LOG.exception('Failed to retrieve filtered project [%s].'
+                                  % self.filter_project)
                     raise
                 self._gnocchi_project_id = project.id
-                LOG.debug("filtered project found: %s",
-                          self._gnocchi_project_id)
+                LOG.debug("Filtered project [%s] found with ID [%s].",
+                          self.filter_project, self._gnocchi_project_id)
             return self._gnocchi_project_id
 
     def _is_swift_account_sample(self, sample):
@@ -320,11 +329,29 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
             if operation:
                 return rd, operation
 
+    def filter_gnocchi_activity_openstack(self, samples):
+        """Skip sample generated by gnocchi itself
+
+        This method will filter out the samples that are generated by
+        Gnocchi itself.
+        """
+        filtered_samples = []
+        for sample in samples:
+            if not self._is_gnocchi_activity(sample):
+                filtered_samples.append(sample)
+                LOG.debug("Sample [%s] is not a Gnocchi activity; therefore, "
+                          "we do not filter it out and push it to Gnocchi.",
+                          sample)
+            else:
+                LOG.debug("Sample [%s] is a Gnocchi activity; therefore, "
+                          "we filter it out and do not push it to Gnocchi.",
+                          sample)
+        return filtered_samples
+
     def publish_samples(self, data):
         self.ensures_archives_policies()
 
-        # NOTE(sileht): skip sample generated by gnocchi itself
-        data = [s for s in data if not self._is_gnocchi_activity(s)]
+        data = self.filter_gnocchi_activity_openstack(data)
 
         def value_to_sort(object_to_sort):
             value = object_to_sort.resource_id
diff --git a/ceilometer/tests/unit/publisher/test_gnocchi.py b/ceilometer/tests/unit/publisher/test_gnocchi.py
index 741594d5..e8264f85 100644
--- a/ceilometer/tests/unit/publisher/test_gnocchi.py
+++ b/ceilometer/tests/unit/publisher/test_gnocchi.py
@@ -339,9 +339,9 @@ class PublisherTest(base.BaseTestCase):
     def test_activity_gnocchi_project_not_found(self, logger):
         self.ks_client.projects.find.side_effect = ka_exceptions.NotFound
         self._do_test_activity_filter(2)
-        logger.warning.assert_called_with('project %s not found in '
-                                          'keystone, ignoring the '
-                                          'filter_project option', 'service')
+        logger.warning.assert_called_with(
+            'Filtered project [service] not found in keystone, ignoring the '
+            'filter_project option')
 
     def test_activity_filter_match_swift_event(self):
         self.samples[0].name = 'storage.objects.outgoing.bytes'
@@ -749,8 +749,11 @@ class PublisherWorkflowTest(base.BaseTestCase,
         resource_type = resource_definition.cfg['resource_type']
 
         expected_debug = [
-            mock.call('filtered project found: %s',
+            mock.call('Filtered project [%s] found with ID [%s].', 'service',
                       'a2d42c23-d518-46b6-96ab-3fba2e146859'),
+            mock.call('Sample [%s] is not a Gnocchi activity; therefore, we '
+                      'do not filter it out and push it to Gnocchi.',
+                      self.sample),
             mock.call('Processing sample [%s] for resource ID [%s].',
                       self.sample, resource_id),
             mock.call('Executing batch resource metrics measures for resource '