From 463594b229017b30a1c457aaf6d4cdfef2bb421c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Weing=C3=A4rtner?= Date: Thu, 8 Dec 2022 08:16:14 -0300 Subject: NoUniqueMatch: ClientException on Gnocchi publisher Ceilometer can ignore/discard measurements that come from possible VMs that can be used to host Gnocchi. The assumption is that one can use OpenStack itself to host the VMs that are used to run Gnocchi. The configuration is called `filter_project`. This configuration can be used in `event_pipeline.yaml` and `pipeline.yaml` configuration files. However, when this config is not used, it has a default project name for Gnocchi as `service`, as we can see in the following code snippet: ``` def __init__(self, conf, parsed_url): super(GnocchiPublisher, self).__init__(conf, parsed_url) # TODO(jd) allow to override Gnocchi endpoint via the host in the URL options = urlparse.parse_qs(parsed_url.query) self.filter_project = options.get('filter_project', ['service'])[-1] ``` Which means that if somebody creates a project called `service`, this project would not push measurements to Gnocchi. This configuration is then used by the following code: ``` def gnocchi_project_id(self): if self._gnocchi_project_id is not None: return self._gnocchi_project_id with self._gnocchi_project_id_lock: if self._gnocchi_project_id is None: try: project = self._ks_client.projects.find( name=self.filter_project, domain=self.filter_domain) except ka_exceptions.NotFound: LOG.warning('filtered project not found in keystone,' ' ignoring the filter_project ' 'option') self.filter_project = None return None except Exception: LOG.exception('fail to retrieve filtered project ') raise self._gnocchi_project_id = project.id LOG.debug("filtered project found: %s", self._gnocchi_project_id) return self._gnocchi_project_id ``` Basically, this method will look for the project ID of the project name that is configured in `filter_project` option. If it does not find any project, it returns None, and it changes the value of `filter_project` to None as well. Before this `gnocchi_project_id` method/property is called, there is a verification if `filter_project` is None. Therefore, it is assumed that when we set the value of `filter_project` to None, this method (`gnocchi_project_id`) would not be called anymore. However, that is not taking into account concurrency parallel executions. In the code, we can see `with self._gnocchi_project_id_lock:` statement, which seems to execute locking in the execution flow. However, that will not always be the case because multiple concurrent calls can be queued in that part of the code, and when the first one finishes setting the `filter_project` to None, the others will execute with this variable as None, which will cause Keystone command `self._ks_client.projects.find` to find/list all projects. That command was designed to list/find only one project; therefore, when it finds more than one project, it throws an error. That is the cause for the exception we were seeing from time to time in the log files. Change-Id: I3b4ac918015b2fd3fbe24047c3eb13419f580b27 --- ceilometer/publisher/gnocchi.py | 43 ++++++++++++++++++++----- ceilometer/tests/unit/publisher/test_gnocchi.py | 11 ++++--- 2 files changed, 42 insertions(+), 12 deletions(-) (limited to 'ceilometer') diff --git a/ceilometer/publisher/gnocchi.py b/ceilometer/publisher/gnocchi.py index 19f186c7..79a659e1 100644 --- a/ceilometer/publisher/gnocchi.py +++ b/ceilometer/publisher/gnocchi.py @@ -280,22 +280,31 @@ class GnocchiPublisher(publisher.ConfigPublisherBase): return self._gnocchi_project_id with self._gnocchi_project_id_lock: if self._gnocchi_project_id is None: + if not self.filter_project: + LOG.debug( + "Multiple executions were locked on " + "self._gnocchi_project_id_lock`. This execution " + "should no call `_internal_gnocchi_project_discovery` " + "as `self.filter_project` is None.") + return None try: project = self._ks_client.projects.find( name=self.filter_project, domain=self.filter_domain) except ka_exceptions.NotFound: - LOG.warning('project %s not found in keystone,' - ' ignoring the filter_project ' - 'option', self.filter_project) + LOG.warning('Filtered project [%s] not found in keystone, ' + 'ignoring the filter_project option' % + self.filter_project) + self.filter_project = None return None except Exception: - LOG.exception('fail to retrieve filtered project ') + LOG.exception('Failed to retrieve filtered project [%s].' + % self.filter_project) raise self._gnocchi_project_id = project.id - LOG.debug("filtered project found: %s", - self._gnocchi_project_id) + LOG.debug("Filtered project [%s] found with ID [%s].", + self.filter_project, self._gnocchi_project_id) return self._gnocchi_project_id def _is_swift_account_sample(self, sample): @@ -320,11 +329,29 @@ class GnocchiPublisher(publisher.ConfigPublisherBase): if operation: return rd, operation + def filter_gnocchi_activity_openstack(self, samples): + """Skip sample generated by gnocchi itself + + This method will filter out the samples that are generated by + Gnocchi itself. + """ + filtered_samples = [] + for sample in samples: + if not self._is_gnocchi_activity(sample): + filtered_samples.append(sample) + LOG.debug("Sample [%s] is not a Gnocchi activity; therefore, " + "we do not filter it out and push it to Gnocchi.", + sample) + else: + LOG.debug("Sample [%s] is a Gnocchi activity; therefore, " + "we filter it out and do not push it to Gnocchi.", + sample) + return filtered_samples + def publish_samples(self, data): self.ensures_archives_policies() - # NOTE(sileht): skip sample generated by gnocchi itself - data = [s for s in data if not self._is_gnocchi_activity(s)] + data = self.filter_gnocchi_activity_openstack(data) def value_to_sort(object_to_sort): value = object_to_sort.resource_id diff --git a/ceilometer/tests/unit/publisher/test_gnocchi.py b/ceilometer/tests/unit/publisher/test_gnocchi.py index 741594d5..e8264f85 100644 --- a/ceilometer/tests/unit/publisher/test_gnocchi.py +++ b/ceilometer/tests/unit/publisher/test_gnocchi.py @@ -339,9 +339,9 @@ class PublisherTest(base.BaseTestCase): def test_activity_gnocchi_project_not_found(self, logger): self.ks_client.projects.find.side_effect = ka_exceptions.NotFound self._do_test_activity_filter(2) - logger.warning.assert_called_with('project %s not found in ' - 'keystone, ignoring the ' - 'filter_project option', 'service') + logger.warning.assert_called_with( + 'Filtered project [service] not found in keystone, ignoring the ' + 'filter_project option') def test_activity_filter_match_swift_event(self): self.samples[0].name = 'storage.objects.outgoing.bytes' @@ -749,8 +749,11 @@ class PublisherWorkflowTest(base.BaseTestCase, resource_type = resource_definition.cfg['resource_type'] expected_debug = [ - mock.call('filtered project found: %s', + mock.call('Filtered project [%s] found with ID [%s].', 'service', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), + mock.call('Sample [%s] is not a Gnocchi activity; therefore, we ' + 'do not filter it out and push it to Gnocchi.', + self.sample), mock.call('Processing sample [%s] for resource ID [%s].', self.sample, resource_id), mock.call('Executing batch resource metrics measures for resource ' -- cgit v1.2.1