diff options
| -rw-r--r-- | ceilometer/declarative.py | 9 | ||||
| -rw-r--r-- | ceilometer/polling/dynamic_pollster.py | 728 | ||||
| -rw-r--r-- | ceilometer/polling/manager.py | 11 | ||||
| -rw-r--r-- | ceilometer/polling/non_openstack_dynamic_pollster.py | 144 | ||||
| -rw-r--r-- | ceilometer/tests/unit/polling/test_dynamic_pollster.py | 275 | ||||
| -rw-r--r-- | ceilometer/tests/unit/polling/test_manager.py | 17 | ||||
| -rw-r--r-- | ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py | 220 | ||||
| -rw-r--r-- | doc/source/admin/telemetry-dynamic-pollster.rst | 207 |
8 files changed, 1205 insertions, 406 deletions
diff --git a/ceilometer/declarative.py b/ceilometer/declarative.py index 11480b6f..9fe367e9 100644 --- a/ceilometer/declarative.py +++ b/ceilometer/declarative.py @@ -42,11 +42,16 @@ class ResourceDefinitionException(DefinitionException): pass -class DynamicPollsterDefinitionException(DefinitionException): +class DynamicPollsterException(DefinitionException): pass -class NonOpenStackApisDynamicPollsterException(DefinitionException): +class DynamicPollsterDefinitionException(DynamicPollsterException): + pass + + +class NonOpenStackApisDynamicPollsterException\ + (DynamicPollsterDefinitionException): pass diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index dc7e76b1..913e30cd 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -17,6 +17,9 @@ '/etc/ceilometer/pollsters.d/'. The pollster are defined in YAML files similar to the idea used for handling notifications. """ +import copy +import re + from oslo_log import log from oslo_utils import timeutils @@ -24,7 +27,7 @@ from requests import RequestException from ceilometer import declarative from ceilometer.polling import plugin_base -from ceilometer import sample +from ceilometer import sample as ceilometer_sample from functools import reduce import operator @@ -35,203 +38,447 @@ from six.moves.urllib import parse as url_parse LOG = log.getLogger(__name__) -class DynamicPollster(plugin_base.PollsterBase): +def validate_sample_type(sample_type): + if sample_type not in ceilometer_sample.TYPES: + raise declarative.DynamicPollsterDefinitionException( + "Invalid sample type [%s]. Valid ones are [%s]." + % (sample_type, ceilometer_sample.TYPES)) - OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values', - 'value_mapping', 'default_value', - 'metadata_mapping', - 'preserve_mapped_metadata', - 'response_entries_key'] - REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', - 'value_attribute', 'endpoint_type', - 'url_path'] +class PollsterDefinitionBuilder(object): - # Mandatory name field - name = "" + def __init__(self, definitions): + self.definitions = definitions - def __init__(self, pollster_definitions, conf=None): - super(DynamicPollster, self).__init__(conf) + def build_definitions(self, configurations): + supported_definitions = [] + for definition in self.definitions: + if definition.is_field_applicable_to_definition(configurations): + supported_definitions.append(definition) - self.ALL_POLLSTER_FIELDS =\ - self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS + if not supported_definitions: + raise declarative.DynamicPollsterDefinitionException( + "Your configurations do not fit any type of DynamicPollsters, " + "please recheck them. Used configurations = [%s]." % + configurations) - LOG.debug("%s instantiated with [%s]", __name__, - pollster_definitions) + definition_name = self.join_supported_definitions_names( + supported_definitions) - self.pollster_definitions = pollster_definitions - self.validate_pollster_definition() + definition_parents = tuple(supported_definitions) + definition_attribs = {'extra_definitions': reduce( + lambda d1, d2: d1 + d2, map(lambda df: df.extra_definitions, + supported_definitions))} + definition_type = type(definition_name, definition_parents, + definition_attribs) + return definition_type(configurations) - if 'metadata_fields' in self.pollster_definitions: - LOG.debug("Metadata fields configured to [%s].", - self.pollster_definitions['metadata_fields']) + @staticmethod + def join_supported_definitions_names(supported_definitions): + return ''.join(map(lambda df: df.__name__, + supported_definitions)) - self.set_default_values() - self.name = self.pollster_definitions['name'] - self.obj = self +class PollsterSampleExtractor(object): - def set_default_values(self): - if 'skip_sample_values' not in self.pollster_definitions: - self.pollster_definitions['skip_sample_values'] = [] + def __init__(self, definitions): + self.definitions = definitions - if 'value_mapping' not in self.pollster_definitions: - self.pollster_definitions['value_mapping'] = {} + def generate_new_metadata_fields(self, metadata=None, + pollster_definitions=None): + pollster_definitions =\ + pollster_definitions or self.definitions.configurations + metadata_mapping = pollster_definitions['metadata_mapping'] + if not metadata_mapping or not metadata: + return - if 'default_value' not in self.pollster_definitions: - self.pollster_definitions['default_value'] = -1 + metadata_keys = list(metadata.keys()) + for k in metadata_keys: + if k not in metadata_mapping: + continue - if 'preserve_mapped_metadata' not in self.pollster_definitions: - self.pollster_definitions['preserve_mapped_metadata'] = True + new_key = metadata_mapping[k] + metadata[new_key] = metadata[k] + LOG.debug("Generating new key [%s] with content [%s] of key [%s]", + new_key, metadata[k], k) + if pollster_definitions['preserve_mapped_metadata']: + continue - if 'metadata_mapping' not in self.pollster_definitions: - self.pollster_definitions['metadata_mapping'] = {} + k_value = metadata.pop(k) + LOG.debug("Removed key [%s] with value [%s] from " + "metadata set that is sent to Gnocchi.", k, k_value) - if 'response_entries_key' not in self.pollster_definitions: - self.pollster_definitions['response_entries_key'] = None + def generate_sample(self, pollster_sample, pollster_definitons=None): + pollster_definitions =\ + pollster_definitons or self.definitions.configurations + metadata = [] + if 'metadata_fields' in pollster_definitions: + metadata = dict((k, pollster_sample.get(k)) + for k in pollster_definitions['metadata_fields']) + + self.generate_new_metadata_fields( + metadata=metadata, pollster_definitions=pollster_definitions) + return ceilometer_sample.Sample( + timestamp=timeutils.isotime(), + name=pollster_definitions['name'], + type=pollster_definitions['sample_type'], + unit=pollster_definitions['unit'], + volume=pollster_sample['value'], + user_id=pollster_sample.get("user_id"), + project_id=pollster_sample.get("project_id"), + resource_id=pollster_sample.get("id"), + resource_metadata=metadata) + + def retrieve_attribute_nested_value(self, json_object, + value_attribute=None): + + attribute_key = value_attribute or self.definitions.\ + extract_attribute_key() + LOG.debug("Retrieving the nested keys [%s] from [%s].", + attribute_key, json_object) + keys_and_operations = attribute_key.split("|") + attribute_key = keys_and_operations[0].strip() + nested_keys = attribute_key.split(".") + value = reduce(operator.getitem, nested_keys, json_object) - def validate_pollster_definition(self): - missing_required_fields = \ - [field for field in self.REQUIRED_POLLSTER_FIELDS - if field not in self.pollster_definitions] + return self.operate_value(keys_and_operations, value) + + def operate_value(self, keys_and_operations, value): + # We do not have operations to be executed against the value extracted + if len(keys_and_operations) < 2: + return value + for operation in keys_and_operations[1::]: + # The operation must be performed onto the 'value' variable + if 'value' not in operation: + raise declarative.DynamicPollsterDefinitionException( + "The attribute field operation [%s] must use the [" + "value] variable." % operation, + self.definitions.configurations) + LOG.debug("Executing operation [%s] against value [%s].", + operation, value) + value = eval(operation.strip()) + LOG.debug("Result [%s] of operation [%s].", + value, operation) + return value - if missing_required_fields: - raise declarative.DynamicPollsterDefinitionException( - "Required fields %s not specified." - % missing_required_fields, self.pollster_definitions) - sample_type = self.pollster_definitions['sample_type'] - if sample_type not in sample.TYPES: - raise declarative.DynamicPollsterDefinitionException( - "Invalid sample type [%s]. Valid ones are [%s]." - % (sample_type, sample.TYPES), self.pollster_definitions) +class SimplePollsterSampleExtractor(PollsterSampleExtractor): - for definition_key in self.pollster_definitions: - if definition_key not in self.ALL_POLLSTER_FIELDS: - LOG.warning( - "Field [%s] defined in [%s] is unknown " - "and will be ignored. Valid fields are [%s].", - definition_key, self.pollster_definitions, - self.ALL_POLLSTER_FIELDS) + def generate_single_sample(self, pollster_sample): + value = self.retrieve_attribute_nested_value(pollster_sample) + value = self.definitions.value_mapper.map_or_skip_value( + value, pollster_sample) - def get_samples(self, manager, cache, resources): - if not resources: - LOG.debug("No resources received for processing.") - yield None + if isinstance(value, SkippedSample): + return value - for r in resources: - LOG.debug("Executing get sample for resource [%s].", r) + pollster_sample['value'] = value - samples = list([]) - try: - samples = self.execute_request_get_samples( - keystone_client=manager._keystone, resource=r) - except RequestException as e: - LOG.warning("Error [%s] while loading samples for [%s] " - "for dynamic pollster [%s].", - e, r, self.name) + return self.generate_sample(pollster_sample) + + def extract_sample(self, pollster_sample): + sample = self.generate_single_sample(pollster_sample) + if isinstance(sample, SkippedSample): + return sample + yield sample - for pollster_sample in samples: - response_value_attribute_name = self.pollster_definitions[ - 'value_attribute'] - value = self.retrieve_attribute_nested_value( - pollster_sample, response_value_attribute_name) - - skip_sample_values = \ - self.pollster_definitions['skip_sample_values'] - if skip_sample_values and value in skip_sample_values: - LOG.debug("Skipping sample [%s] because value [%s] " - "is configured to be skipped in skip list [%s].", - pollster_sample, value, skip_sample_values) - continue - value = self.execute_value_mapping(value) +class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor): - user_id = None - if 'user_id' in pollster_sample: - user_id = pollster_sample["user_id"] + def extract_sample(self, pollster_sample): + pollster_definitions = self.definitions.configurations + value = self.retrieve_attribute_nested_value(pollster_sample) + LOG.debug("We are dealing with a multi metric pollster. The " + "value we are processing is the following: [%s].", + value) - project_id = None - if 'project_id' in pollster_sample: - project_id = pollster_sample["project_id"] + self.validate_sample_is_list(value) + sub_metric_placeholder, pollster_name, sub_metric_attribute_name = \ + self.extract_names_attrs() - resource_id = None - if 'id' in pollster_sample: - resource_id = pollster_sample["id"] + value_attribute = \ + self.extract_field_name_from_value_attribute_configuration() + LOG.debug("Using attribute [%s] to look for values in the " + "multi metric pollster [%s] with sample [%s]", + value_attribute, pollster_definitions, value) - metadata = [] - if 'metadata_fields' in self.pollster_definitions: - metadata = dict((k, pollster_sample.get(k)) - for k in self.pollster_definitions[ - 'metadata_fields']) - self.generate_new_metadata_fields(metadata=metadata) - yield sample.Sample( - timestamp=timeutils.isotime(), + pollster_definitions = copy.deepcopy(pollster_definitions) + yield from self.extract_sub_samples(value, sub_metric_attribute_name, + pollster_name, value_attribute, + sub_metric_placeholder, + pollster_definitions, + pollster_sample) - name=self.pollster_definitions['name'], - type=self.pollster_definitions['sample_type'], - unit=self.pollster_definitions['unit'], - volume=value, + def extract_sub_samples(self, value, sub_metric_attribute_name, + pollster_name, value_attribute, + sub_metric_placeholder, pollster_definitions, + pollster_sample): - user_id=user_id, - project_id=project_id, - resource_id=resource_id, + for sub_sample in value: + sub_metric_name = sub_sample[sub_metric_attribute_name] + new_metric_name = pollster_name.replace( + sub_metric_placeholder, sub_metric_name) + pollster_definitions['name'] = new_metric_name - resource_metadata=metadata - ) + actual_value = self.retrieve_attribute_nested_value( + sub_sample, value_attribute) + + pollster_sample['value'] = actual_value + + if self.should_skip_generate_sample(actual_value, sub_sample, + sub_metric_name): + continue + + yield self.generate_sample(pollster_sample, pollster_definitions) + + def extract_field_name_from_value_attribute_configuration(self): + value_attribute = self.definitions.configurations['value_attribute'] + return self.definitions.pattern_pollster_value_attribute.match( + value_attribute).group(3)[1::] + + def extract_names_attrs(self): + pollster_name = self.definitions.configurations['name'] + sub_metric_placeholder = pollster_name.split(".").pop() + return (sub_metric_placeholder, + pollster_name, + self.definitions.pattern_pollster_name.match( + "." + sub_metric_placeholder).group(2)) + + def validate_sample_is_list(self, value): + pollster_definitions = self.definitions.configurations + if not isinstance(value, list): + raise declarative.DynamicPollsterException( + "Multi metric pollster defined, but the value [%s]" + " obtained with [%s] attribute is not a list" + " of objects." + % (value, + pollster_definitions['value_attribute']), + pollster_definitions) + + def should_skip_generate_sample(self, actual_value, sub_sample, + sub_metric_name): + skip_sample_values = \ + self.definitions.configurations['skip_sample_values'] + if actual_value in skip_sample_values: + LOG.debug( + "Skipping multi metric sample [%s] because " + "value [%s] is configured to be skipped in " + "skip list [%s].", sub_sample, actual_value, + skip_sample_values) + return True + if sub_metric_name in skip_sample_values: + LOG.debug( + "Skipping sample [%s] because its sub-metric " + "name [%s] is configured to be skipped in " + "skip list [%s].", sub_sample, sub_metric_name, + skip_sample_values) + return True + return False + + +class PollsterValueMapper(object): + + def __init__(self, definitions): + self.definitions = definitions + + def map_or_skip_value(self, value, pollster_sample): + skip_sample_values = \ + self.definitions.configurations['skip_sample_values'] + + if value in skip_sample_values: + LOG.debug("Skipping sample [%s] because value [%s] " + "is configured to be skipped in skip list [%s].", + pollster_sample, value, skip_sample_values) + return SkippedSample() + + return self.execute_value_mapping(value) def execute_value_mapping(self, value): - value_mapping = self.pollster_definitions['value_mapping'] - if value_mapping: - if value in value_mapping: - old_value = value - value = value_mapping[value] - LOG.debug("Value mapped from [%s] to [%s]", - old_value, value) - else: - default_value = \ - self.pollster_definitions['default_value'] - LOG.warning( - "Value [%s] was not found in value_mapping [%s]; " - "therefore, we will use the default [%s].", - value, value_mapping, default_value) - value = default_value + value_mapping = self.definitions.configurations['value_mapping'] + if not value_mapping: + return value + + if value in value_mapping: + old_value = value + value = value_mapping[value] + LOG.debug("Value mapped from [%s] to [%s]", + old_value, value) + else: + default_value = \ + self.definitions.configurations['default_value'] + LOG.warning( + "Value [%s] was not found in value_mapping [%s]; " + "therefore, we will use the default [%s].", + value, value_mapping, default_value) + value = default_value return value - def generate_new_metadata_fields(self, metadata=None): - metadata_mapping = self.pollster_definitions['metadata_mapping'] - if not metadata_mapping or not metadata: - return - metadata_keys = list(metadata.keys()) - for k in metadata_keys: - if k not in metadata_mapping: - continue +class PollsterDefinition(object): + + def __init__(self, name, required=False, on_missing=lambda df: df.default, + default=None, validation_regex=None, creatable=True, + validator=None): + self.name = name + self.required = required + self.on_missing = on_missing + self.validation_regex = validation_regex + self.creatable = creatable + self.default = default + if self.validation_regex: + self.validation_pattern = re.compile(self.validation_regex) + self.validator = validator + + def validate(self, val): + if val is None: + return self.on_missing(self) + if self.validation_regex and not self.validation_pattern.match(val): + raise declarative.DynamicPollsterDefinitionException( + "Pollster %s [%s] does not match [%s]." + % (self.name, val, self.validation_regex)) + + if self.validator: + self.validator(val) + + return val + + +class PollsterDefinitions(object): + + POLLSTER_VALID_NAMES_REGEXP = "^([\w-]+)(\.[\w-]+)*(\.{[\w-]+})?$" + + standard_definitions = [ + PollsterDefinition(name='name', required=True, + validation_regex=POLLSTER_VALID_NAMES_REGEXP), + PollsterDefinition(name='sample_type', required=True, + validator=validate_sample_type), + PollsterDefinition(name='unit', required=True), + PollsterDefinition(name='endpoint_type', required=True), + PollsterDefinition(name='url_path', required=True), + PollsterDefinition(name='metadata_fields', creatable=False), + PollsterDefinition(name='skip_sample_values', default=[]), + PollsterDefinition(name='value_mapping', default={}), + PollsterDefinition(name='default_value', default=-1), + PollsterDefinition(name='metadata_mapping', default={}), + PollsterDefinition(name='preserve_mapped_metadata', default=True), + PollsterDefinition(name='response_entries_key')] + + extra_definitions = [] + + def __init__(self, configurations): + self.configurations = configurations + self.value_mapper = PollsterValueMapper(self) + self.definitions = self.map_definitions() + self.validate_configurations(configurations) + self.validate_missing() + self.sample_gatherer = PollsterSampleGatherer(self) + self.sample_extractor = SimplePollsterSampleExtractor(self) + + def validate_configurations(self, configurations): + for k, v in self.definitions.items(): + if configurations.get(k) is not None: + self.configurations[k] = self.definitions[k].validate( + self.configurations[k]) + elif self.definitions[k].creatable: + self.configurations[k] = self.definitions[k].default + + @staticmethod + def is_field_applicable_to_definition(configurations): + return True + + def map_definitions(self): + definitions = dict( + map(lambda df: (df.name, df), self.standard_definitions)) + extra_definitions = dict( + map(lambda df: (df.name, df), self.extra_definitions)) + definitions.update(extra_definitions) + return definitions + + def extract_attribute_key(self): + pass + + def validate_missing(self): + required_configurations = map(lambda fdf: fdf.name, + filter(lambda df: df.required, + self.definitions.values())) + + missing = list(filter( + lambda rf: rf not in map(lambda f: f[0], + filter(lambda f: f[1], + self.configurations.items())), + required_configurations)) + + if missing: + raise declarative.DynamicPollsterDefinitionException( + "Required fields %s not specified." + % missing, self.configurations) - new_key = metadata_mapping[k] - metadata[new_key] = metadata[k] - LOG.debug("Generating new key [%s] with content [%s] of key [%s]", - new_key, metadata[k], k) - if self.pollster_definitions['preserve_mapped_metadata']: - continue - k_value = metadata.pop(k) - LOG.debug("Removed key [%s] with value [%s] from " - "metadata set that is sent to Gnocchi.", k, k_value) +class MultiMetricPollsterDefinitions(PollsterDefinitions): + + MULTI_METRIC_POLLSTER_NAME_REGEXP = ".*(\.{(\w+)})$" + pattern_pollster_name = re.compile( + MULTI_METRIC_POLLSTER_NAME_REGEXP) + MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP = "^(\[(\w+)\])((\.\w+)+)$" + pattern_pollster_value_attribute = re.compile( + MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP) + + extra_definitions = [ + PollsterDefinition( + name='value_attribute', required=True, + validation_regex=MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP), + ] + + def __init__(self, configurations): + super(MultiMetricPollsterDefinitions, self).__init__(configurations) + self.sample_extractor = MultiMetricPollsterSampleExtractor(self) + + @staticmethod + def is_field_applicable_to_definition(configurations): + return configurations.get( + 'name') and MultiMetricPollsterDefinitions.\ + pattern_pollster_name.match(configurations['name']) + + def extract_attribute_key(self): + return self.pattern_pollster_value_attribute.match( + self.configurations['value_attribute']).group(2) + + +class SingleMetricPollsterDefinitions(PollsterDefinitions): + + extra_definitions = [ + PollsterDefinition(name='value_attribute', required=True)] + + def __init__(self, configurations): + super(SingleMetricPollsterDefinitions, self).__init__(configurations) + + def extract_attribute_key(self): + return self.configurations['value_attribute'] + + @staticmethod + def is_field_applicable_to_definition(configurations): + return not MultiMetricPollsterDefinitions. \ + is_field_applicable_to_definition(configurations) + + +class PollsterSampleGatherer(object): + + def __init__(self, definitions): + self.definitions = definitions @property def default_discovery(self): - return 'endpoint:' + self.pollster_definitions['endpoint_type'] + return 'endpoint:' + self.definitions.configurations['endpoint_type'] def execute_request_get_samples(self, **kwargs): - resp, url = self.internal_execute_request_get_samples(kwargs) + resp, url = self.definitions.sample_gatherer. \ + internal_execute_request_get_samples(kwargs) response_json = resp.json() - entry_size = len(response_json) LOG.debug("Entries [%s] in the JSON for request [%s] " "for dynamic pollster [%s].", - response_json, url, self.name) + response_json, url, self.definitions.configurations['name']) if entry_size > 0: return self.retrieve_entries_from_response(response_json) @@ -241,7 +488,7 @@ class DynamicPollster(plugin_base.PollsterBase): keystone_client = kwargs['keystone_client'] endpoint = kwargs['resource'] url = url_parse.urljoin( - endpoint, self.pollster_definitions['url_path']) + endpoint, self.definitions.configurations['url_path']) resp = keystone_client.session.get(url, authenticated=True) if resp.status_code != requests.codes.ok: resp.raise_for_status() @@ -251,41 +498,172 @@ class DynamicPollster(plugin_base.PollsterBase): if isinstance(response_json, list): return response_json - first_entry_name = self.pollster_definitions['response_entries_key'] + first_entry_name = \ + self.definitions.configurations['response_entries_key'] if not first_entry_name: try: first_entry_name = next(iter(response_json)) except RuntimeError as e: LOG.debug("Generator threw a StopIteration " "and we need to catch it [%s].", e) - return self.retrieve_attribute_nested_value(response_json, - first_entry_name) + return self.definitions.sample_extractor. \ + retrieve_attribute_nested_value(response_json, first_entry_name) - def retrieve_attribute_nested_value(self, json_object, attribute_key): - LOG.debug("Retrieving the nested keys [%s] from [%s].", - attribute_key, json_object) - keys_and_operations = attribute_key.split("|") - attribute_key = keys_and_operations[0].strip() +class NonOpenStackApisPollsterDefinition(PollsterDefinitions): - nested_keys = attribute_key.split(".") - value = reduce(operator.getitem, nested_keys, json_object) + extra_definitions = [ + PollsterDefinition(name='value_attribute', required=True), + PollsterDefinition(name='module', required=True), + PollsterDefinition(name='authentication_object', required=True), + PollsterDefinition(name='user_id_attribute'), + PollsterDefinition(name='resource_id_attribute'), + PollsterDefinition(name='barbican_secret_id', default=""), + PollsterDefinition(name='authentication_parameters', default=""), + PollsterDefinition(name='project_id_attribute'), + PollsterDefinition(name='endpoint_type')] - # We have operations to be executed against the value extracted - if len(keys_and_operations) > 1: - for operation in keys_and_operations[1::]: - # The operation must be performed onto the 'value' variable - if 'value' not in operation: - raise declarative.DynamicPollsterDefinitionException( - "The attribute field operation [%s] must use the [" - "value] variable." % operation, - self.pollster_definitions) + def __init__(self, configurations): + super(NonOpenStackApisPollsterDefinition, self).__init__( + configurations) + self.sample_gatherer = NonOpenStackApisSamplesGatherer(self) - LOG.debug("Executing operation [%s] against value[%s].", - operation, value) + @staticmethod + def is_field_applicable_to_definition(configurations): + return configurations.get('module') - value = eval(operation.strip()) - LOG.debug("Result [%s] of operation [%s].", - value, operation) - return value +class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): + + @property + def default_discovery(self): + return 'barbican:' + \ + self.definitions.configurations['barbican_secret_id'] + + def internal_execute_request_get_samples(self, kwargs): + credentials = kwargs['resource'] + + override_credentials = self.definitions.configurations[ + 'authentication_parameters'] + if override_credentials: + credentials = override_credentials + + url = self.definitions.configurations['url_path'] + + authenticator_module_name = self.definitions.configurations['module'] + authenticator_class_name = \ + self.definitions.configurations['authentication_object'] + + imported_module = __import__(authenticator_module_name) + authenticator_class = getattr(imported_module, + authenticator_class_name) + + authenticator_arguments = list(map(str.strip, credentials.split(","))) + authenticator_instance = authenticator_class(*authenticator_arguments) + + resp = requests.get( + url, + auth=authenticator_instance) + + if resp.status_code != requests.codes.ok: + raise declarative.NonOpenStackApisDynamicPollsterException( + "Error while executing request[%s]." + " Status[%s] and reason [%s]." + % (url, resp.status_code, resp.reason)) + + return resp, url + + def execute_request_get_samples(self, **kwargs): + samples = super(NonOpenStackApisSamplesGatherer, + self).execute_request_get_samples(**kwargs) + + if samples: + user_id_attribute = self.definitions.configurations[ + 'user_id_attribute'] + project_id_attribute = self.definitions.configurations[ + 'project_id_attribute'] + resource_id_attribute = self.definitions.configurations[ + 'resource_id_attribute'] + + for request_sample in samples: + self.generate_new_attributes_in_sample( + request_sample, user_id_attribute, 'user_id') + self.generate_new_attributes_in_sample( + request_sample, project_id_attribute, 'project_id') + self.generate_new_attributes_in_sample( + request_sample, resource_id_attribute, 'id') + + return samples + + def generate_new_attributes_in_sample( + self, sample, attribute_key, new_attribute_key): + if attribute_key: + attribute_value = self.definitions.sample_extractor. \ + retrieve_attribute_nested_value(sample, attribute_key) + + LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].", + attribute_key, attribute_value, sample) + + sample[new_attribute_key] = attribute_value + + +class SkippedSample(object): + pass + + +class DynamicPollster(plugin_base.PollsterBase): + # Mandatory name field + name = "" + + def __init__(self, pollster_definitions={}, conf=None, + supported_definitions=[NonOpenStackApisPollsterDefinition, + MultiMetricPollsterDefinitions, + SingleMetricPollsterDefinitions]): + super(DynamicPollster, self).__init__(conf) + self.supported_definitions = supported_definitions + LOG.debug("%s instantiated with [%s]", __name__, + pollster_definitions) + + self.definitions = PollsterDefinitionBuilder( + self.supported_definitions).build_definitions(pollster_definitions) + self.pollster_definitions = self.definitions.configurations + if 'metadata_fields' in self.pollster_definitions: + LOG.debug("Metadata fields configured to [%s].", + self.pollster_definitions['metadata_fields']) + + self.name = self.pollster_definitions['name'] + self.obj = self + + @property + def default_discovery(self): + return self.definitions.sample_gatherer.default_discovery() + + def load_samples(self, resource, manager): + try: + return self.definitions.sample_gatherer.\ + execute_request_get_samples(keystone_client=manager._keystone, + resource=resource) + except RequestException as e: + LOG.warning("Error [%s] while loading samples for [%s] " + "for dynamic pollster [%s].", + e, resource, self.name) + + return list([]) + + def get_samples(self, manager, cache, resources): + if not resources: + LOG.debug("No resources received for processing.") + yield None + + for r in resources: + LOG.debug("Executing get sample for resource [%s].", r) + samples = self.load_samples(r, manager) + for pollster_sample in samples: + sample = self.extract_sample(pollster_sample) + if isinstance(sample, SkippedSample): + continue + yield from sample + + def extract_sample(self, pollster_sample): + return self.definitions.sample_extractor.extract_sample( + pollster_sample) diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py index 3329d16e..fbdce8b5 100644 --- a/ceilometer/polling/manager.py +++ b/ceilometer/polling/manager.py @@ -40,7 +40,6 @@ from ceilometer import declarative from ceilometer import keystone_client from ceilometer import messaging from ceilometer.polling import dynamic_pollster -from ceilometer.polling import non_openstack_dynamic_pollster from ceilometer.polling import plugin_base from ceilometer.publisher import utils as publisher_utils from ceilometer import utils @@ -340,7 +339,8 @@ class AgentManager(cotyledon.Service): pollster_name, pollsters_definitions_file) try: pollsters_definitions[pollster_name] =\ - self.instantiate_dynamic_pollster(pollster_cfg) + dynamic_pollster.DynamicPollster( + pollster_cfg, self.conf) except Exception as e: LOG.error( "Error [%s] while loading dynamic pollster [%s].", @@ -355,13 +355,6 @@ class AgentManager(cotyledon.Service): len(pollsters_definitions)) return pollsters_definitions.values() - def instantiate_dynamic_pollster(self, pollster_cfg): - if 'module' in pollster_cfg: - return non_openstack_dynamic_pollster\ - .NonOpenStackApisDynamicPollster(pollster_cfg, self.conf) - else: - return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf) - @staticmethod def _get_ext_mgr(namespace, *args, **kwargs): def _catch_extension_load_error(mgr, ep, exc): diff --git a/ceilometer/polling/non_openstack_dynamic_pollster.py b/ceilometer/polling/non_openstack_dynamic_pollster.py deleted file mode 100644 index 2854b921..00000000 --- a/ceilometer/polling/non_openstack_dynamic_pollster.py +++ /dev/null @@ -1,144 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -"""Non-OpenStack Dynamic pollster component - This component enables operators to create pollsters on the fly - via configuration for non-OpenStack APIs. This appraoch is quite - useful when adding metrics from APIs such as RadosGW into the Cloud - rating and billing modules. -""" -import copy -import requests - -from ceilometer.declarative import NonOpenStackApisDynamicPollsterException -from ceilometer.polling.dynamic_pollster import DynamicPollster -from oslo_log import log - -LOG = log.getLogger(__name__) - - -class NonOpenStackApisDynamicPollster(DynamicPollster): - - POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object'] - - POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute', - 'project_id_attribute', - 'resource_id_attribute', - 'barbican_secret_id', - 'authentication_parameters' - ] - - def __init__(self, pollster_definitions, conf=None): - # Making sure that we do not change anything in parent classes - self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy( - DynamicPollster.REQUIRED_POLLSTER_FIELDS) - self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy( - DynamicPollster.OPTIONAL_POLLSTER_FIELDS) - - # Non-OpenStack dynamic pollster do not need the 'endpoint_type'. - self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type') - - self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS - self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS - - super(NonOpenStackApisDynamicPollster, self).__init__( - pollster_definitions, conf) - - def set_default_values(self): - super(NonOpenStackApisDynamicPollster, self).set_default_values() - - if 'user_id_attribute' not in self.pollster_definitions: - self.pollster_definitions['user_id_attribute'] = None - - if 'project_id_attribute' not in self.pollster_definitions: - self.pollster_definitions['project_id_attribute'] = None - - if 'resource_id_attribute' not in self.pollster_definitions: - self.pollster_definitions['resource_id_attribute'] = None - - if 'barbican_secret_id' not in self.pollster_definitions: - self.pollster_definitions['barbican_secret_id'] = "" - - if 'authentication_parameters' not in self.pollster_definitions: - self.pollster_definitions['authentication_parameters'] = "" - - @property - def default_discovery(self): - return 'barbican:' + self.pollster_definitions['barbican_secret_id'] - - def internal_execute_request_get_samples(self, kwargs): - credentials = kwargs['resource'] - - override_credentials = self.pollster_definitions[ - 'authentication_parameters'] - if override_credentials: - credentials = override_credentials - - url = self.pollster_definitions['url_path'] - - authenticator_module_name = self.pollster_definitions['module'] - authenticator_class_name = \ - self.pollster_definitions['authentication_object'] - - imported_module = __import__(authenticator_module_name) - authenticator_class = getattr(imported_module, - authenticator_class_name) - - authenticator_arguments = list(map(str.strip, credentials.split(","))) - authenticator_instance = authenticator_class(*authenticator_arguments) - - resp = requests.get( - url, - auth=authenticator_instance) - - if resp.status_code != requests.codes.ok: - raise NonOpenStackApisDynamicPollsterException( - "Error while executing request[%s]." - " Status[%s] and reason [%s]." - % (url, resp.status_code, resp.reason)) - - return resp, url - - def execute_request_get_samples(self, **kwargs): - samples = super(NonOpenStackApisDynamicPollster, - self).execute_request_get_samples(**kwargs) - - if samples: - user_id_attribute = self.pollster_definitions[ - 'user_id_attribute'] - project_id_attribute = self.pollster_definitions[ - 'project_id_attribute'] - resource_id_attribute = self.pollster_definitions[ - 'resource_id_attribute'] - - for sample in samples: - self.generate_new_attributes_in_sample( - sample, user_id_attribute, 'user_id') - self.generate_new_attributes_in_sample( - sample, project_id_attribute, 'project_id') - self.generate_new_attributes_in_sample( - sample, resource_id_attribute, 'id') - - return samples - - def generate_new_attributes_in_sample( - self, sample, attribute_key, new_attribute_key): - if attribute_key: - attribute_value = self.retrieve_attribute_nested_value( - sample, attribute_key) - - LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].", - attribute_key, attribute_value, sample) - - sample[new_attribute_key] = attribute_value diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index 52a2b916..912f7b8f 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -14,6 +14,7 @@ """Tests for ceilometer/polling/dynamic_pollster.py """ +from oslotest import base from ceilometer.declarative import DynamicPollsterDefinitionException from ceilometer.polling import dynamic_pollster @@ -23,13 +24,87 @@ import copy import logging import mock -from oslotest import base - import requests LOG = logging.getLogger(__name__) +REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', + 'value_attribute', 'endpoint_type', + 'url_path'] + + +class SampleGenerator(object): + + def __init__(self, samples_dict, turn_to_list=False): + self.turn_to_list = turn_to_list + self.samples_dict = {} + for k, v in samples_dict.items(): + if isinstance(v, list): + self.samples_dict[k] = [0, v] + else: + self.samples_dict[k] = [0, [v]] + + def get_next_sample_dict(self): + _dict = {} + for key in self.samples_dict.keys(): + _dict[key] = self.get_next_sample(key) + + if self.turn_to_list: + _dict = [_dict] + return _dict + + def get_next_sample(self, key): + samples = self.samples_dict[key][1] + samples_next_iteration = self.samples_dict[key][0] % len(samples) + self.samples_dict[key][0] += 1 + _sample = samples[samples_next_iteration] + if isinstance(_sample, SampleGenerator): + return _sample.get_next_sample_dict() + return _sample + + +class PagedSamplesGenerator(SampleGenerator): + + def __init__(self, samples_dict, dict_name, page_link_name): + super(PagedSamplesGenerator, self).__init__(samples_dict) + self.dict_name = dict_name + self.page_link_name = page_link_name + self.response = {} + + def generate_samples(self, page_base_link, page_links, last_page_size): + self.response.clear() + current_page_link = page_base_link + for page_link, page_size in page_links.items(): + page_link = page_base_link + "/" + page_link + self.response[current_page_link] = { + self.page_link_name: page_link, + self.dict_name: self.populate_page(page_size) + } + current_page_link = page_link + + self.response[current_page_link] = { + self.dict_name: self.populate_page(last_page_size) + } + + def populate_page(self, page_size): + page = [] + for item_number in range(0, page_size): + page.append(self.get_next_sample_dict()) + + return page + + +class PagedSamplesGeneratorHttpRequestMock(PagedSamplesGenerator): + + def mock_request(self, url, **kwargs): + return_value = TestDynamicPollster.FakeResponse() + return_value.status_code = requests.codes.ok + return_value.json_object = self.response[url] + + return return_value + + class TestDynamicPollster(base.BaseTestCase): class FakeResponse(object): status_code = None @@ -42,7 +117,8 @@ class TestDynamicPollster(base.BaseTestCase): raise requests.HTTPError("Mock HTTP error.", response=self) class FakeManager(object): - _keystone = None + def __init__(self, keystone=None): + self._keystone = keystone def setUp(self): super(TestDynamicPollster, self).setUp() @@ -62,19 +138,61 @@ class TestDynamicPollster(base.BaseTestCase): 'old-metadata-name': "new-metadata-name" }, 'preserve_mapped_metadata': False} + self.pollster_definition_all_fields.update( self.pollster_definition_only_required_fields) + self.multi_metric_pollster_definition = { + 'name': "test-pollster.{category}", 'sample_type': "gauge", + 'unit': "test", 'value_attribute': "[categories].ops", + 'endpoint_type': "test", 'url_path': "v1/test/endpoint/fake"} + def execute_basic_asserts(self, pollster, pollster_definition): self.assertEqual(pollster, pollster.obj) self.assertEqual(pollster_definition['name'], pollster.name) - for key in pollster.REQUIRED_POLLSTER_FIELDS: + for key in REQUIRED_POLLSTER_FIELDS: self.assertEqual(pollster_definition[key], pollster.pollster_definitions[key]) self.assertEqual(pollster_definition, pollster.pollster_definitions) + @mock.patch('keystoneclient.v2_0.client.Client') + def test_skip_samples(self, keystone_mock): + generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={ + 'volume': SampleGenerator(samples_dict={ + 'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], + 'tmp': ['ra', 'rb', 'rc', 'rd', 're', 'rf', 'rg', 'rh']}, + turn_to_list=True), + 'id': [1, 2, 3, 4, 5, 6, 7, 8], + 'name': ['a1', 'b2', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'] + }, dict_name='servers', page_link_name='server_link') + + generator.generate_samples('http://test.com/v1/test-volumes', { + 'marker=c3': 3, + 'marker=f6': 3 + }, 2) + + keystone_mock.session.get.side_effect = generator.mock_request + fake_manager = self.FakeManager(keystone=keystone_mock) + + pollster_definition = dict(self.multi_metric_pollster_definition) + pollster_definition['name'] = 'test-pollster.{name}' + pollster_definition['value_attribute'] = '[volume].tmp' + pollster_definition['skip_sample_values'] = ['rb'] + pollster_definition['url_path'] = 'v1/test-volumes' + pollster_definition['response_entries_key'] = 'servers' + pollster = dynamic_pollster.DynamicPollster(pollster_definition) + samples = pollster.get_samples(fake_manager, None, ['http://test.com']) + self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples))) + + pollster_definition['name'] = 'test-pollster' + pollster_definition['value_attribute'] = 'name' + pollster_definition['skip_sample_values'] = ['b2'] + pollster = dynamic_pollster.DynamicPollster(pollster_definition) + samples = pollster.get_samples(fake_manager, None, ['http://test.com']) + self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples))) + def test_all_required_fields_ok(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) @@ -112,8 +230,7 @@ class TestDynamicPollster(base.BaseTestCase): False, pollster.pollster_definitions['preserve_mapped_metadata']) def test_all_required_fields_exceptions(self): - for key in dynamic_pollster.\ - DynamicPollster.REQUIRED_POLLSTER_FIELDS: + for key in REQUIRED_POLLSTER_FIELDS: pollster_definition = copy.deepcopy( self.pollster_definition_only_required_fields) pollster_definition.pop(key) @@ -148,7 +265,8 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - self.assertEqual("endpoint:test", pollster.default_discovery) + self.assertEqual("endpoint:test", pollster.definitions.sample_gatherer + .default_discovery) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_get_samples_empty_response(self, client_mock): @@ -161,9 +279,10 @@ class TestDynamicPollster(base.BaseTestCase): client_mock.session.get.return_value = return_value - samples = pollster.execute_request_get_samples( - keystone_client=client_mock, - resource="https://endpoint.server.name/") + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") self.assertEqual(0, len(samples)) @@ -179,9 +298,10 @@ class TestDynamicPollster(base.BaseTestCase): client_mock.session.get.return_value = return_value - samples = pollster.execute_request_get_samples( - keystone_client=client_mock, - resource="https://endpoint.server.name/") + samples = pollster.definitions.sample_gatherer. \ + execute_request_get_samples( + keystone_client=client_mock, + resource="https://endpoint.server.name/") self.assertEqual(3, len(samples)) @@ -197,7 +317,8 @@ class TestDynamicPollster(base.BaseTestCase): client_mock.session.get.return_value = return_value exception = self.assertRaises(requests.HTTPError, - pollster.execute_request_get_samples, + pollster.definitions.sample_gatherer. + execute_request_get_samples, keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual("Mock HTTP error.", str(exception)) @@ -211,7 +332,8 @@ class TestDynamicPollster(base.BaseTestCase): self.pollster_definition_only_required_fields['metadata_mapping'] = {} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - pollster.generate_new_metadata_fields(metadata) + pollster.definitions.sample_extractor.generate_new_metadata_fields( + metadata, self.pollster_definition_only_required_fields) self.assertEqual(metadata_before_call, metadata) @@ -227,7 +349,8 @@ class TestDynamicPollster(base.BaseTestCase): 'preserve_mapped_metadata'] = True pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - pollster.generate_new_metadata_fields(metadata) + pollster.definitions.sample_extractor.generate_new_metadata_fields( + metadata, self.pollster_definition_only_required_fields) self.assertEqual(expected_metadata, metadata) @@ -244,7 +367,8 @@ class TestDynamicPollster(base.BaseTestCase): 'preserve_mapped_metadata'] = False pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - pollster.generate_new_metadata_fields(metadata) + pollster.definitions.sample_extractor.generate_new_metadata_fields( + metadata, self.pollster_definition_only_required_fields) self.assertEqual(expected_clean_metadata, metadata) @@ -255,7 +379,8 @@ class TestDynamicPollster(base.BaseTestCase): value_to_be_mapped = "test" expected_value = value_to_be_mapped - value = pollster.execute_value_mapping(value_to_be_mapped) + value = pollster.definitions.value_mapper. \ + execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) @@ -267,7 +392,8 @@ class TestDynamicPollster(base.BaseTestCase): value_to_be_mapped = "test" expected_value = -1 - value = pollster.execute_value_mapping(value_to_be_mapped) + value = pollster.definitions.value_mapper. \ + execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) @@ -282,7 +408,8 @@ class TestDynamicPollster(base.BaseTestCase): value_to_be_mapped = "test" expected_value = 0 - value = pollster.execute_value_mapping(value_to_be_mapped) + value = pollster.definitions.value_mapper. \ + execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) @@ -294,7 +421,8 @@ class TestDynamicPollster(base.BaseTestCase): value_to_be_mapped = "test" expected_value = 'new-value' - value = pollster.execute_value_mapping(value_to_be_mapped) + value = pollster.definitions.value_mapper. \ + execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) @@ -306,7 +434,7 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(None, next(samples)) @mock.patch('ceilometer.polling.dynamic_pollster.' - 'DynamicPollster.execute_request_get_samples') + 'PollsterSampleGatherer.execute_request_get_samples') def test_get_samples_empty_samples(self, execute_request_get_samples_mock): execute_request_get_samples_mock.side_effect = [] @@ -346,7 +474,7 @@ class TestDynamicPollster(base.BaseTestCase): return samples_list @mock.patch.object( - dynamic_pollster.DynamicPollster, + dynamic_pollster.PollsterSampleGatherer, 'execute_request_get_samples', fake_sample_list) def test_get_samples(self): @@ -383,7 +511,8 @@ class TestDynamicPollster(base.BaseTestCase): self.pollster_definition_only_required_fields) response = [{"object1-attr1": 1}, {"object1-attr2": 2}] - entries = pollster.retrieve_entries_from_response(response) + entries = pollster.definitions.sample_gatherer. \ + retrieve_entries_from_response(response) self.assertEqual(response, entries) @@ -401,7 +530,8 @@ class TestDynamicPollster(base.BaseTestCase): response = {"first": first_entries_from_response, "second": second_entries_from_response} - entries = pollster.retrieve_entries_from_response(response) + entries = pollster.definitions.sample_gatherer. \ + retrieve_entries_from_response(response) self.assertEqual(first_entries_from_response, entries) @@ -418,7 +548,8 @@ class TestDynamicPollster(base.BaseTestCase): response = {"first": first_entries_from_response, "second": second_entries_from_response} - entries = pollster.retrieve_entries_from_response(response) + entries = pollster.definitions.sample_gatherer. \ + retrieve_entries_from_response(response) self.assertEqual(second_entries_from_response, entries) @@ -431,8 +562,8 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - returned_value = pollster.retrieve_attribute_nested_value( - json_object, key) + returned_value = pollster.definitions.sample_extractor.\ + retrieve_attribute_nested_value(json_object, key) self.assertEqual(value, returned_value) @@ -447,8 +578,8 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - returned_value = pollster.retrieve_attribute_nested_value( - json_object, key) + returned_value = pollster.definitions.sample_extractor. \ + retrieve_attribute_nested_value(json_object, key) self.assertEqual(sub_value, returned_value) @@ -465,8 +596,8 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - returned_value = pollster.retrieve_attribute_nested_value( - json_object, key) + returned_value = pollster.definitions.sample_extractor.\ + retrieve_attribute_nested_value(json_object, key) self.assertEqual(expected_value_after_operations, returned_value) @@ -496,7 +627,83 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) - returned_value = pollster.retrieve_attribute_nested_value(json_object, - key) + returned_value = pollster.definitions.sample_extractor.\ + retrieve_attribute_nested_value(json_object, key) self.assertEqual(expected_value_after_operations, returned_value) + + def fake_sample_multi_metric(self, keystone_client=None, resource=None): + multi_metric_sample_list = [ + {"categories": [ + { + "bytes_received": 0, + "bytes_sent": 0, + "category": "create_bucket", + "ops": 2, + "successful_ops": 2 + }, + { + "bytes_received": 0, + "bytes_sent": 2120428, + "category": "get_obj", + "ops": 46, + "successful_ops": 46 + }, + { + "bytes_received": 0, + "bytes_sent": 21484, + "category": "list_bucket", + "ops": 8, + "successful_ops": 8 + }, + { + "bytes_received": 6889056, + "bytes_sent": 0, + "category": "put_obj", + "ops": 46, + "successful_ops": 6 + }], + "total": { + "bytes_received": 6889056, + "bytes_sent": 2141912, + "ops": 102, + "successful_ops": 106 + }, + "user": "test-user"}] + return multi_metric_sample_list + + @mock.patch.object( + dynamic_pollster.PollsterSampleGatherer, + 'execute_request_get_samples', + fake_sample_multi_metric) + def test_get_samples_multi_metric_pollster(self): + pollster = dynamic_pollster.DynamicPollster( + self.multi_metric_pollster_definition) + + fake_manager = self.FakeManager() + samples = pollster.get_samples( + fake_manager, None, ["https://endpoint.server.name.com/"]) + + samples_list = list(samples) + self.assertEqual(4, len(samples_list)) + + create_bucket_sample = [ + s for s in samples_list + if s.name == "test-pollster.create_bucket"][0] + + get_obj_sample = [ + s for s in samples_list + if s.name == "test-pollster.get_obj"][0] + + list_bucket_sample = [ + s for s in samples_list + if s.name == "test-pollster.list_bucket"][0] + + put_obj_sample = [ + s for s in samples_list + if s.name == "test-pollster.put_obj"][0] + + self.assertEqual(2, create_bucket_sample.volume) + self.assertEqual(46, get_obj_sample.volume) + self.assertEqual(8, list_bucket_sample.volume) + self.assertEqual(46, put_obj_sample.volume) diff --git a/ceilometer/tests/unit/polling/test_manager.py b/ceilometer/tests/unit/polling/test_manager.py index 3ac7af7c..322bb3ad 100644 --- a/ceilometer/tests/unit/polling/test_manager.py +++ b/ceilometer/tests/unit/polling/test_manager.py @@ -27,9 +27,10 @@ from stevedore import extension from ceilometer.compute import discovery as nova_discover from ceilometer.hardware import discovery from ceilometer.polling.dynamic_pollster import DynamicPollster +from ceilometer.polling.dynamic_pollster import \ + NonOpenStackApisPollsterDefinition +from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions from ceilometer.polling import manager -from ceilometer.polling.non_openstack_dynamic_pollster import \ - NonOpenStackApisDynamicPollster from ceilometer.polling import plugin_base from ceilometer import sample from ceilometer import service @@ -900,10 +901,10 @@ class TestPollingAgentPartitioned(BaseAgent): 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", 'value_attribute': "volume", 'endpoint_type': "test", 'url_path': "v1/test/endpoint/fake"} - pollster = self.mgr.instantiate_dynamic_pollster( - pollster_definition_only_required_fields) + pollster = DynamicPollster(pollster_definition_only_required_fields) - self.assertIsInstance(pollster, DynamicPollster) + self.assertIsInstance(pollster.definitions, + SingleMetricPollsterDefinitions) def test_instantiate_dynamic_pollster_non_openstack_api(self): pollster_definition_only_required_fields = { @@ -911,7 +912,7 @@ class TestPollingAgentPartitioned(BaseAgent): 'value_attribute': "volume", 'url_path': "v1/test/endpoint/fake", 'module': "module-name", 'authentication_object': "authentication_object"} - pollster = self.mgr.instantiate_dynamic_pollster( - pollster_definition_only_required_fields) + pollster = DynamicPollster(pollster_definition_only_required_fields) - self.assertIsInstance(pollster, NonOpenStackApisDynamicPollster) + self.assertIsInstance(pollster.definitions, + NonOpenStackApisPollsterDefinition) diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py index f0a35b64..2a4d269e 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py @@ -22,13 +22,76 @@ import requests from ceilometer.declarative import DynamicPollsterDefinitionException from ceilometer.declarative import NonOpenStackApisDynamicPollsterException from ceilometer.polling.dynamic_pollster import DynamicPollster -from ceilometer.polling.non_openstack_dynamic_pollster\ - import NonOpenStackApisDynamicPollster +from ceilometer.polling.dynamic_pollster import MultiMetricPollsterDefinitions +from ceilometer.polling.dynamic_pollster import \ + NonOpenStackApisPollsterDefinition +from ceilometer.polling.dynamic_pollster import PollsterSampleGatherer +from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions + from oslotest import base +REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', 'value_attribute', + 'url_path', 'module', 'authentication_object'] + +OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values', + 'value_mapping', 'default_value', + 'metadata_mapping', 'preserve_mapped_metadata', + 'response_entries_key', 'user_id_attribute', + 'resource_id_attribute', 'barbican_secret_id', + 'authentication_parameters', + 'project_id_attribute'] + +ALL_POLLSTER_FIELDS = REQUIRED_POLLSTER_FIELDS + OPTIONAL_POLLSTER_FIELDS + + +def fake_sample_multi_metric(self, keystone_client=None, resource=None): + multi_metric_sample_list = [ + {"user_id": "UID-U007", + "project_id": "UID-P007", + "id": "UID-007", + "categories": [ + { + "bytes_received": 0, + "bytes_sent": 0, + "category": "create_bucket", + "ops": 2, + "successful_ops": 2 + }, + { + "bytes_received": 0, + "bytes_sent": 2120428, + "category": "get_obj", + "ops": 46, + "successful_ops": 46 + }, + { + "bytes_received": 0, + "bytes_sent": 21484, + "category": "list_bucket", + "ops": 8, + "successful_ops": 8 + }, + { + "bytes_received": 6889056, + "bytes_sent": 0, + "category": "put_obj", + "ops": 46, + "successful_ops": 6 + }], + "total": { + "bytes_received": 6889056, + "bytes_sent": 2141912, + "ops": 102, + "successful_ops": 106 + }, + "user": "test-user"}] + return multi_metric_sample_list + class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): + class FakeManager(object): + _keystone = None class FakeResponse(object): status_code = None @@ -42,6 +105,16 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): def setUp(self): super(TestNonOpenStackApisDynamicPollster, self).setUp() + self.pollster_definition_only_openstack_required_single_metric = { + 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", + 'value_attribute': "volume", "endpoint_type": "type", + 'url_path': "v1/test/endpoint/fake"} + + self.pollster_definition_only_openstack_required_multi_metric = { + 'name': "test-pollster.{category}", 'sample_type': "gauge", + 'unit': "test", 'value_attribute': "[categories].ops", + 'url_path': "v1/test/endpoint/fake", "endpoint_type": "type"} + self.pollster_definition_only_required_fields = { 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", 'value_attribute': "volume", @@ -58,10 +131,17 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): 'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id', 'authentication_parameters': 'parameters'} - def test_all_fields(self): - pollster = NonOpenStackApisDynamicPollster( - self.pollster_definition_only_required_fields) + self.pollster_definition_all_fields_multi_metrics = { + 'name': "test-pollster.{category}", 'sample_type': "gauge", + 'unit': "test", 'value_attribute': "[categories].ops", + 'url_path': "v1/test/endpoint/fake", 'module': "module-name", + 'authentication_object': "authentication_object", + 'user_id_attribute': 'user_id', + 'project_id_attribute': 'project_id', + 'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id', + 'authentication_parameters': 'parameters'} + def test_all_fields(self): all_required = ['module', 'authentication_object', 'name', 'sample_type', 'unit', 'value_attribute', 'url_path'] @@ -74,27 +154,27 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): 'response_entries_key'] + all_required for field in all_required: - self.assertIn(field, pollster.REQUIRED_POLLSTER_FIELDS) + self.assertIn(field, REQUIRED_POLLSTER_FIELDS) for field in all_optional: - self.assertIn(field, pollster.ALL_POLLSTER_FIELDS) + self.assertIn(field, ALL_POLLSTER_FIELDS) def test_all_required_fields_exceptions(self): - pollster = NonOpenStackApisDynamicPollster( - self.pollster_definition_only_required_fields) - - for key in pollster.REQUIRED_POLLSTER_FIELDS: + for key in REQUIRED_POLLSTER_FIELDS: + if key == 'module': + continue pollster_definition = copy.deepcopy( self.pollster_definition_only_required_fields) pollster_definition.pop(key) - exception = self.assertRaises(DynamicPollsterDefinitionException, - NonOpenStackApisDynamicPollster, - pollster_definition) + exception = self.assertRaises( + DynamicPollsterDefinitionException, DynamicPollster, + pollster_definition, None, + [NonOpenStackApisPollsterDefinition]) self.assertEqual("Required fields ['%s'] not specified." % key, exception.brief_message) def test_set_default_values(self): - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_only_required_fields) pollster_definitions = pollster.pollster_definitions @@ -106,7 +186,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): self.assertEqual('', pollster_definitions['authentication_parameters']) def test_user_set_optional_parameters(self): - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_all_fields) pollster_definitions = pollster.pollster_definitions @@ -122,23 +202,25 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): pollster_definitions['authentication_parameters']) def test_default_discovery_empty_secret_id(self): - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_only_required_fields) - self.assertEqual("barbican:", pollster.default_discovery) + self.assertEqual("barbican:", pollster.definitions.sample_gatherer. + default_discovery) def test_default_discovery_not_empty_secret_id(self): - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_all_fields) - self.assertEqual("barbican:barbican_id", pollster.default_discovery) + self.assertEqual("barbican:barbican_id", pollster.definitions. + sample_gatherer.default_discovery) @mock.patch('requests.get') def test_internal_execute_request_get_samples_status_code_ok( self, get_mock): sys.modules['module-name'] = mock.MagicMock() - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_only_required_fields) return_value = self.FakeResponse() @@ -150,7 +232,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): kwargs = {'resource': "credentials"} - resp, url = pollster.internal_execute_request_get_samples(kwargs) + resp, url = pollster.definitions.sample_gatherer. \ + internal_execute_request_get_samples(kwargs) self.assertEqual( self.pollster_definition_only_required_fields['url_path'], url) @@ -161,7 +244,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): self, get_mock): sys.modules['module-name'] = mock.MagicMock() - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_only_required_fields) for http_status_code in requests.status_codes._codes.keys(): @@ -177,7 +260,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): kwargs = {'resource': "credentials"} exception = self.assertRaises( NonOpenStackApisDynamicPollsterException, - pollster.internal_execute_request_get_samples, kwargs) + pollster.definitions.sample_gatherer. + internal_execute_request_get_samples, kwargs) self.assertEqual( "NonOpenStackApisDynamicPollsterException" @@ -188,25 +272,28 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): http_status_code, return_value.reason), str(exception)) def test_generate_new_attributes_in_sample_attribute_key_none(self): - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_only_required_fields) sample = {"test": "2"} new_key = "new-key" - pollster.generate_new_attributes_in_sample(sample, None, new_key) - pollster.generate_new_attributes_in_sample(sample, "", new_key) + pollster.definitions.sample_gatherer. \ + generate_new_attributes_in_sample(sample, None, new_key) + pollster.definitions.sample_gatherer. \ + generate_new_attributes_in_sample(sample, "", new_key) self.assertNotIn(new_key, sample) def test_generate_new_attributes_in_sample(self): - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_only_required_fields) sample = {"test": "2"} new_key = "new-key" - pollster.generate_new_attributes_in_sample(sample, "test", new_key) + pollster.definitions.sample_gatherer. \ + generate_new_attributes_in_sample(sample, "test", new_key) self.assertIn(new_key, sample) self.assertEqual(sample["test"], sample[new_key]) @@ -220,7 +307,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): samples = [sample] return samples - DynamicPollster.execute_request_get_samples =\ + PollsterSampleGatherer.execute_request_get_samples = \ execute_request_get_samples_mock self.pollster_definition_all_fields[ @@ -230,11 +317,12 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): self.pollster_definition_all_fields[ 'resource_id_attribute'] = 'resource_id_attribute' - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_all_fields) params = {"d": "d"} - response = pollster.execute_request_get_samples(**params) + response = pollster.definitions.sample_gatherer. \ + execute_request_get_samples(**params) self.assertEqual(sample['user_id_attribute'], response[0]['user_id']) @@ -252,7 +340,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): samples = [sample] return samples - DynamicPollster.execute_request_get_samples =\ + DynamicPollster.execute_request_get_samples = \ execute_request_get_samples_mock self.pollster_definition_all_fields[ @@ -262,7 +350,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): self.pollster_definition_all_fields[ 'resource_id_attribute'] = None - pollster = NonOpenStackApisDynamicPollster( + pollster = DynamicPollster( self.pollster_definition_all_fields) params = {"d": "d"} @@ -271,3 +359,67 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): self.assertNotIn('user_id', response[0]) self.assertNotIn('project_id', response[0]) self.assertNotIn('id', response[0]) + + def test_pollster_defintions_instantiation(self): + def validate_definitions_instance(instance, isNonOpenstack, + isMultiMetric, isSingleMetric): + self.assertIs( + isinstance(instance, NonOpenStackApisPollsterDefinition), + isNonOpenstack) + self.assertIs(isinstance(instance, MultiMetricPollsterDefinitions), + isMultiMetric) + self.assertIs( + isinstance(instance, SingleMetricPollsterDefinitions), + isSingleMetric) + + pollster = DynamicPollster( + self.pollster_definition_all_fields_multi_metrics) + validate_definitions_instance(pollster.definitions, True, True, False) + + pollster = DynamicPollster( + self.pollster_definition_all_fields) + validate_definitions_instance(pollster.definitions, True, False, True) + + pollster = DynamicPollster( + self.pollster_definition_only_openstack_required_multi_metric) + validate_definitions_instance(pollster.definitions, False, True, False) + + pollster = DynamicPollster( + self.pollster_definition_only_openstack_required_single_metric) + validate_definitions_instance(pollster.definitions, False, False, True) + + @mock.patch.object( + PollsterSampleGatherer, + 'execute_request_get_samples', + fake_sample_multi_metric) + def test_get_samples_multi_metric_pollster(self): + pollster = DynamicPollster( + self.pollster_definition_all_fields_multi_metrics) + + fake_manager = self.FakeManager() + samples = pollster.get_samples( + fake_manager, None, ["https://endpoint.server.name.com/"]) + + samples_list = list(samples) + self.assertEqual(4, len(samples_list)) + + create_bucket_sample = [ + s for s in samples_list + if s.name == "test-pollster.create_bucket"][0] + + get_obj_sample = [ + s for s in samples_list + if s.name == "test-pollster.get_obj"][0] + + list_bucket_sample = [ + s for s in samples_list + if s.name == "test-pollster.list_bucket"][0] + + put_obj_sample = [ + s for s in samples_list + if s.name == "test-pollster.put_obj"][0] + + self.assertEqual(2, create_bucket_sample.volume) + self.assertEqual(46, get_obj_sample.volume) + self.assertEqual(8, list_bucket_sample.volume) + self.assertEqual(46, put_obj_sample.volume) diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst index 2c829836..0ea50ace 100644 --- a/doc/source/admin/telemetry-dynamic-pollster.rst +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -376,3 +376,210 @@ following: * user_id_attribute * project_id_attribute * resource_id_attribute + +Multi metric dynamic pollsters (handling attribute values with list of objects) +------------------------------------------------------------------------------- + +The initial idea for this feature comes from the `categories` fields that we +can find in the `summary` object of the RadosGW API. Each user has a +`categories` attribute in the response; in the `categories` list, we can find +the object that presents in a granular fashion the consumption of different +RadosGW API operations such as GET, PUT, POST, and may others. + +As follows we present an example of such a JSON response. + +.. code-block:: json + + { + "entries": [ + { + "buckets": [ + { + "bucket": "", + "categories": [ + { + "bytes_received": 0, + "bytes_sent": 40, + "category": "list_buckets", + "ops": 2, + "successful_ops": 2 + } + ], + "epoch": 1572969600, + "owner": "user", + "time": "2019-11-21 00:00:00.000000Z" + }, + { + "bucket": "-", + "categories": [ + { + "bytes_received": 0, + "bytes_sent": 0, + "category": "get_obj", + "ops": 1, + "successful_ops": 0 + } + ], + "epoch": 1572969600, + "owner": "someOtherUser", + "time": "2019-11-21 00:00:00.000000Z" + } + ] + } + ] + "summary": [ + { + "categories": [ + { + "bytes_received": 0, + "bytes_sent": 0, + "category": "create_bucket", + "ops": 2, + "successful_ops": 2 + }, + { + "bytes_received": 0, + "bytes_sent": 2120428, + "category": "get_obj", + "ops": 46, + "successful_ops": 46 + }, + { + "bytes_received": 0, + "bytes_sent": 21484, + "category": "list_bucket", + "ops": 8, + "successful_ops": 8 + }, + { + "bytes_received": 6889056, + "bytes_sent": 0, + "category": "put_obj", + "ops": 46, + "successful_ops": 46 + } + ], + "total": { + "bytes_received": 6889056, + "bytes_sent": 2141912, + "ops": 102, + "successful_ops": 102 + }, + "user": "user" + }, + { + "categories": [ + { + "bytes_received": 0, + "bytes_sent": 0, + "category": "create_bucket", + "ops": 1, + "successful_ops": 1 + }, + { + "bytes_received": 0, + "bytes_sent": 0, + "category": "delete_obj", + "ops": 23, + "successful_ops": 23 + }, + { + "bytes_received": 0, + "bytes_sent": 5371, + "category": "list_bucket", + "ops": 2, + "successful_ops": 2 + }, + { + "bytes_received": 3444350, + "bytes_sent": 0, + "category": "put_obj", + "ops": 23, + "successful_ops": 23 + } + ], + "total": { + "bytes_received": 3444350, + "bytes_sent": 5371, + "ops": 49, + "successful_ops": 49 + }, + "user": "someOtherUser" + } + ] + } + +In that context, and having in mind that we have APIs with similar data +structures, we developed an extension for the dynamic pollster that enables +multi-metric processing for a single pollster. It works as follows. + +The pollster name will contain a placeholder for the variable that +identifies the "submetric". E.g. `dynamic.radosgw.api.request.{category}`. +The placeholder `{category}` indicates the object's attribute that is in the +list of objects that we use to load the sub metric name. Then, we must use a +special notation in the `value_attribute` configuration to indicate that we are +dealing with a list of objects. This is achieved via `[]` (brackets); for +instance, in the `dynamic.radosgw.api.request.{category}`, we can use +`[categories].ops` as the `value_attribute`. This indicates that the value we +retrieve is a list of objects, and when the dynamic pollster processes it, we +want it (the pollster) to load the `ops` value for the sub metrics being +generated. + +Examples on how to create multi-metric pollster to handle data from RadosGW API +are presented as follows: + +.. code-block:: yaml + + --- + + - name: "dynamic.radosgw.api.request.{category}" + sample_type: "gauge" + unit: "request" + value_attribute: "[categories].ops" + url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage" + module: "awsauth" + authentication_object: "S3Auth" + authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>" + user_id_attribute: "user | value.split('$')[0]" + project_id_attribute: "user | value.split('$') | value[0]" + resource_id_attribute: "user | value.split('$') | value[0]" + response_entries_key: "summary" + + - name: "dynamic.radosgw.api.request.successful_ops.{category}" + sample_type: "gauge" + unit: "request" + value_attribute: "[categories].successful_ops" + url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage" + module: "awsauth" + authentication_object: "S3Auth" + authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>" + user_id_attribute: "user | value.split('$')[0]" + project_id_attribute: "user | value.split('$') | value[0]" + resource_id_attribute: "user | value.split('$') | value[0]" + response_entries_key: "summary" + + - name: "dynamic.radosgw.api.bytes_sent.{category}" + sample_type: "gauge" + unit: "request" + value_attribute: "[categories].bytes_sent" + url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage" + module: "awsauth" + authentication_object: "S3Auth" + authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>" + user_id_attribute: "user | value.split('$')[0]" + project_id_attribute: "user | value.split('$') | value[0]" + resource_id_attribute: "user | value.split('$') | value[0]" + response_entries_key: "summary" + + - name: "dynamic.radosgw.api.bytes_received.{category}" + sample_type: "gauge" + unit: "request" + value_attribute: "[categories].bytes_received" + url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage" + module: "awsauth" + authentication_object: "S3Auth" + authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>" + user_id_attribute: "user | value.split('$')[0]" + project_id_attribute: "user | value.split('$') | value[0]" + resource_id_attribute: "user | value.split('$') | value[0]" + response_entries_key: "summary" |