diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..a914011a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stackhpc/openstack diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml new file mode 100644 index 00000000..4ef4d261 --- /dev/null +++ b/.github/workflows/tag-and-release.yml @@ -0,0 +1,12 @@ +--- +name: Tag & Release +'on': + push: + branches: + - stackhpc/yoga +permissions: + actions: read + contents: write +jobs: + tag-and-release: + uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml new file mode 100644 index 00000000..8713f0e0 --- /dev/null +++ b/.github/workflows/tox.yml @@ -0,0 +1,7 @@ +--- +name: Tox Continuous Integration +'on': + pull_request: +jobs: + tox: + uses: stackhpc/.github/.github/workflows/tox.yml@main diff --git a/cloudkitty/collector/__init__.py b/cloudkitty/collector/__init__.py index 19e17891..827d66ec 100644 --- a/cloudkitty/collector/__init__.py +++ b/cloudkitty/collector/__init__.py @@ -93,7 +93,9 @@ def MetricDict(value): # (NONE, NUMBOOL, NOTNUMBOOL, FLOOR, CEIL). # Defaults to NONE Required('mutate', default='NONE'): - In(['NONE', 'NUMBOOL', 'NOTNUMBOOL', 'FLOOR', 'CEIL']), + In(['NONE', 'NUMBOOL', 'NOTNUMBOOL', 'FLOOR', 'CEIL', 'MAP']), + # Map dict used if mutate == 'MAP' + Optional('mutate_map'): dict, # Collector-specific args. Should be overriden by schema provided for # the given collector Optional('extra_args'): dict, @@ -270,6 +272,22 @@ def check_duplicates(metric_name, metric): return metric +def validate_map_mutator(metric_name, metric): + """Validates MAP mutator""" + mutate = metric.get('mutate') + mutate_map = metric.get('mutate_map') + + if mutate == 'MAP' and mutate_map is None: + raise InvalidConfiguration( + 'Metric {} uses MAP mutator but mutate_map is missing: {}'.format( + metric_name, metric)) + + if mutate != 'MAP' and mutate_map is not None: + raise InvalidConfiguration( + 'Metric {} not using MAP mutator but mutate_map is present: ' + '{}'.format(metric_name, metric)) + + def validate_conf(conf): """Validates the provided configuration.""" collector = get_collector_without_invoke() @@ -278,4 +296,5 @@ def validate_conf(conf): if 'alt_name' not in metric.keys(): metric['alt_name'] = metric_name check_duplicates(metric_name, metric) + validate_map_mutator(metric_name, metric) return output diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py index da17d400..b7e21cbf 100644 --- a/cloudkitty/collector/gnocchi.py +++ b/cloudkitty/collector/gnocchi.py @@ -434,7 +434,9 @@ def _format_data(self, metconf, data, resources_info=None): qty = data['measures']['measures']['aggregated'][0][2] converted_qty = ck_utils.convert_unit( qty, metconf['factor'], metconf['offset']) - mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate']) + mutate_map = metconf.get('mutate_map') + mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'], + mutate_map=mutate_map) return metadata, groupby, mutated_qty def fetch_all(self, metric_name, start, end, diff --git a/cloudkitty/collector/monasca.py b/cloudkitty/collector/monasca.py index 31a642f4..4b11c087 100644 --- a/cloudkitty/collector/monasca.py +++ b/cloudkitty/collector/monasca.py @@ -199,7 +199,9 @@ def _format_data(self, metconf, data, resources_info=None): qty = data['statistics'][0][1] converted_qty = ck_utils.convert_unit( qty, metconf['factor'], metconf['offset']) - mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate']) + mutate_map = metconf.get('mutate_map') + mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'], + mutate_map=mutate_map) return metadata, groupby, mutated_qty def fetch_all(self, metric_name, start, end, diff --git a/cloudkitty/collector/prometheus.py b/cloudkitty/collector/prometheus.py index 8217fef3..572676a7 100644 --- a/cloudkitty/collector/prometheus.py +++ b/cloudkitty/collector/prometheus.py @@ -148,7 +148,9 @@ def _format_data(self, metric_name, scope_key, scope_id, start, end, data): self.conf[metric_name]['factor'], self.conf[metric_name]['offset'], ) - qty = ck_utils.mutate(qty, self.conf[metric_name]['mutate']) + mutate_map = self.conf[metric_name].get('mutate_map') + qty = ck_utils.mutate(qty, self.conf[metric_name]['mutate'], + mutate_map=mutate_map) return metadata, groupby, qty @@ -211,6 +213,8 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): if query_suffix: query = "{0} {1}".format(query, query_suffix) + LOG.debug("Calling Prometheus with query: %s", query) + try: res = self._conn.get_instant( query, diff --git a/cloudkitty/storage/v2/elasticsearch/client.py b/cloudkitty/storage/v2/elasticsearch/client.py index b0b1032d..93db3c18 100644 --- a/cloudkitty/storage/v2/elasticsearch/client.py +++ b/cloudkitty/storage/v2/elasticsearch/client.py @@ -98,14 +98,15 @@ def _build_composite(self, groupby): sources = [] for elem in groupby: if elem == 'type': - sources.append({'type': {'terms': {'field': 'type'}}}) + sources.append({'type': {'terms': {'field': 'type.keyword'}}}) elif elem == 'time': # Not doing a date_histogram aggregation because we don't know # the period sources.append({'begin': {'terms': {'field': 'start'}}}) sources.append({'end': {'terms': {'field': 'end'}}}) else: - sources.append({elem: {'terms': {'field': 'groupby.' + elem}}}) + field = 'groupby.' + elem + '.keyword' + sources.append({elem: {'terms': {'field': field}}}) return {"sources": sources} @@ -158,12 +159,9 @@ def put_mapping(self, mapping): :rtype: requests.models.Response """ url = '/'.join( - (self._url, self._index_name, '_mapping', self._mapping_name)) - # NOTE(peschk_l): This is done for compatibility with - # Elasticsearch 6 and 7. - param = {"include_type_name": "true"} + (self._url, self._index_name, self._mapping_name)) return self._req( - self._sess.put, url, json.dumps(mapping), param, deserialize=False) + self._sess.post, url, json.dumps(mapping), {}, deserialize=False) def get_index(self): """Does a GET request against ES's index API. @@ -228,7 +226,7 @@ def bulk_with_instruction(self, instruction, terms): """Does a POST request against ES's bulk API The POST request will be done against - `///_bulk` + `//_bulk` The instruction will be appended before each term. For example, bulk_with_instruction('instr', ['one', 'two']) will produce:: @@ -249,7 +247,7 @@ def bulk_with_instruction(self, instruction, terms): *[(instruction, json.dumps(term)) for term in terms] )) + '\n' url = '/'.join( - (self._url, self._index_name, self._mapping_name, '_bulk')) + (self._url, self._index_name, '_bulk')) return self._req(self._sess.post, url, data, None, deserialize=False) def bulk_index(self, terms): diff --git a/cloudkitty/tests/collectors/test_validation.py b/cloudkitty/tests/collectors/test_validation.py index 20ebf0f7..4a576b63 100644 --- a/cloudkitty/tests/collectors/test_validation.py +++ b/cloudkitty/tests/collectors/test_validation.py @@ -188,3 +188,26 @@ def test_check_duplicates(self): self.assertRaises( collector.InvalidConfiguration, collector.check_duplicates, metric_name, metric) + + def test_validate_map_mutator(self): + data = copy.deepcopy(self.base_data) + + # Check that validation succeeds when MAP mutator is not used + for metric_name, metric in data['metrics'].items(): + collector.validate_map_mutator(metric_name, metric) + + # Check that validation raises an exception when mutate_map is missing + for metric_name, metric in data['metrics'].items(): + metric['mutate'] = 'MAP' + self.assertRaises( + collector.InvalidConfiguration, + collector.validate_map_mutator, metric_name, metric) + + data = copy.deepcopy(self.base_data) + # Check that validation raises an exception when mutate_map is present + # but MAP mutator is not used + for metric_name, metric in data['metrics'].items(): + metric['mutate_map'] = {} + self.assertRaises( + collector.InvalidConfiguration, + collector.validate_map_mutator, metric_name, metric) diff --git a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py index 8948be91..7c95db0f 100644 --- a/cloudkitty/tests/storage/v2/elasticsearch/test_client.py +++ b/cloudkitty/tests/storage/v2/elasticsearch/test_client.py @@ -86,9 +86,9 @@ def test_build_composite(self): self.assertEqual( self.client._build_composite(['one', 'type', 'two']), {'sources': [ - {'one': {'terms': {'field': 'groupby.one'}}}, - {'type': {'terms': {'field': 'type'}}}, - {'two': {'terms': {'field': 'groupby.two'}}}, + {'one': {'terms': {'field': 'groupby.one.keyword'}}}, + {'type': {'terms': {'field': 'type.keyword'}}}, + {'two': {'terms': {'field': 'groupby.two.keyword'}}}, ]}, ) @@ -186,9 +186,9 @@ def test_put_mapping(self): with mock.patch.object(self.client, '_req') as rmock: self.client.put_mapping(mapping) rmock.assert_called_once_with( - self.client._sess.put, - 'http://elasticsearch:9200/index_name/_mapping/test_mapping', - '{"a": "b"}', {'include_type_name': 'true'}, deserialize=False) + self.client._sess.post, + 'http://elasticsearch:9200/index_name/test_mapping', + '{"a": "b"}', {}, deserialize=False) def test_get_index(self): with mock.patch.object(self.client, '_req') as rmock: @@ -259,7 +259,7 @@ def test_bulk_with_instruction(self): self.client.bulk_with_instruction(instruction, terms) rmock.assert_called_once_with( self.client._sess.post, - 'http://elasticsearch:9200/index_name/test_mapping/_bulk', + 'http://elasticsearch:9200/index_name/_bulk', expected_data, None, deserialize=False) def test_bulk_index(self): diff --git a/cloudkitty/utils/__init__.py b/cloudkitty/utils/__init__.py index d345d2d9..eb503f24 100644 --- a/cloudkitty/utils/__init__.py +++ b/cloudkitty/utils/__init__.py @@ -251,8 +251,8 @@ def tempdir(**kwargs): LOG.debug('Could not remove tmpdir: %s', e) -def mutate(value, mode='NONE'): - """Mutate value according provided mode.""" +def mutate(value, mode='NONE', mutate_map=None): + """Mutate value according to provided mode.""" if mode == 'NUMBOOL': return float(value != 0.0) @@ -266,6 +266,12 @@ def mutate(value, mode='NONE'): if mode == 'CEIL': return math.ceil(value) + if mode == 'MAP': + ret = 0.0 + if mutate_map is not None: + ret = mutate_map.get(value, 0.0) + return ret + return value diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 199c83cd..349943f7 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -323,11 +323,11 @@ function install_elasticsearch_fedora { sudo yum localinstall -y ${opensearch_file} } -function install_elasticsearch { +function install_opensearch { if is_ubuntu; then - install_elasticsearch_ubuntu + install_opensearch_ubuntu elif is_fedora; then - install_elasticsearch_fedora + install_opensearch_fedora else die $LINENO "Distribution must be Debian or Fedora-based" fi diff --git a/doc/source/admin/configuration/collector.rst b/doc/source/admin/configuration/collector.rst index 8c334a33..42afc796 100644 --- a/doc/source/admin/configuration/collector.rst +++ b/doc/source/admin/configuration/collector.rst @@ -177,7 +177,7 @@ Quantity mutation ~~~~~~~~~~~~~~~~~ It is also possible to mutate the collected qty with the ``mutate`` option. -Four values are accepted for this parameter: +Five values are accepted for this parameter: * ``NONE``: This is the default. The collected data is not modifed. @@ -190,6 +190,11 @@ Four values are accepted for this parameter: * ``NOTNUMBOOL``: If the collected qty equals 0, set it to 1. Else, set it to 0. +* ``MAP``: Map arbritrary values to new values as defined through the + ``mutate_map`` option (dictionary). If the value is not found in + ``mutate_map``, set it to 0. If ``mutate_map`` is not defined or is empty, + all values are set to 0. + .. warning:: Quantity mutation is done **after** conversion. Example:: @@ -233,6 +238,26 @@ when the instance is in ACTIVE state but 4 if the instance is in ERROR state: metadata: - flavor_id +The ``MAP`` mutator is useful when multiple statuses should be billabled. For +example, the following Prometheus metric has a value of 0 when the instance is +in ACTIVE state, but operators may want to rate other non-zero states: + +.. code-block:: yaml + + metrics: + openstack_nova_server_status: + unit: instance + mutate: MAP + mutate_map: + 0.0: 1.0 # ACTIVE + 11.0: 1.0 # SHUTOFF + 12.0: 1.0 # SUSPENDED + 16.0: 1.0 # PAUSED + groupby: + - id + metadata: + - flavor_id + Display name ~~~~~~~~~~~~ diff --git a/releasenotes/notes/map-mutator-632b8629c0482e94.yaml b/releasenotes/notes/map-mutator-632b8629c0482e94.yaml new file mode 100644 index 00000000..2156b5d0 --- /dev/null +++ b/releasenotes/notes/map-mutator-632b8629c0482e94.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds a ``MAP`` mutator to map arbitrary values to new values. This is + useful with metrics reporting resource status as their value, but multiple + statuses are billable.