From 06f8aa712a780fb9547aa80beb627ab540bbb65a Mon Sep 17 00:00:00 2001 From: Dantali0n Date: Fri, 24 May 2019 16:18:02 +0200 Subject: [PATCH] Implement the configuration for Grafana datasource This implements the configuration parameters to implement Grafana as a datasource including the influxdb translator Change-Id: If1f27dc01e853c5b24bdb21f1e810f64eaee2e5c Partially-implements: blueprint grafana-proxy-datasource --- watcher/conf/__init__.py | 4 + watcher/conf/grafana_client.py | 138 +++++++++++++++++++++++++++ watcher/conf/grafana_translators.py | 44 +++++++++ watcher/tests/conf/test_list_opts.py | 9 +- 4 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 watcher/conf/grafana_client.py create mode 100644 watcher/conf/grafana_translators.py diff --git a/watcher/conf/__init__.py b/watcher/conf/__init__.py index 17b7293b6..3bdc120a8 100755 --- a/watcher/conf/__init__.py +++ b/watcher/conf/__init__.py @@ -31,6 +31,8 @@ from watcher.conf import decision_engine from watcher.conf import exception from watcher.conf import glance_client from watcher.conf import gnocchi_client +from watcher.conf import grafana_client +from watcher.conf import grafana_translators from watcher.conf import ironic_client from watcher.conf import keystone_client from watcher.conf import monasca_client @@ -57,6 +59,8 @@ nova_client.register_opts(CONF) glance_client.register_opts(CONF) gnocchi_client.register_opts(CONF) keystone_client.register_opts(CONF) +grafana_client.register_opts(CONF) +grafana_translators.register_opts(CONF) cinder_client.register_opts(CONF) ceilometer_client.register_opts(CONF) neutron_client.register_opts(CONF) diff --git a/watcher/conf/grafana_client.py b/watcher/conf/grafana_client.py new file mode 100644 index 000000000..47910c77b --- /dev/null +++ b/watcher/conf/grafana_client.py @@ -0,0 +1,138 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2019 European Organization for Nuclear Research (CERN) +# +# Authors: Corne Lukken +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +grafana_client = cfg.OptGroup(name='grafana_client', + title='Configuration Options for Grafana') + +GRAFANA_CLIENT_OPTS = [ + # TODO(Dantali0n) each individual metric could have its own token. + # A similar structure to the database_map would solve this. + cfg.StrOpt('token', + default=None, + help="Authentication token to gain access"), + # TODO(Dantali0n) each individual metric could have its own base url. + # A similar structure to the database_map would solve this. + cfg.StrOpt('base_url', + default=None, + help="first part of the url (including https:// or http://) up " + "until project id part. " + "Example: https://secure.org/api/datasource/proxy/"), + cfg.DictOpt('project_id_map', + default={ + 'host_cpu_usage': None, + 'host_ram_usage': None, + 'host_outlet_temp': None, + 'host_inlet_temp': None, + 'host_airflow': None, + 'host_power': None, + 'instance_cpu_usage': None, + 'instance_ram_usage': None, + 'instance_ram_allocated': None, + 'instance_l3_cache_usage': None, + 'instance_root_disk_size': None, + }, + help="Mapping of grafana project ids to datasource metrics. " + "Dictionary values should be positive integers. " + "Example: 7465"), + cfg.DictOpt('database_map', + default={ + 'host_cpu_usage': None, + 'host_ram_usage': None, + 'host_outlet_temp': None, + 'host_inlet_temp': None, + 'host_airflow': None, + 'host_power': None, + 'instance_cpu_usage': None, + 'instance_ram_usage': None, + 'instance_ram_allocated': None, + 'instance_l3_cache_usage': None, + 'instance_root_disk_size': None, + }, + help="Mapping of grafana databases to datasource metrics. " + "Values should be strings. Example: influx_production"), + cfg.DictOpt('attribute_map', + default={ + 'host_cpu_usage': None, + 'host_ram_usage': None, + 'host_outlet_temp': None, + 'host_inlet_temp': None, + 'host_airflow': None, + 'host_power': None, + 'instance_cpu_usage': None, + 'instance_ram_usage': None, + 'instance_ram_allocated': None, + 'instance_l3_cache_usage': None, + 'instance_root_disk_size': None, + }, + help="Mapping of resource attributes to datasource metrics. " + "For a complete list of available attributes see " + "instance.py and node.py in " + "decision_engine/model/element. " + "Values should be strings. Example: hostname"), + cfg.DictOpt('translator_map', + default={ + 'host_cpu_usage': None, + 'host_ram_usage': None, + 'host_outlet_temp': None, + 'host_inlet_temp': None, + 'host_airflow': None, + 'host_power': None, + 'instance_cpu_usage': None, + 'instance_ram_usage': None, + 'instance_ram_allocated': None, + 'instance_l3_cache_usage': None, + 'instance_root_disk_size': None, + }, + help="Mapping of grafana translators to datasource metrics. " + "Values should be strings. Example: influxdb"), + cfg.DictOpt('query_map', + # {0} = aggregate + # {1} = attribute + # {2} = period + # {3} = granularity + # {4} = { influxdb: retention_period, } + default={ + 'host_cpu_usage': None, + 'host_ram_usage': None, + 'host_outlet_temp': None, + 'host_inlet_temp': None, + 'host_airflow': None, + 'host_power': None, + 'instance_cpu_usage': None, + 'instance_ram_usage': None, + 'instance_ram_allocated': None, + 'instance_l3_cache_usage': None, + 'instance_root_disk_size': None, + }, + help="Mapping of grafana queries to datasource metrics. " + "Values should be strings for which the .format method " + "will transform it. These queries will need to be " + "constructed using tools such as Postman. " + "Example: SELECT cpu FROM {4}.cpu_percent WHERE " + "host == '{1}' AND time > now()-{2}s")] + + +def register_opts(conf): + conf.register_group(grafana_client) + conf.register_opts(GRAFANA_CLIENT_OPTS, group=grafana_client) + + +def list_opts(): + return [('grafana_client', GRAFANA_CLIENT_OPTS)] diff --git a/watcher/conf/grafana_translators.py b/watcher/conf/grafana_translators.py new file mode 100644 index 000000000..8755eee9b --- /dev/null +++ b/watcher/conf/grafana_translators.py @@ -0,0 +1,44 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2019 European Organization for Nuclear Research (CERN) +# +# Authors: Corne Lukken +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +grafana_translators = cfg.OptGroup(name='grafana_translators', + title='Configuration Options for Grafana ' + 'transalators') + +GRAFANA_TRANSLATOR_INFLUX_OPTS = [ + cfg.DictOpt('retention_periods', + default={ + 'one_week': 604800, + 'one_month': 2592000, + 'five_years': 31556952 + }, + help="Keys are the names of retention periods in InfluxDB and " + "the values should correspond with the maximum time they " + "can retain in seconds. Example: {'one_day': 86400}")] + + +def register_opts(conf): + conf.register_group(grafana_translators) + conf.register_opts(GRAFANA_TRANSLATOR_INFLUX_OPTS, + group=grafana_translators) + + +def list_opts(): + return [('grafana_translators', GRAFANA_TRANSLATOR_INFLUX_OPTS)] diff --git a/watcher/tests/conf/test_list_opts.py b/watcher/tests/conf/test_list_opts.py index a54670ac1..f6a63d410 100755 --- a/watcher/tests/conf/test_list_opts.py +++ b/watcher/tests/conf/test_list_opts.py @@ -30,10 +30,11 @@ class TestListOpts(base.TestCase): self.base_sections = [ 'DEFAULT', 'api', 'database', 'watcher_decision_engine', 'watcher_applier', 'watcher_datasources', 'watcher_planner', - 'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client', - 'ceilometer_client', 'monasca_client', 'ironic_client', - 'keystone_client', 'neutron_client', 'watcher_clients_auth', - 'collector', 'placement_client'] + 'nova_client', 'glance_client', 'gnocchi_client', 'grafana_client', + 'grafana_translators', 'cinder_client', 'ceilometer_client', + 'monasca_client', 'ironic_client', 'keystone_client', + 'neutron_client', 'watcher_clients_auth', 'collector', + 'placement_client'] self.opt_sections = list(dict(opts.list_opts()).keys()) def test_run_list_opts(self):