Implement the configuration for Grafana datasource

This implements the configuration parameters to implement
Grafana as a datasource including the influxdb translator

Change-Id: If1f27dc01e853c5b24bdb21f1e810f64eaee2e5c
Partially-implements: blueprint grafana-proxy-datasource

watcher/conf/__init__.py

@@ -31,6 +31,8 @@ from watcher.conf import decision_engine
 from watcher.conf import exception
 from watcher.conf import glance_client
 from watcher.conf import gnocchi_client
+from watcher.conf import grafana_client
+from watcher.conf import grafana_translators
 from watcher.conf import ironic_client
 from watcher.conf import keystone_client
 from watcher.conf import monasca_client
@@ -57,6 +59,8 @@ nova_client.register_opts(CONF)
 glance_client.register_opts(CONF)
 gnocchi_client.register_opts(CONF)
 keystone_client.register_opts(CONF)
+grafana_client.register_opts(CONF)
+grafana_translators.register_opts(CONF)
 cinder_client.register_opts(CONF)
 ceilometer_client.register_opts(CONF)
 neutron_client.register_opts(CONF)

watcher/conf/grafana_client.py

@@ -0,0 +1,138 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 European Organization for Nuclear Research (CERN)
+#
+# Authors: Corne Lukken <info@dantalion.nl>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+grafana_client = cfg.OptGroup(name='grafana_client',
+                              title='Configuration Options for Grafana')
+
+GRAFANA_CLIENT_OPTS = [
+    # TODO(Dantali0n) each individual metric could have its own token.
+    # A similar structure to the database_map would solve this.
+    cfg.StrOpt('token',
+               default=None,
+               help="Authentication token to gain access"),
+    # TODO(Dantali0n) each individual metric could have its own base url.
+    # A similar structure to the database_map would solve this.
+    cfg.StrOpt('base_url',
+               default=None,
+               help="first part of the url (including https:// or http://) up "
+                    "until project id part. "
+                    "Example: https://secure.org/api/datasource/proxy/"),
+    cfg.DictOpt('project_id_map',
+                default={
+                    'host_cpu_usage': None,
+                    'host_ram_usage': None,
+                    'host_outlet_temp': None,
+                    'host_inlet_temp': None,
+                    'host_airflow': None,
+                    'host_power': None,
+                    'instance_cpu_usage': None,
+                    'instance_ram_usage': None,
+                    'instance_ram_allocated': None,
+                    'instance_l3_cache_usage': None,
+                    'instance_root_disk_size': None,
+                },
+                help="Mapping of grafana project ids to datasource metrics. "
+                     "Dictionary values should be positive integers. "
+                     "Example: 7465"),
+    cfg.DictOpt('database_map',
+                default={
+                    'host_cpu_usage': None,
+                    'host_ram_usage': None,
+                    'host_outlet_temp': None,
+                    'host_inlet_temp': None,
+                    'host_airflow': None,
+                    'host_power': None,
+                    'instance_cpu_usage': None,
+                    'instance_ram_usage': None,
+                    'instance_ram_allocated': None,
+                    'instance_l3_cache_usage': None,
+                    'instance_root_disk_size': None,
+                },
+                help="Mapping of grafana databases to datasource metrics. "
+                     "Values should be strings. Example: influx_production"),
+    cfg.DictOpt('attribute_map',
+                default={
+                    'host_cpu_usage': None,
+                    'host_ram_usage': None,
+                    'host_outlet_temp': None,
+                    'host_inlet_temp': None,
+                    'host_airflow': None,
+                    'host_power': None,
+                    'instance_cpu_usage': None,
+                    'instance_ram_usage': None,
+                    'instance_ram_allocated': None,
+                    'instance_l3_cache_usage': None,
+                    'instance_root_disk_size': None,
+                },
+                help="Mapping of resource attributes to datasource metrics. "
+                     "For a complete list of available attributes see "
+                     "instance.py and node.py in "
+                     "decision_engine/model/element. "
+                     "Values should be strings. Example: hostname"),
+    cfg.DictOpt('translator_map',
+                default={
+                    'host_cpu_usage': None,
+                    'host_ram_usage': None,
+                    'host_outlet_temp': None,
+                    'host_inlet_temp': None,
+                    'host_airflow': None,
+                    'host_power': None,
+                    'instance_cpu_usage': None,
+                    'instance_ram_usage': None,
+                    'instance_ram_allocated': None,
+                    'instance_l3_cache_usage': None,
+                    'instance_root_disk_size': None,
+                },
+                help="Mapping of grafana translators to datasource metrics. "
+                     "Values should be strings. Example: influxdb"),
+    cfg.DictOpt('query_map',
+                # {0} = aggregate
+                # {1} = attribute
+                # {2} = period
+                # {3} = granularity
+                # {4} = { influxdb: retention_period, }
+                default={
+                    'host_cpu_usage': None,
+                    'host_ram_usage': None,
+                    'host_outlet_temp': None,
+                    'host_inlet_temp': None,
+                    'host_airflow': None,
+                    'host_power': None,
+                    'instance_cpu_usage': None,
+                    'instance_ram_usage': None,
+                    'instance_ram_allocated': None,
+                    'instance_l3_cache_usage': None,
+                    'instance_root_disk_size': None,
+                },
+                help="Mapping of grafana queries to datasource metrics. "
+                     "Values should be strings for which the .format method "
+                     "will transform it. These queries will need to be "
+                     "constructed using tools such as Postman. "
+                     "Example: SELECT cpu FROM {4}.cpu_percent WHERE "
+                     "host == '{1}' AND time > now()-{2}s")]
+
+
+def register_opts(conf):
+    conf.register_group(grafana_client)
+    conf.register_opts(GRAFANA_CLIENT_OPTS, group=grafana_client)
+
+
+def list_opts():
+    return [('grafana_client', GRAFANA_CLIENT_OPTS)]

watcher/conf/grafana_translators.py

@@ -0,0 +1,44 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 European Organization for Nuclear Research (CERN)
+#
+# Authors: Corne Lukken <info@dantalion.nl>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+grafana_translators = cfg.OptGroup(name='grafana_translators',
+                                   title='Configuration Options for Grafana '
+                                   'transalators')
+
+GRAFANA_TRANSLATOR_INFLUX_OPTS = [
+    cfg.DictOpt('retention_periods',
+                default={
+                    'one_week': 604800,
+                    'one_month': 2592000,
+                    'five_years': 31556952
+                },
+                help="Keys are the names of retention periods in InfluxDB and "
+                     "the values should correspond with the maximum time they "
+                     "can retain in seconds. Example: {'one_day': 86400}")]
+
+
+def register_opts(conf):
+    conf.register_group(grafana_translators)
+    conf.register_opts(GRAFANA_TRANSLATOR_INFLUX_OPTS,
+                       group=grafana_translators)
+
+
+def list_opts():
+    return [('grafana_translators', GRAFANA_TRANSLATOR_INFLUX_OPTS)]

watcher/tests/conf/test_list_opts.py

@@ -30,10 +30,11 @@ class TestListOpts(base.TestCase):
         self.base_sections = [
             'DEFAULT', 'api', 'database', 'watcher_decision_engine',
             'watcher_applier', 'watcher_datasources', 'watcher_planner',
-            'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client',
+            'nova_client', 'glance_client', 'gnocchi_client', 'grafana_client',
-            'ceilometer_client', 'monasca_client', 'ironic_client',
+            'grafana_translators', 'cinder_client', 'ceilometer_client',
-            'keystone_client', 'neutron_client', 'watcher_clients_auth',
+            'monasca_client', 'ironic_client', 'keystone_client',
-            'collector', 'placement_client']
+            'neutron_client', 'watcher_clients_auth', 'collector',
+            'placement_client']
         self.opt_sections = list(dict(opts.list_opts()).keys())
 
     def test_run_list_opts(self):