Merge "Add Gnocchi datasource"

requirements.txt

@@ -27,6 +27,7 @@ pbr>=2.0.0 # Apache-2.0
 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
 PrettyTable<0.8,>=0.7.1 # BSD
 voluptuous>=0.8.9 # BSD License
+gnocchiclient>=2.7.0  # Apache-2.0
 python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient>=2.0.1 # Apache-2.0
 python-glanceclient>=2.5.0 # Apache-2.0

watcher/common/clients.py

@@ -13,6 +13,7 @@
 from ceilometerclient import client as ceclient
 from cinderclient import client as ciclient
 from glanceclient import client as glclient
+from gnocchiclient import client as gnclient
 from keystoneauth1 import loading as ka_loading
 from keystoneclient import client as keyclient
 from monascaclient import client as monclient
@@ -39,6 +40,7 @@ class OpenStackClients(object):
         self._keystone = None
         self._nova = None
         self._glance = None
+        self._gnocchi = None
         self._cinder = None
         self._ceilometer = None
         self._monasca = None
@@ -92,6 +94,17 @@ class OpenStackClients(object):
                                        session=self.session)
         return self._glance
 
+    @exception.wrap_keystone_exception
+    def gnocchi(self):
+        if self._gnocchi:
+            return self._gnocchi
+
+        gnocchiclient_version = self._get_client_option('gnocchi',
+                                                        'api_version')
+        self._gnocchi = gnclient.Client(gnocchiclient_version,
+                                        session=self.session)
+        return self._gnocchi
+
     @exception.wrap_keystone_exception
     def cinder(self):
         if self._cinder:

watcher/common/exception.py

@@ -130,7 +130,7 @@ class OperationNotPermitted(NotAuthorized):
     msg_fmt = _("Operation not permitted")
 
 
-class Invalid(WatcherException):
+class Invalid(WatcherException, ValueError):
     msg_fmt = _("Unacceptable parameters")
     code = 400
 
@@ -149,6 +149,10 @@ class ResourceNotFound(ObjectNotFound):
     code = 404
 
 
+class InvalidParameter(Invalid):
+        msg_fmt = _("%(parameter)s has to be of type %(parameter_type)s")
+
+
 class InvalidIdentity(Invalid):
     msg_fmt = _("Expected a uuid or int but received %(identity)s")
 

watcher/conf/__init__.py

@@ -28,6 +28,7 @@ from watcher.conf import db
 from watcher.conf import decision_engine
 from watcher.conf import exception
 from watcher.conf import glance_client
+from watcher.conf import gnocchi_client
 from watcher.conf import monasca_client
 from watcher.conf import neutron_client
 from watcher.conf import nova_client
@@ -50,6 +51,7 @@ decision_engine.register_opts(CONF)
 monasca_client.register_opts(CONF)
 nova_client.register_opts(CONF)
 glance_client.register_opts(CONF)
+gnocchi_client.register_opts(CONF)
 cinder_client.register_opts(CONF)
 ceilometer_client.register_opts(CONF)
 neutron_client.register_opts(CONF)

watcher/conf/gnocchi_client.py

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 Servionica
+#
+# Authors: Alexander Chadin <a.chadin@servionica.ru>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+gnocchi_client = cfg.OptGroup(name='gnocchi_client',
+                              title='Configuration Options for Gnocchi')
+
+GNOCCHI_CLIENT_OPTS = [
+    cfg.StrOpt('api_version',
+               default='1',
+               help='Version of Gnocchi API to use in gnocchiclient.'),
+    cfg.IntOpt('query_max_retries',
+               default=10,
+               help='How many times Watcher is trying to query again'),
+    cfg.IntOpt('query_timeout',
+               default=1,
+               help='How many seconds Watcher should wait to do query again')]
+
+
+def register_opts(conf):
+    conf.register_group(gnocchi_client)
+    conf.register_opts(GNOCCHI_CLIENT_OPTS, group=gnocchi_client)
+
+
+def list_opts():
+    return [('gnocchi_client', GNOCCHI_CLIENT_OPTS)]

watcher/datasource/gnocchi.py

@@ -0,0 +1,92 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 Servionica
+#
+# Authors: Alexander Chadin <a.chadin@servionica.ru>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+import time
+
+from oslo_config import cfg
+from oslo_log import log
+
+from watcher.common import clients
+from watcher.common import exception
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+class GnocchiHelper(object):
+
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        self.osc = osc if osc else clients.OpenStackClients()
+        self.gnocchi = self.osc.gnocchi()
+
+    def query_retry(self, f, *args, **kwargs):
+        for i in range(CONF.gnocchi_client.query_max_retries):
+            try:
+                return f(*args, **kwargs)
+            except Exception as e:
+                LOG.exception(e)
+                time.sleep(CONF.gnocchi_client.query_timeout)
+        raise
+
+    def statistic_aggregation(self,
+                              resource_id,
+                              metric,
+                              granularity,
+                              start_time=None,
+                              stop_time=None,
+                              aggregation='mean'):
+        """Representing a statistic aggregate by operators
+
+        :param metric: metric name of which we want the statistics
+        :param resource_id: id of resource to list statistics for
+        :param start_time: Start datetime from which metrics will be used
+        :param stop_time: End datetime from which metrics will be used
+        :param granularity: frequency of marking metric point, in seconds
+        :param aggregation: Should be chosen in accrodance with policy
+                            aggregations
+        :return: value of aggregated metric
+        """
+
+        if start_time is not None and not isinstance(start_time, datetime):
+            raise exception.InvalidParameter(parameter='start_time',
+                                             parameter_type=datetime)
+
+        if stop_time is not None and not isinstance(stop_time, datetime):
+            raise exception.InvalidParameter(parameter='stop_time',
+                                             parameter_type=datetime)
+
+        raw_kwargs = dict(
+            metric=metric,
+            start=start_time,
+            stop=stop_time,
+            resource_id=resource_id,
+            granularity=granularity,
+            aggregation=aggregation,
+        )
+
+        kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
+
+        statistics = self.query_retry(
+            f=self.gnocchi.metric.get_measures, **kwargs)
+
+        if statistics:
+            # return value of latest measure
+            # measure has structure [time, granularity, value]
+            return statistics[-1][2]

watcher/tests/common/test_clients.py

@@ -15,6 +15,8 @@ import ceilometerclient.v2.client as ceclient_v2
 from cinderclient import client as ciclient
 from cinderclient.v1 import client as ciclient_v1
 from glanceclient import client as glclient
+from gnocchiclient import client as gnclient
+from gnocchiclient.v1 import client as gnclient_v1
 from keystoneauth1 import loading as ka_loading
 import mock
 from monascaclient import client as monclient
@@ -157,6 +159,32 @@ class TestClients(base.TestCase):
         glance_cached = osc.glance()
         self.assertEqual(glance, glance_cached)
 
+    @mock.patch.object(gnclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_gnocchi(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._gnocchi = None
+        osc.gnocchi()
+        mock_call.assert_called_once_with(CONF.gnocchi_client.api_version,
+                                          session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_gnocchi_diff_vers(self, mock_session):
+        # gnocchiclient currently only has one version (v1)
+        CONF.set_override('api_version', '1', group='gnocchi_client')
+        osc = clients.OpenStackClients()
+        osc._gnocchi = None
+        osc.gnocchi()
+        self.assertEqual(gnclient_v1.Client, type(osc.gnocchi()))
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_gnocchi_cached(self, mock_session):
+        osc = clients.OpenStackClients()
+        osc._gnocchi = None
+        gnocchi = osc.gnocchi()
+        gnocchi_cached = osc.gnocchi()
+        self.assertEqual(gnocchi, gnocchi_cached)
+
     @mock.patch.object(ciclient, 'Client')
     @mock.patch.object(clients.OpenStackClients, 'session')
     def test_clients_cinder(self, mock_session, mock_call):

watcher/tests/conf/test_list_opts.py

@@ -30,8 +30,9 @@ class TestListOpts(base.TestCase):
         self.base_sections = [
             'DEFAULT', 'api', 'database', 'watcher_decision_engine',
             'watcher_applier', 'watcher_planner', 'nova_client',
-            'glance_client', 'cinder_client', 'ceilometer_client',
+            'glance_client', 'gnocchi_client', 'cinder_client',
-            'monasca_client', 'neutron_client', 'watcher_clients_auth']
+            'ceilometer_client', 'monasca_client',
+            'neutron_client', 'watcher_clients_auth']
         self.opt_sections = list(dict(opts.list_opts()).keys())
 
     def test_run_list_opts(self):

watcher/tests/datasource/test_gnocchi_helper.py

@@ -0,0 +1,68 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 Servionica
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+from oslo_config import cfg
+from oslo_utils import timeutils
+
+from watcher.common import clients
+from watcher.common import exception
+from watcher.datasource import gnocchi as gnocchi_helper
+from watcher.tests import base
+
+CONF = cfg.CONF
+
+
+@mock.patch.object(clients.OpenStackClients, 'gnocchi')
+class TestGnocchiHelper(base.BaseTestCase):
+
+    def test_gnocchi_statistic_aggregation(self, mock_gnocchi):
+        gnocchi = mock.MagicMock()
+        expected_result = 5.5
+
+        expected_measures = [["2017-02-02T09:00:00.000000", 360, 5.5]]
+
+        gnocchi.metric.get_measures.return_value = expected_measures
+        mock_gnocchi.return_value = gnocchi
+
+        helper = gnocchi_helper.GnocchiHelper()
+        result = helper.statistic_aggregation(
+            resource_id='16a86790-327a-45f9-bc82-45839f062fdc',
+            metric='cpu_util',
+            granularity=360,
+            start_time=timeutils.parse_isotime("2017-02-02T09:00:00.000000"),
+            stop_time=timeutils.parse_isotime("2017-02-02T10:00:00.000000"),
+            aggregation='mean'
+        )
+        self.assertEqual(expected_result, result)
+
+    def test_gnocchi_wrong_datetime(self, mock_gnocchi):
+        gnocchi = mock.MagicMock()
+
+        expected_measures = [["2017-02-02T09:00:00.000000", 360, 5.5]]
+
+        gnocchi.metric.get_measures.return_value = expected_measures
+        mock_gnocchi.return_value = gnocchi
+
+        helper = gnocchi_helper.GnocchiHelper()
+        self.assertRaises(
+            exception.InvalidParameter, helper.statistic_aggregation,
+            resource_id='16a86790-327a-45f9-bc82-45839f062fdc',
+            metric='cpu_util',
+            granularity=360,
+            start_time="2017-02-02T09:00:00.000000",
+            stop_time=timeutils.parse_isotime("2017-02-02T10:00:00.000000"),
+            aggregation='mean')