From 0b213a873438b200a46aa6305da1a5a727100969 Mon Sep 17 00:00:00 2001 From: Alexander Chadin Date: Mon, 27 Feb 2017 17:48:56 +0300 Subject: [PATCH] Add Gnocchi datasource This patch set adds Gnocchi datasource support Implements: blueprint gnocchi-watcher Change-Id: I41653149435fd355548071de855586004371c4fc --- requirements.txt | 1 + watcher/common/clients.py | 13 +++ watcher/common/exception.py | 6 +- watcher/conf/__init__.py | 2 + watcher/conf/gnocchi_client.py | 42 +++++++++ watcher/datasource/gnocchi.py | 92 +++++++++++++++++++ watcher/tests/common/test_clients.py | 28 ++++++ watcher/tests/conf/test_list_opts.py | 5 +- .../tests/datasource/test_gnocchi_helper.py | 68 ++++++++++++++ 9 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 watcher/conf/gnocchi_client.py create mode 100644 watcher/datasource/gnocchi.py create mode 100644 watcher/tests/datasource/test_gnocchi_helper.py diff --git a/requirements.txt b/requirements.txt index 08bb48eb7..f0ea29668 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ pbr>=1.8 # Apache-2.0 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD PrettyTable<0.8,>=0.7.1 # BSD voluptuous>=0.8.9 # BSD License +gnocchiclient>=2.7.0 # Apache-2.0 python-ceilometerclient>=2.5.0 # Apache-2.0 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0 python-glanceclient>=2.5.0 # Apache-2.0 diff --git a/watcher/common/clients.py b/watcher/common/clients.py index 21f812402..a734b68c7 100644 --- a/watcher/common/clients.py +++ b/watcher/common/clients.py @@ -13,6 +13,7 @@ from ceilometerclient import client as ceclient from cinderclient import client as ciclient from glanceclient import client as glclient +from gnocchiclient import client as gnclient from keystoneauth1 import loading as ka_loading from keystoneclient import client as keyclient from monascaclient import client as monclient @@ -39,6 +40,7 @@ class OpenStackClients(object): self._keystone = None self._nova = None self._glance = None + self._gnocchi = None self._cinder = None self._ceilometer = None self._monasca = None @@ -92,6 +94,17 @@ class OpenStackClients(object): session=self.session) return self._glance + @exception.wrap_keystone_exception + def gnocchi(self): + if self._gnocchi: + return self._gnocchi + + gnocchiclient_version = self._get_client_option('gnocchi', + 'api_version') + self._gnocchi = gnclient.Client(gnocchiclient_version, + session=self.session) + return self._gnocchi + @exception.wrap_keystone_exception def cinder(self): if self._cinder: diff --git a/watcher/common/exception.py b/watcher/common/exception.py index 06e434144..29f2d240f 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -130,7 +130,7 @@ class OperationNotPermitted(NotAuthorized): msg_fmt = _("Operation not permitted") -class Invalid(WatcherException): +class Invalid(WatcherException, ValueError): msg_fmt = _("Unacceptable parameters") code = 400 @@ -149,6 +149,10 @@ class ResourceNotFound(ObjectNotFound): code = 404 +class InvalidParameter(Invalid): + msg_fmt = _("%(parameter)s has to be of type %(parameter_type)s") + + class InvalidIdentity(Invalid): msg_fmt = _("Expected a uuid or int but received %(identity)s") diff --git a/watcher/conf/__init__.py b/watcher/conf/__init__.py index ccf810aa3..a561d7609 100644 --- a/watcher/conf/__init__.py +++ b/watcher/conf/__init__.py @@ -28,6 +28,7 @@ from watcher.conf import db from watcher.conf import decision_engine from watcher.conf import exception from watcher.conf import glance_client +from watcher.conf import gnocchi_client from watcher.conf import monasca_client from watcher.conf import neutron_client from watcher.conf import nova_client @@ -50,6 +51,7 @@ decision_engine.register_opts(CONF) monasca_client.register_opts(CONF) nova_client.register_opts(CONF) glance_client.register_opts(CONF) +gnocchi_client.register_opts(CONF) cinder_client.register_opts(CONF) ceilometer_client.register_opts(CONF) neutron_client.register_opts(CONF) diff --git a/watcher/conf/gnocchi_client.py b/watcher/conf/gnocchi_client.py new file mode 100644 index 000000000..1ab0b5585 --- /dev/null +++ b/watcher/conf/gnocchi_client.py @@ -0,0 +1,42 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 Servionica +# +# Authors: Alexander Chadin +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +gnocchi_client = cfg.OptGroup(name='gnocchi_client', + title='Configuration Options for Gnocchi') + +GNOCCHI_CLIENT_OPTS = [ + cfg.StrOpt('api_version', + default='1', + help='Version of Gnocchi API to use in gnocchiclient.'), + cfg.IntOpt('query_max_retries', + default=10, + help='How many times Watcher is trying to query again'), + cfg.IntOpt('query_timeout', + default=1, + help='How many seconds Watcher should wait to do query again')] + + +def register_opts(conf): + conf.register_group(gnocchi_client) + conf.register_opts(GNOCCHI_CLIENT_OPTS, group=gnocchi_client) + + +def list_opts(): + return [('gnocchi_client', GNOCCHI_CLIENT_OPTS)] diff --git a/watcher/datasource/gnocchi.py b/watcher/datasource/gnocchi.py new file mode 100644 index 000000000..00851125d --- /dev/null +++ b/watcher/datasource/gnocchi.py @@ -0,0 +1,92 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 Servionica +# +# Authors: Alexander Chadin +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import time + +from oslo_config import cfg +from oslo_log import log + +from watcher.common import clients +from watcher.common import exception + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +class GnocchiHelper(object): + + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" + self.osc = osc if osc else clients.OpenStackClients() + self.gnocchi = self.osc.gnocchi() + + def query_retry(self, f, *args, **kwargs): + for i in range(CONF.gnocchi_client.query_max_retries): + try: + return f(*args, **kwargs) + except Exception as e: + LOG.exception(e) + time.sleep(CONF.gnocchi_client.query_timeout) + raise + + def statistic_aggregation(self, + resource_id, + metric, + granularity, + start_time=None, + stop_time=None, + aggregation='mean'): + """Representing a statistic aggregate by operators + + :param metric: metric name of which we want the statistics + :param resource_id: id of resource to list statistics for + :param start_time: Start datetime from which metrics will be used + :param stop_time: End datetime from which metrics will be used + :param granularity: frequency of marking metric point, in seconds + :param aggregation: Should be chosen in accrodance with policy + aggregations + :return: value of aggregated metric + """ + + if start_time is not None and not isinstance(start_time, datetime): + raise exception.InvalidParameter(parameter='start_time', + parameter_type=datetime) + + if stop_time is not None and not isinstance(stop_time, datetime): + raise exception.InvalidParameter(parameter='stop_time', + parameter_type=datetime) + + raw_kwargs = dict( + metric=metric, + start=start_time, + stop=stop_time, + resource_id=resource_id, + granularity=granularity, + aggregation=aggregation, + ) + + kwargs = {k: v for k, v in raw_kwargs.items() if k and v} + + statistics = self.query_retry( + f=self.gnocchi.metric.get_measures, **kwargs) + + if statistics: + # return value of latest measure + # measure has structure [time, granularity, value] + return statistics[-1][2] diff --git a/watcher/tests/common/test_clients.py b/watcher/tests/common/test_clients.py index 87b00ad03..fbc8e65f9 100644 --- a/watcher/tests/common/test_clients.py +++ b/watcher/tests/common/test_clients.py @@ -15,6 +15,8 @@ import ceilometerclient.v2.client as ceclient_v2 from cinderclient import client as ciclient from cinderclient.v1 import client as ciclient_v1 from glanceclient import client as glclient +from gnocchiclient import client as gnclient +from gnocchiclient.v1 import client as gnclient_v1 from keystoneauth1 import loading as ka_loading import mock from monascaclient import client as monclient @@ -157,6 +159,32 @@ class TestClients(base.TestCase): glance_cached = osc.glance() self.assertEqual(glance, glance_cached) + @mock.patch.object(gnclient, 'Client') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_gnocchi(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc._gnocchi = None + osc.gnocchi() + mock_call.assert_called_once_with(CONF.gnocchi_client.api_version, + session=mock_session) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_gnocchi_diff_vers(self, mock_session): + # gnocchiclient currently only has one version (v1) + CONF.set_override('api_version', '1', group='gnocchi_client') + osc = clients.OpenStackClients() + osc._gnocchi = None + osc.gnocchi() + self.assertEqual(gnclient_v1.Client, type(osc.gnocchi())) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_gnocchi_cached(self, mock_session): + osc = clients.OpenStackClients() + osc._gnocchi = None + gnocchi = osc.gnocchi() + gnocchi_cached = osc.gnocchi() + self.assertEqual(gnocchi, gnocchi_cached) + @mock.patch.object(ciclient, 'Client') @mock.patch.object(clients.OpenStackClients, 'session') def test_clients_cinder(self, mock_session, mock_call): diff --git a/watcher/tests/conf/test_list_opts.py b/watcher/tests/conf/test_list_opts.py index 4e15cea78..d724f03ab 100644 --- a/watcher/tests/conf/test_list_opts.py +++ b/watcher/tests/conf/test_list_opts.py @@ -30,8 +30,9 @@ class TestListOpts(base.TestCase): self.base_sections = [ 'DEFAULT', 'api', 'database', 'watcher_decision_engine', 'watcher_applier', 'watcher_planner', 'nova_client', - 'glance_client', 'cinder_client', 'ceilometer_client', - 'monasca_client', 'neutron_client', 'watcher_clients_auth'] + 'glance_client', 'gnocchi_client', 'cinder_client', + 'ceilometer_client', 'monasca_client', + 'neutron_client', 'watcher_clients_auth'] self.opt_sections = list(dict(opts.list_opts()).keys()) def test_run_list_opts(self): diff --git a/watcher/tests/datasource/test_gnocchi_helper.py b/watcher/tests/datasource/test_gnocchi_helper.py new file mode 100644 index 000000000..8b481a3ca --- /dev/null +++ b/watcher/tests/datasource/test_gnocchi_helper.py @@ -0,0 +1,68 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 Servionica +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo_config import cfg +from oslo_utils import timeutils + +from watcher.common import clients +from watcher.common import exception +from watcher.datasource import gnocchi as gnocchi_helper +from watcher.tests import base + +CONF = cfg.CONF + + +@mock.patch.object(clients.OpenStackClients, 'gnocchi') +class TestGnocchiHelper(base.BaseTestCase): + + def test_gnocchi_statistic_aggregation(self, mock_gnocchi): + gnocchi = mock.MagicMock() + expected_result = 5.5 + + expected_measures = [["2017-02-02T09:00:00.000000", 360, 5.5]] + + gnocchi.metric.get_measures.return_value = expected_measures + mock_gnocchi.return_value = gnocchi + + helper = gnocchi_helper.GnocchiHelper() + result = helper.statistic_aggregation( + resource_id='16a86790-327a-45f9-bc82-45839f062fdc', + metric='cpu_util', + granularity=360, + start_time=timeutils.parse_isotime("2017-02-02T09:00:00.000000"), + stop_time=timeutils.parse_isotime("2017-02-02T10:00:00.000000"), + aggregation='mean' + ) + self.assertEqual(expected_result, result) + + def test_gnocchi_wrong_datetime(self, mock_gnocchi): + gnocchi = mock.MagicMock() + + expected_measures = [["2017-02-02T09:00:00.000000", 360, 5.5]] + + gnocchi.metric.get_measures.return_value = expected_measures + mock_gnocchi.return_value = gnocchi + + helper = gnocchi_helper.GnocchiHelper() + self.assertRaises( + exception.InvalidParameter, helper.statistic_aggregation, + resource_id='16a86790-327a-45f9-bc82-45839f062fdc', + metric='cpu_util', + granularity=360, + start_time="2017-02-02T09:00:00.000000", + stop_time=timeutils.parse_isotime("2017-02-02T10:00:00.000000"), + aggregation='mean')