From b02cf3bea567b715290871575c523d5ba88b2727 Mon Sep 17 00:00:00 2001 From: Santhosh Fernandes Date: Wed, 29 Mar 2017 16:00:44 +0530 Subject: [PATCH] Add gnocchi support in uniform_airflow strategy This patch adds gnocchi support in uniform_airflow strategy and adds unit tests corresponding to that change. Change-Id: I347d0129da94a2fc88229d09297765795c5eeb1a Partiallly-Implements: bp gnocchi-watcher --- .../strategy/strategies/uniform_airflow.py | 130 ++++++++++++++---- .../strategies/test_uniform_airflow.py | 62 ++++++++- 2 files changed, 161 insertions(+), 31 deletions(-) diff --git a/watcher/decision_engine/strategy/strategies/uniform_airflow.py b/watcher/decision_engine/strategy/strategies/uniform_airflow.py index a779ed259..e58b73329 100644 --- a/watcher/decision_engine/strategy/strategies/uniform_airflow.py +++ b/watcher/decision_engine/strategy/strategies/uniform_airflow.py @@ -42,12 +42,15 @@ airflow is higher than the specified threshold. - It assumes that live migrations are possible. """ +import datetime +from oslo_config import cfg from oslo_log import log from watcher._i18n import _ from watcher.common import exception as wexc from watcher.datasource import ceilometer as ceil +from watcher.datasource import gnocchi as gnoc from watcher.decision_engine.model import element from watcher.decision_engine.strategy.strategies import base @@ -80,15 +83,28 @@ class UniformAirflow(base.BaseStrategy): - It assumes that live migrations are possible. """ - # The meter to report Airflow of physical server in ceilometer - METER_NAME_AIRFLOW = "hardware.ipmi.node.airflow" - # The meter to report inlet temperature of physical server in ceilometer - METER_NAME_INLET_T = "hardware.ipmi.node.temperature" - # The meter to report system power of physical server in ceilometer - METER_NAME_POWER = "hardware.ipmi.node.power" # choose 300 seconds as the default duration of meter aggregation PERIOD = 300 + METRIC_NAMES = dict( + ceilometer=dict( + # The meter to report Airflow of physical server in ceilometer + host_airflow='hardware.ipmi.node.airflow', + # The meter to report inlet temperature of physical server + # in ceilometer + host_inlet_temp='hardware.ipmi.node.temperature', + # The meter to report system power of physical server in ceilometer + host_power='hardware.ipmi.node.power'), + gnocchi=dict( + # The meter to report Airflow of physical server in gnocchi + host_airflow='hardware.ipmi.node.airflow', + # The meter to report inlet temperature of physical server + # in gnocchi + host_inlet_temp='hardware.ipmi.node.temperature', + # The meter to report system power of physical server in gnocchi + host_power='hardware.ipmi.node.power'), + ) + MIGRATION = "migrate" def __init__(self, config, osc=None): @@ -101,10 +117,14 @@ class UniformAirflow(base.BaseStrategy): super(UniformAirflow, self).__init__(config, osc) # The migration plan will be triggered when the airflow reaches # threshold - self.meter_name_airflow = self.METER_NAME_AIRFLOW - self.meter_name_inlet_t = self.METER_NAME_INLET_T - self.meter_name_power = self.METER_NAME_POWER + self.meter_name_airflow = self.METRIC_NAMES[ + self.config.datasource]['host_airflow'] + self.meter_name_inlet_t = self.METRIC_NAMES[ + self.config.datasource]['host_inlet_temp'] + self.meter_name_power = self.METRIC_NAMES[ + self.config.datasource]['host_power'] self._ceilometer = None + self._gnocchi = None self._period = self.PERIOD @property @@ -117,6 +137,16 @@ class UniformAirflow(base.BaseStrategy): def ceilometer(self, c): self._ceilometer = c + @property + def gnocchi(self): + if self._gnocchi is None: + self._gnocchi = gnoc.GnocchiHelper(osc=self.osc) + return self._gnocchi + + @gnocchi.setter + def gnocchi(self, g): + self._gnocchi = g + @classmethod def get_name(cls): return "uniform_airflow" @@ -133,6 +163,10 @@ class UniformAirflow(base.BaseStrategy): def get_goal_name(cls): return "airflow_optimization" + @property + def granularity(self): + return self.input_parameters.get('granularity', 300) + @classmethod def get_schema(cls): # Mandatory default setting for each element @@ -161,9 +195,25 @@ class UniformAirflow(base.BaseStrategy): "type": "number", "default": 300 }, + "granularity": { + "description": "The time between two measures in an " + "aggregated timeseries of a metric.", + "type": "number", + "default": 300 + }, }, } + @classmethod + def get_config_opts(cls): + return [ + cfg.StrOpt( + "datasource", + help="Data source to use in order to query the needed metrics", + default="ceilometer", + choices=["ceilometer", "gnocchi"]) + ] + def calculate_used_resource(self, node): """Compute the used vcpus, memory and disk based on instance flavors""" instances = self.compute_model.get_node_instances(node) @@ -188,16 +238,35 @@ class UniformAirflow(base.BaseStrategy): source_instances = self.compute_model.get_node_instances( source_node) if source_instances: - inlet_t = self.ceilometer.statistic_aggregation( - resource_id=source_node.uuid, - meter_name=self.meter_name_inlet_t, - period=self._period, - aggregate='avg') - power = self.ceilometer.statistic_aggregation( - resource_id=source_node.uuid, - meter_name=self.meter_name_power, - period=self._period, - aggregate='avg') + if self.config.datasource == "ceilometer": + inlet_t = self.ceilometer.statistic_aggregation( + resource_id=source_node.uuid, + meter_name=self.meter_name_inlet_t, + period=self._period, + aggregate='avg') + power = self.ceilometer.statistic_aggregation( + resource_id=source_node.uuid, + meter_name=self.meter_name_power, + period=self._period, + aggregate='avg') + elif self.config.datasource == "gnocchi": + stop_time = datetime.datetime.utcnow() + start_time = stop_time - datetime.timedelta( + seconds=int(self._period)) + inlet_t = self.gnocchi.statistic_aggregation( + resource_id=source_node.uuid, + metric=self.meter_name_inlet_t, + granularity=self.granularity, + start_time=start_time, + stop_time=stop_time, + aggregation='mean') + power = self.gnocchi.statistic_aggregation( + resource_id=source_node.uuid, + metric=self.meter_name_power, + granularity=self.granularity, + start_time=start_time, + stop_time=stop_time, + aggregation='mean') if (power < self.threshold_power and inlet_t < self.threshold_inlet_t): # hardware issue, migrate all instances from this node @@ -271,14 +340,27 @@ class UniformAirflow(base.BaseStrategy): overload_hosts = [] nonoverload_hosts = [] for node_id in nodes: + airflow = None node = self.compute_model.get_node_by_uuid( node_id) resource_id = node.uuid - airflow = self.ceilometer.statistic_aggregation( - resource_id=resource_id, - meter_name=self.meter_name_airflow, - period=self._period, - aggregate='avg') + if self.config.datasource == "ceilometer": + airflow = self.ceilometer.statistic_aggregation( + resource_id=resource_id, + meter_name=self.meter_name_airflow, + period=self._period, + aggregate='avg') + elif self.config.datasource == "gnocchi": + stop_time = datetime.datetime.utcnow() + start_time = stop_time - datetime.timedelta( + seconds=int(self._period)) + airflow = self.gnocchi.statistic_aggregation( + resource_id=resource_id, + metric=self.meter_name_airflow, + granularity=self.granularity, + start_time=start_time, + stop_time=stop_time, + aggregation='mean') # some hosts may not have airflow meter, remove from target if airflow is None: LOG.warning("%s: no airflow data", resource_id) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py b/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py index c5a575e35..63076a12e 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py @@ -17,6 +17,7 @@ # limitations under the License. # import collections +import datetime import mock from watcher.applier.loading import default @@ -27,14 +28,24 @@ from watcher.decision_engine.strategy import strategies from watcher.tests import base from watcher.tests.decision_engine.model import ceilometer_metrics from watcher.tests.decision_engine.model import faker_cluster_state +from watcher.tests.decision_engine.model import gnocchi_metrics class TestUniformAirflow(base.TestCase): + scenarios = [ + ("Ceilometer", + {"datasource": "ceilometer", + "fake_datasource_cls": ceilometer_metrics.FakeCeilometerMetrics}), + ("Gnocchi", + {"datasource": "gnocchi", + "fake_datasource_cls": gnocchi_metrics.FakeGnocchiMetrics}), + ] + def setUp(self): super(TestUniformAirflow, self).setUp() # fake metrics - self.fake_metrics = ceilometer_metrics.FakeCeilometerMetrics() + self.fake_metrics = self.fake_datasource_cls() # fake cluster self.fake_cluster = faker_cluster_state.FakerModelCollector() @@ -44,11 +55,11 @@ class TestUniformAirflow(base.TestCase): self.m_model = p_model.start() self.addCleanup(p_model.stop) - p_ceilometer = mock.patch.object( - strategies.UniformAirflow, "ceilometer", + p_datasource = mock.patch.object( + strategies.UniformAirflow, self.datasource, new_callable=mock.PropertyMock) - self.m_ceilometer = p_ceilometer.start() - self.addCleanup(p_ceilometer.stop) + self.m_datasource = p_datasource.start() + self.addCleanup(p_datasource.stop) p_audit_scope = mock.patch.object( strategies.UniformAirflow, "audit_scope", @@ -60,9 +71,10 @@ class TestUniformAirflow(base.TestCase): self.m_audit_scope.return_value = mock.Mock() self.m_model.return_value = model_root.ModelRoot() - self.m_ceilometer.return_value = mock.Mock( + self.m_datasource.return_value = mock.Mock( statistic_aggregation=self.fake_metrics.mock_get_statistics) - self.strategy = strategies.UniformAirflow(config=mock.Mock()) + self.strategy = strategies.UniformAirflow( + config=mock.Mock(datasource=self.datasource)) self.strategy.input_parameters = utils.Struct() self.strategy.input_parameters.update({'threshold_airflow': 400.0, 'threshold_inlet_t': 28.0, @@ -199,3 +211,39 @@ class TestUniformAirflow(base.TestCase): loaded_action = loader.load(action['action_type']) loaded_action.input_parameters = action['input_parameters'] loaded_action.validate_parameters() + + def test_periods(self): + model = self.fake_cluster.generate_scenario_7_with_2_nodes() + self.m_model.return_value = model + p_ceilometer = mock.patch.object( + strategies.UniformAirflow, "ceilometer") + m_ceilometer = p_ceilometer.start() + self.addCleanup(p_ceilometer.stop) + p_gnocchi = mock.patch.object(strategies.UniformAirflow, "gnocchi") + m_gnocchi = p_gnocchi.start() + self.addCleanup(p_gnocchi.stop) + datetime_patcher = mock.patch.object( + datetime, 'datetime', + mock.Mock(wraps=datetime.datetime) + ) + mocked_datetime = datetime_patcher.start() + mocked_datetime.utcnow.return_value = datetime.datetime( + 2017, 3, 19, 18, 53, 11, 657417) + self.addCleanup(datetime_patcher.stop) + m_ceilometer.statistic_aggregation = mock.Mock( + side_effect=self.fake_metrics.mock_get_statistics) + m_gnocchi.statistic_aggregation = mock.Mock( + side_effect=self.fake_metrics.mock_get_statistics) + self.strategy.group_hosts_by_airflow() + if self.strategy.config.datasource == "ceilometer": + m_ceilometer.statistic_aggregation.assert_any_call( + aggregate='avg', meter_name='hardware.ipmi.node.airflow', + period=300, resource_id=mock.ANY) + elif self.strategy.config.datasource == "gnocchi": + stop_time = datetime.datetime.utcnow() + start_time = stop_time - datetime.timedelta( + seconds=int('300')) + m_gnocchi.statistic_aggregation.assert_called_with( + resource_id=mock.ANY, metric='hardware.ipmi.node.airflow', + granularity=300, start_time=start_time, stop_time=stop_time, + aggregation='mean')