From ce196b68c4bd62381c12468d9a22a6828d7832a2 Mon Sep 17 00:00:00 2001 From: Alexander Chadin Date: Wed, 13 Dec 2017 11:53:37 +0300 Subject: [PATCH] Adapt workload_stabilization strategy to new datasource backend This patch set: 1. Removes nova, ceilometer and gnocchi properties. 2. Adds using of datasource_backend properties along with statistic_aggregation method. 3. Changes type of datasource config. Change-Id: I4a2f05772248fddd97a41e27be4094eb59ee0bdb Partially-Implements: blueprint watcher-multi-datasource --- watcher/datasource/gnocchi.py | 4 +- watcher/datasource/manager.py | 2 +- .../strategies/workload_stabilization.py | 94 +++---------------- watcher/tests/datasource/test_manager.py | 4 +- .../model/ceilometer_metrics.py | 6 ++ .../decision_engine/model/gnocchi_metrics.py | 7 ++ .../strategies/test_workload_stabilization.py | 56 +---------- 7 files changed, 34 insertions(+), 139 deletions(-) diff --git a/watcher/datasource/gnocchi.py b/watcher/datasource/gnocchi.py index 0496b89f4..c771f7807 100644 --- a/watcher/datasource/gnocchi.py +++ b/watcher/datasource/gnocchi.py @@ -108,8 +108,8 @@ class GnocchiHelper(base.DataSourceBase): # measure has structure [time, granularity, value] return statistics[-1][2] - def statistic_aggregation(self, resource_id, metric, period, granularity, - aggregation='mean'): + def statistic_aggregation(self, resource_id, metric, period, aggregation, + granularity=300): stop_time = datetime.utcnow() start_time = stop_time - timedelta(seconds=(int(period))) return self._statistic_aggregation( diff --git a/watcher/datasource/manager.py b/watcher/datasource/manager.py index e5c301ec9..6d8032b0a 100644 --- a/watcher/datasource/manager.py +++ b/watcher/datasource/manager.py @@ -33,7 +33,7 @@ class DataSourceManager(object): self._monasca = None self._gnocchi = None self.metric_map = base.DataSourceBase.METRIC_MAP - self.datasources = self.config.datasource + self.datasources = self.config.datasources @property def ceilometer(self): diff --git a/watcher/decision_engine/strategy/strategies/workload_stabilization.py b/watcher/decision_engine/strategy/strategies/workload_stabilization.py index 20356a55e..15259b351 100644 --- a/watcher/decision_engine/strategy/strategies/workload_stabilization.py +++ b/watcher/decision_engine/strategy/strategies/workload_stabilization.py @@ -28,7 +28,6 @@ It assumes that live migrations are possible in your cluster. """ import copy -import datetime import itertools import math import random @@ -41,8 +40,6 @@ import oslo_utils from watcher._i18n import _ from watcher.common import exception -from watcher.datasource import ceilometer as ceil -from watcher.datasource import gnocchi as gnoc from watcher.decision_engine.model import element from watcher.decision_engine.strategy.strategies import base @@ -73,9 +70,6 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy): :param osc: :py:class:`~.OpenStackClients` instance """ super(WorkloadStabilization, self).__init__(config, osc) - self._ceilometer = None - self._gnocchi = None - self._nova = None self.weights = None self.metrics = None self.thresholds = None @@ -169,43 +163,16 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy): @classmethod def get_config_opts(cls): return [ - cfg.StrOpt( - "datasource", - help="Data source to use in order to query the needed metrics", - default="gnocchi", - choices=["ceilometer", "gnocchi"]) + cfg.ListOpt( + "datasources", + help="Datasources to use in order to query the needed metrics." + " If one of strategy metric isn't available in the first" + " datasource, the next datasource will be chosen.", + item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', + 'monasca']), + default=['gnocchi', 'ceilometer', 'monasca']) ] - @property - def ceilometer(self): - if self._ceilometer is None: - self.ceilometer = ceil.CeilometerHelper(osc=self.osc) - return self._ceilometer - - @property - def nova(self): - if self._nova is None: - self.nova = self.osc.nova() - return self._nova - - @nova.setter - def nova(self, n): - self._nova = n - - @ceilometer.setter - def ceilometer(self, c): - self._ceilometer = c - - @property - def gnocchi(self): - if self._gnocchi is None: - self.gnocchi = gnoc.GnocchiHelper(osc=self.osc) - return self._gnocchi - - @gnocchi.setter - def gnocchi(self, gnocchi): - self._gnocchi = gnocchi - def transform_instance_cpu(self, instance_load, host_vcpus): """Transform instance cpu utilization to overall host cpu utilization. @@ -227,26 +194,9 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy): LOG.debug('get_instance_load started') instance_load = {'uuid': instance.uuid, 'vcpus': instance.vcpus} for meter in self.metrics: - avg_meter = None - if self.config.datasource == "ceilometer": - avg_meter = self.ceilometer.statistic_aggregation( - resource_id=instance.uuid, - meter_name=meter, - period=self.periods['instance'], - aggregate='min' - ) - elif self.config.datasource == "gnocchi": - stop_time = datetime.datetime.utcnow() - start_time = stop_time - datetime.timedelta( - seconds=int(self.periods['instance'])) - avg_meter = self.gnocchi.statistic_aggregation( - resource_id=instance.uuid, - metric=meter, - granularity=self.granularity, - start_time=start_time, - stop_time=stop_time, - aggregation='mean' - ) + avg_meter = self.datasource_backend.statistic_aggregation( + instance.uuid, meter, self.periods['instance'], 'mean', + granularity=self.granularity) if avg_meter is None: LOG.warning( "No values returned by %(resource_id)s " @@ -287,25 +237,9 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy): resource_id = "%s_%s" % (node.uuid, node.hostname) else: resource_id = node_id - if self.config.datasource == "ceilometer": - avg_meter = self.ceilometer.statistic_aggregation( - resource_id=resource_id, - meter_name=self.instance_metrics[metric], - period=self.periods['node'], - aggregate='avg' - ) - elif self.config.datasource == "gnocchi": - stop_time = datetime.datetime.utcnow() - start_time = stop_time - datetime.timedelta( - seconds=int(self.periods['node'])) - avg_meter = self.gnocchi.statistic_aggregation( - resource_id=resource_id, - metric=self.instance_metrics[metric], - granularity=self.granularity, - start_time=start_time, - stop_time=stop_time, - aggregation='mean' - ) + avg_meter = self.datasource_backend.statistic_aggregation( + resource_id, self.instance_metrics[metric], + self.periods['node'], 'mean', granularity=self.granularity) if avg_meter is None: if meter_name == 'hardware.memory.used': diff --git a/watcher/tests/datasource/test_manager.py b/watcher/tests/datasource/test_manager.py index 39b65c5d1..ba6018835 100644 --- a/watcher/tests/datasource/test_manager.py +++ b/watcher/tests/datasource/test_manager.py @@ -28,7 +28,7 @@ class TestDataSourceManager(base.BaseTestCase): def test_get_backend(self, mock_gnoc): manager = ds_manager.DataSourceManager( config=mock.MagicMock( - datasource=['gnocchi', 'ceilometer', 'monasca']), + datasources=['gnocchi', 'ceilometer', 'monasca']), osc=mock.MagicMock()) backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage']) @@ -37,7 +37,7 @@ class TestDataSourceManager(base.BaseTestCase): def test_get_backend_wrong_metric(self): manager = ds_manager.DataSourceManager( config=mock.MagicMock( - datasource=['gnocchi', 'ceilometer', 'monasca']), + datasources=['gnocchi', 'ceilometer', 'monasca']), osc=mock.MagicMock()) self.assertRaises(exception.NoSuchMetric, manager.get_backend, ['host_cpu', 'instance_cpu_usage']) diff --git a/watcher/tests/decision_engine/model/ceilometer_metrics.py b/watcher/tests/decision_engine/model/ceilometer_metrics.py index 1103fb41f..b9d90d0ea 100644 --- a/watcher/tests/decision_engine/model/ceilometer_metrics.py +++ b/watcher/tests/decision_engine/model/ceilometer_metrics.py @@ -26,6 +26,12 @@ class FakeCeilometerMetrics(object): def empty_one_metric(self, emptytype): self.emptytype = emptytype + # TODO(alexchadin): This method is added as temporary solution until + # all strategies use datasource_backend property. + def temp_mock_get_statistics(self, resource_id, meter_name, period, + aggregate, granularity=300): + return self.mock_get_statistics(resource_id, meter_name, period) + def mock_get_statistics(self, resource_id, meter_name, period, aggregate='avg'): result = 0 diff --git a/watcher/tests/decision_engine/model/gnocchi_metrics.py b/watcher/tests/decision_engine/model/gnocchi_metrics.py index 8c0a75e4b..8d87d6101 100644 --- a/watcher/tests/decision_engine/model/gnocchi_metrics.py +++ b/watcher/tests/decision_engine/model/gnocchi_metrics.py @@ -21,6 +21,13 @@ class FakeGnocchiMetrics(object): def empty_one_metric(self, emptytype): self.emptytype = emptytype + # TODO(alexchadin): This method is added as temporary solution until + # all strategies use datasource_backend property. + def temp_mock_get_statistics(self, resource_id, metric, period, aggregate, + granularity=300): + return self.mock_get_statistics(resource_id, metric, granularity, + 0, 0, aggregation='mean') + def mock_get_statistics(self, resource_id, metric, granularity, start_time, stop_time, aggregation='mean'): result = 0 diff --git a/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py b/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py index 8c9e656ed..b5c751b35 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py @@ -17,7 +17,6 @@ # limitations under the License. # -import datetime import mock from watcher.common import clients @@ -69,7 +68,7 @@ class TestWorkloadStabilization(base.TestCase): self.addCleanup(p_model.stop) p_datasource = mock.patch.object( - strategies.WorkloadStabilization, self.datasource, + strategies.WorkloadStabilization, "datasource_backend", new_callable=mock.PropertyMock) self.m_datasource = p_datasource.start() self.addCleanup(p_datasource.stop) @@ -84,7 +83,7 @@ class TestWorkloadStabilization(base.TestCase): self.m_model.return_value = model_root.ModelRoot() self.m_audit_scope.return_value = mock.Mock() self.m_datasource.return_value = mock.Mock( - statistic_aggregation=self.fake_metrics.mock_get_statistics) + statistic_aggregation=self.fake_metrics.temp_mock_get_statistics) self.strategy = strategies.WorkloadStabilization( config=mock.Mock(datasource=self.datasource)) @@ -121,57 +120,6 @@ class TestWorkloadStabilization(base.TestCase): self.assertEqual( instance_0_dict, self.strategy.get_instance_load(instance0)) - def test_periods(self): - model = self.fake_cluster.generate_scenario_1() - self.m_model.return_value = model - p_ceilometer = mock.patch.object( - strategies.WorkloadStabilization, "ceilometer") - m_ceilometer = p_ceilometer.start() - self.addCleanup(p_ceilometer.stop) - p_gnocchi = mock.patch.object(strategies.WorkloadStabilization, - "gnocchi") - m_gnocchi = p_gnocchi.start() - self.addCleanup(p_gnocchi.stop) - datetime_patcher = mock.patch.object( - datetime, 'datetime', - mock.Mock(wraps=datetime.datetime) - ) - mocked_datetime = datetime_patcher.start() - mocked_datetime.utcnow.return_value = datetime.datetime( - 2017, 3, 19, 18, 53, 11, 657417) - self.addCleanup(datetime_patcher.stop) - m_ceilometer.return_value = mock.Mock( - statistic_aggregation=self.fake_metrics.mock_get_statistics) - m_gnocchi.return_value = mock.Mock( - statistic_aggregation=self.fake_metrics.mock_get_statistics) - instance0 = model.get_instance_by_uuid("INSTANCE_0") - self.strategy.get_instance_load(instance0) - if self.strategy.config.datasource == "ceilometer": - m_ceilometer.statistic_aggregation.assert_called_with( - aggregate='min', meter_name='memory.resident', - period=720, resource_id=instance0.uuid) - elif self.strategy.config.datasource == "gnocchi": - stop_time = datetime.datetime.utcnow() - start_time = stop_time - datetime.timedelta( - seconds=int('720')) - m_gnocchi.statistic_aggregation.assert_called_with( - resource_id=instance0.uuid, metric='memory.resident', - granularity=300, start_time=start_time, stop_time=stop_time, - aggregation='mean') - self.strategy.get_hosts_load() - if self.strategy.config.datasource == "ceilometer": - m_ceilometer.statistic_aggregation.assert_called_with( - aggregate='avg', meter_name='hardware.memory.used', - period=600, resource_id=mock.ANY) - elif self.strategy.config.datasource == "gnocchi": - stop_time = datetime.datetime.utcnow() - start_time = stop_time - datetime.timedelta( - seconds=int('600')) - m_gnocchi.statistic_aggregation.assert_called_with( - resource_id=mock.ANY, metric='hardware.memory.used', - granularity=300, start_time=start_time, stop_time=stop_time, - aggregation='mean') - def test_normalize_hosts_load(self): self.m_model.return_value = self.fake_cluster.generate_scenario_1() fake_hosts = {'Node_0': {'cpu_util': 0.07, 'memory.resident': 7},