Adapt workload_stabilization strategy to new datasource backend

This patch set:
1. Removes nova, ceilometer and gnocchi properties.
2. Adds using of datasource_backend properties along with
   statistic_aggregation method.
3. Changes type of datasource config.

Change-Id: I4a2f05772248fddd97a41e27be4094eb59ee0bdb
Partially-Implements: blueprint watcher-multi-datasource

watcher/datasource/gnocchi.py

@@ -108,8 +108,8 @@ class GnocchiHelper(base.DataSourceBase):
             # measure has structure [time, granularity, value]
             return statistics[-1][2]
 
-    def statistic_aggregation(self, resource_id, metric, period, granularity,
+    def statistic_aggregation(self, resource_id, metric, period, aggregation,
-                              aggregation='mean'):
+                              granularity=300):
         stop_time = datetime.utcnow()
         start_time = stop_time - timedelta(seconds=(int(period)))
         return self._statistic_aggregation(

watcher/datasource/manager.py

@@ -33,7 +33,7 @@ class DataSourceManager(object):
         self._monasca = None
         self._gnocchi = None
         self.metric_map = base.DataSourceBase.METRIC_MAP
-        self.datasources = self.config.datasource
+        self.datasources = self.config.datasources
 
     @property
     def ceilometer(self):

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -28,7 +28,6 @@ It assumes that live migrations are possible in your cluster.
 """
 
 import copy
-import datetime
 import itertools
 import math
 import random
@@ -41,8 +40,6 @@ import oslo_utils
 
 from watcher._i18n import _
 from watcher.common import exception
-from watcher.datasource import ceilometer as ceil
-from watcher.datasource import gnocchi as gnoc
 from watcher.decision_engine.model import element
 from watcher.decision_engine.strategy.strategies import base
 
@@ -73,9 +70,6 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         :param osc: :py:class:`~.OpenStackClients` instance
         """
         super(WorkloadStabilization, self).__init__(config, osc)
-        self._ceilometer = None
-        self._gnocchi = None
-        self._nova = None
         self.weights = None
         self.metrics = None
         self.thresholds = None
@@ -169,43 +163,16 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
     @classmethod
     def get_config_opts(cls):
         return [
-            cfg.StrOpt(
+            cfg.ListOpt(
-                "datasource",
+                "datasources",
-                help="Data source to use in order to query the needed metrics",
+                help="Datasources to use in order to query the needed metrics."
-                default="gnocchi",
+                     " If one of strategy metric isn't available in the first"
-                choices=["ceilometer", "gnocchi"])
+                     " datasource, the next datasource will be chosen.",
+                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
+                                                    'monasca']),
+                default=['gnocchi', 'ceilometer', 'monasca'])
         ]
 
-    @property
-    def ceilometer(self):
-        if self._ceilometer is None:
-            self.ceilometer = ceil.CeilometerHelper(osc=self.osc)
-        return self._ceilometer
-
-    @property
-    def nova(self):
-        if self._nova is None:
-            self.nova = self.osc.nova()
-        return self._nova
-
-    @nova.setter
-    def nova(self, n):
-        self._nova = n
-
-    @ceilometer.setter
-    def ceilometer(self, c):
-        self._ceilometer = c
-
-    @property
-    def gnocchi(self):
-        if self._gnocchi is None:
-            self.gnocchi = gnoc.GnocchiHelper(osc=self.osc)
-        return self._gnocchi
-
-    @gnocchi.setter
-    def gnocchi(self, gnocchi):
-        self._gnocchi = gnocchi
-
     def transform_instance_cpu(self, instance_load, host_vcpus):
         """Transform instance cpu utilization to overall host cpu utilization.
 
@@ -227,26 +194,9 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         LOG.debug('get_instance_load started')
         instance_load = {'uuid': instance.uuid, 'vcpus': instance.vcpus}
         for meter in self.metrics:
-            avg_meter = None
+            avg_meter = self.datasource_backend.statistic_aggregation(
-            if self.config.datasource == "ceilometer":
+                instance.uuid, meter, self.periods['instance'], 'mean',
-                avg_meter = self.ceilometer.statistic_aggregation(
+                granularity=self.granularity)
-                    resource_id=instance.uuid,
-                    meter_name=meter,
-                    period=self.periods['instance'],
-                    aggregate='min'
-                )
-            elif self.config.datasource == "gnocchi":
-                stop_time = datetime.datetime.utcnow()
-                start_time = stop_time - datetime.timedelta(
-                    seconds=int(self.periods['instance']))
-                avg_meter = self.gnocchi.statistic_aggregation(
-                    resource_id=instance.uuid,
-                    metric=meter,
-                    granularity=self.granularity,
-                    start_time=start_time,
-                    stop_time=stop_time,
-                    aggregation='mean'
-                )
             if avg_meter is None:
                 LOG.warning(
                     "No values returned by %(resource_id)s "
@@ -287,25 +237,9 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     resource_id = "%s_%s" % (node.uuid, node.hostname)
                 else:
                     resource_id = node_id
-                if self.config.datasource == "ceilometer":
+                avg_meter = self.datasource_backend.statistic_aggregation(
-                    avg_meter = self.ceilometer.statistic_aggregation(
+                    resource_id, self.instance_metrics[metric],
-                        resource_id=resource_id,
+                    self.periods['node'], 'mean', granularity=self.granularity)
-                        meter_name=self.instance_metrics[metric],
-                        period=self.periods['node'],
-                        aggregate='avg'
-                    )
-                elif self.config.datasource == "gnocchi":
-                    stop_time = datetime.datetime.utcnow()
-                    start_time = stop_time - datetime.timedelta(
-                        seconds=int(self.periods['node']))
-                    avg_meter = self.gnocchi.statistic_aggregation(
-                        resource_id=resource_id,
-                        metric=self.instance_metrics[metric],
-                        granularity=self.granularity,
-                        start_time=start_time,
-                        stop_time=stop_time,
-                        aggregation='mean'
-                    )
 
                 if avg_meter is None:
                     if meter_name == 'hardware.memory.used':

watcher/tests/datasource/test_manager.py

@@ -28,7 +28,7 @@ class TestDataSourceManager(base.BaseTestCase):
     def test_get_backend(self, mock_gnoc):
         manager = ds_manager.DataSourceManager(
             config=mock.MagicMock(
-                datasource=['gnocchi', 'ceilometer', 'monasca']),
+                datasources=['gnocchi', 'ceilometer', 'monasca']),
             osc=mock.MagicMock())
         backend = manager.get_backend(['host_cpu_usage',
                                        'instance_cpu_usage'])
@@ -37,7 +37,7 @@ class TestDataSourceManager(base.BaseTestCase):
     def test_get_backend_wrong_metric(self):
         manager = ds_manager.DataSourceManager(
             config=mock.MagicMock(
-                datasource=['gnocchi', 'ceilometer', 'monasca']),
+                datasources=['gnocchi', 'ceilometer', 'monasca']),
             osc=mock.MagicMock())
         self.assertRaises(exception.NoSuchMetric, manager.get_backend,
                           ['host_cpu', 'instance_cpu_usage'])

watcher/tests/decision_engine/model/ceilometer_metrics.py

@@ -26,6 +26,12 @@ class FakeCeilometerMetrics(object):
     def empty_one_metric(self, emptytype):
         self.emptytype = emptytype
 
+    # TODO(alexchadin): This method is added as temporary solution until
+    # all strategies use datasource_backend property.
+    def temp_mock_get_statistics(self, resource_id, meter_name, period,
+                                 aggregate, granularity=300):
+        return self.mock_get_statistics(resource_id, meter_name, period)
+
     def mock_get_statistics(self, resource_id, meter_name, period,
                             aggregate='avg'):
         result = 0

watcher/tests/decision_engine/model/gnocchi_metrics.py

@@ -21,6 +21,13 @@ class FakeGnocchiMetrics(object):
     def empty_one_metric(self, emptytype):
         self.emptytype = emptytype
 
+    # TODO(alexchadin): This method is added as temporary solution until
+    # all strategies use datasource_backend property.
+    def temp_mock_get_statistics(self, resource_id, metric, period, aggregate,
+                                 granularity=300):
+        return self.mock_get_statistics(resource_id, metric, granularity,
+                                        0, 0, aggregation='mean')
+
     def mock_get_statistics(self, resource_id, metric, granularity,
                             start_time, stop_time, aggregation='mean'):
         result = 0

watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py

@@ -17,7 +17,6 @@
 # limitations under the License.
 #
 
-import datetime
 import mock
 
 from watcher.common import clients
@@ -69,7 +68,7 @@ class TestWorkloadStabilization(base.TestCase):
         self.addCleanup(p_model.stop)
 
         p_datasource = mock.patch.object(
-            strategies.WorkloadStabilization, self.datasource,
+            strategies.WorkloadStabilization, "datasource_backend",
             new_callable=mock.PropertyMock)
         self.m_datasource = p_datasource.start()
         self.addCleanup(p_datasource.stop)
@@ -84,7 +83,7 @@ class TestWorkloadStabilization(base.TestCase):
         self.m_model.return_value = model_root.ModelRoot()
         self.m_audit_scope.return_value = mock.Mock()
         self.m_datasource.return_value = mock.Mock(
-            statistic_aggregation=self.fake_metrics.mock_get_statistics)
+            statistic_aggregation=self.fake_metrics.temp_mock_get_statistics)
 
         self.strategy = strategies.WorkloadStabilization(
             config=mock.Mock(datasource=self.datasource))
@@ -121,57 +120,6 @@ class TestWorkloadStabilization(base.TestCase):
         self.assertEqual(
             instance_0_dict, self.strategy.get_instance_load(instance0))
 
-    def test_periods(self):
-        model = self.fake_cluster.generate_scenario_1()
-        self.m_model.return_value = model
-        p_ceilometer = mock.patch.object(
-            strategies.WorkloadStabilization, "ceilometer")
-        m_ceilometer = p_ceilometer.start()
-        self.addCleanup(p_ceilometer.stop)
-        p_gnocchi = mock.patch.object(strategies.WorkloadStabilization,
-                                      "gnocchi")
-        m_gnocchi = p_gnocchi.start()
-        self.addCleanup(p_gnocchi.stop)
-        datetime_patcher = mock.patch.object(
-            datetime, 'datetime',
-            mock.Mock(wraps=datetime.datetime)
-        )
-        mocked_datetime = datetime_patcher.start()
-        mocked_datetime.utcnow.return_value = datetime.datetime(
-            2017, 3, 19, 18, 53, 11, 657417)
-        self.addCleanup(datetime_patcher.stop)
-        m_ceilometer.return_value = mock.Mock(
-            statistic_aggregation=self.fake_metrics.mock_get_statistics)
-        m_gnocchi.return_value = mock.Mock(
-            statistic_aggregation=self.fake_metrics.mock_get_statistics)
-        instance0 = model.get_instance_by_uuid("INSTANCE_0")
-        self.strategy.get_instance_load(instance0)
-        if self.strategy.config.datasource == "ceilometer":
-            m_ceilometer.statistic_aggregation.assert_called_with(
-                aggregate='min', meter_name='memory.resident',
-                period=720, resource_id=instance0.uuid)
-        elif self.strategy.config.datasource == "gnocchi":
-            stop_time = datetime.datetime.utcnow()
-            start_time = stop_time - datetime.timedelta(
-                seconds=int('720'))
-            m_gnocchi.statistic_aggregation.assert_called_with(
-                resource_id=instance0.uuid, metric='memory.resident',
-                granularity=300, start_time=start_time, stop_time=stop_time,
-                aggregation='mean')
-        self.strategy.get_hosts_load()
-        if self.strategy.config.datasource == "ceilometer":
-            m_ceilometer.statistic_aggregation.assert_called_with(
-                aggregate='avg', meter_name='hardware.memory.used',
-                period=600, resource_id=mock.ANY)
-        elif self.strategy.config.datasource == "gnocchi":
-            stop_time = datetime.datetime.utcnow()
-            start_time = stop_time - datetime.timedelta(
-                seconds=int('600'))
-            m_gnocchi.statistic_aggregation.assert_called_with(
-                resource_id=mock.ANY, metric='hardware.memory.used',
-                granularity=300, start_time=start_time, stop_time=stop_time,
-                aggregation='mean')
-
     def test_normalize_hosts_load(self):
         self.m_model.return_value = self.fake_cluster.generate_scenario_1()
         fake_hosts = {'Node_0': {'cpu_util': 0.07, 'memory.resident': 7},