Merge "Add gnocchi support in workload_stabilization strategy"

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -28,6 +28,7 @@ It assumes that live migrations are possible in your cluster.
 """
 
 import copy
+import datetime
 import itertools
 import math
 import random
@@ -41,6 +42,7 @@ import oslo_utils
 from watcher._i18n import _
 from watcher.common import exception
 from watcher.datasource import ceilometer as ceil
+from watcher.datasource import gnocchi as gnoc
 from watcher.decision_engine.model import element
 from watcher.decision_engine.strategy.strategies import base
 
@@ -72,6 +74,7 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         """
         super(WorkloadStabilization, self).__init__(config, osc)
         self._ceilometer = None
+        self._gnocchi = None
         self._nova = None
         self.weights = None
         self.metrics = None
@@ -93,6 +96,10 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
     def get_translatable_display_name(cls):
         return "Workload stabilization"
 
+    @property
+    def granularity(self):
+        return self.input_parameters.get('granularity', 300)
+
     @classmethod
     def get_schema(cls):
         return {
@@ -149,10 +156,26 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                                    " ones.",
                     "type": "object",
                     "default": {"instance": 720, "node": 600}
-                }
+                },
+                "granularity": {
+                    "description": "The time between two measures in an "
+                                   "aggregated timeseries of a metric.",
+                    "type": "number",
+                    "default": 300
+                },
             }
         }
 
+    @classmethod
+    def get_config_opts(cls):
+        return [
+            cfg.StrOpt(
+                "datasource",
+                help="Data source to use in order to query the needed metrics",
+                default="ceilometer",
+                choices=["ceilometer", "gnocchi"])
+        ]
+
     @property
     def ceilometer(self):
         if self._ceilometer is None:
@@ -173,6 +196,16 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
     def ceilometer(self, c):
         self._ceilometer = c
 
+    @property
+    def gnocchi(self):
+        if self._gnocchi is None:
+            self._gnocchi = gnoc.GnocchiHelper(osc=self.osc)
+        return self._gnocchi
+
+    @gnocchi.setter
+    def gnocchi(self, gnocchi):
+        self._gnocchi = gnocchi
+
     def transform_instance_cpu(self, instance_load, host_vcpus):
         """Transform instance cpu utilization to overall host cpu utilization.
 
@@ -186,7 +219,7 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
 
     @MEMOIZE
     def get_instance_load(self, instance):
-        """Gathering instance load through ceilometer statistic.
+        """Gathering instance load through ceilometer/gnocchi statistic.
 
         :param instance: instance for which statistic is gathered.
         :return: dict
@@ -194,12 +227,26 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         LOG.debug('get_instance_load started')
         instance_load = {'uuid': instance.uuid, 'vcpus': instance.vcpus}
         for meter in self.metrics:
-            avg_meter = self.ceilometer.statistic_aggregation(
+            avg_meter = None
-                resource_id=instance.uuid,
+            if self.config.datasource == "ceilometer":
-                meter_name=meter,
+                avg_meter = self.ceilometer.statistic_aggregation(
-                period=self.periods['instance'],
+                    resource_id=instance.uuid,
-                aggregate='min'
+                    meter_name=meter,
-            )
+                    period=self.periods['instance'],
+                    aggregate='min'
+                )
+            elif self.config.datasource == "gnocchi":
+                stop_time = datetime.datetime.utcnow()
+                start_time = stop_time - datetime.timedelta(
+                    seconds=int(self.periods['instance']))
+                avg_meter = self.gnocchi.statistic_aggregation(
+                    resource_id=instance.uuid,
+                    metric=meter,
+                    granularity=self.granularity,
+                    start_time=start_time,
+                    stop_time=stop_time,
+                    aggregation='mean'
+                )
             if avg_meter is None:
                 LOG.warning(
                     "No values returned by %(resource_id)s "
@@ -232,21 +279,34 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         for node_id, node in self.get_available_nodes().items():
             hosts_load[node_id] = {}
             hosts_load[node_id]['vcpus'] = node.vcpus
-
             for metric in self.metrics:
                 resource_id = ''
+                avg_meter = None
                 meter_name = self.instance_metrics[metric]
                 if re.match('^compute.node', meter_name) is not None:
                     resource_id = "%s_%s" % (node.uuid, node.hostname)
                 else:
                     resource_id = node_id
+                if self.config.datasource == "ceilometer":
+                    avg_meter = self.ceilometer.statistic_aggregation(
+                        resource_id=resource_id,
+                        meter_name=self.instance_metrics[metric],
+                        period=self.periods['node'],
+                        aggregate='avg'
+                    )
+                elif self.config.datasource == "gnocchi":
+                    stop_time = datetime.datetime.utcnow()
+                    start_time = stop_time - datetime.timedelta(
+                        seconds=int(self.periods['node']))
+                    avg_meter = self.gnocchi.statistic_aggregation(
+                        resource_id=resource_id,
+                        metric=self.instance_metrics[metric],
+                        granularity=self.granularity,
+                        start_time=start_time,
+                        stop_time=stop_time,
+                        aggregation='mean'
+                    )
 
-                avg_meter = self.ceilometer.statistic_aggregation(
-                    resource_id=resource_id,
-                    meter_name=self.instance_metrics[metric],
-                    period=self.periods['node'],
-                    aggregate='avg'
-                )
                 if avg_meter is None:
                     raise exception.NoSuchMetricForHost(
                         metric=meter_name,

watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py

@@ -17,23 +17,35 @@
 # limitations under the License.
 #
 
+import datetime
 import mock
 
+from watcher.common import clients
 from watcher.common import utils
 from watcher.decision_engine.model import model_root
 from watcher.decision_engine.strategy import strategies
 from watcher.tests import base
 from watcher.tests.decision_engine.model import ceilometer_metrics
 from watcher.tests.decision_engine.model import faker_cluster_state
+from watcher.tests.decision_engine.model import gnocchi_metrics
 
 
 class TestWorkloadStabilization(base.TestCase):
 
+    scenarios = [
+        ("Ceilometer",
+         {"datasource": "ceilometer",
+          "fake_datasource_cls": ceilometer_metrics.FakeCeilometerMetrics}),
+        ("Gnocchi",
+         {"datasource": "gnocchi",
+          "fake_datasource_cls": gnocchi_metrics.FakeGnocchiMetrics}),
+    ]
+
     def setUp(self):
         super(TestWorkloadStabilization, self).setUp()
 
         # fake metrics
-        self.fake_metrics = ceilometer_metrics.FakeCeilometerMetrics()
+        self.fake_metrics = self.fake_datasource_cls()
 
         # fake cluster
         self.fake_cluster = faker_cluster_state.FakerModelCollector()
@@ -45,17 +57,22 @@ class TestWorkloadStabilization(base.TestCase):
             'Node_3': {'cpu_util': 0.05, 'memory.resident': 8, 'vcpus': 40},
             'Node_4': {'cpu_util': 0.05, 'memory.resident': 4, 'vcpus': 40}}
 
+        p_osc = mock.patch.object(
+            clients, "OpenStackClients")
+        self.m_osc = p_osc.start()
+        self.addCleanup(p_osc.stop)
+
         p_model = mock.patch.object(
             strategies.WorkloadStabilization, "compute_model",
             new_callable=mock.PropertyMock)
         self.m_model = p_model.start()
         self.addCleanup(p_model.stop)
 
-        p_ceilometer = mock.patch.object(
+        p_datasource = mock.patch.object(
-            strategies.WorkloadStabilization, "ceilometer",
+            strategies.WorkloadStabilization, self.datasource,
             new_callable=mock.PropertyMock)
-        self.m_ceilometer = p_ceilometer.start()
+        self.m_datasource = p_datasource.start()
-        self.addCleanup(p_ceilometer.stop)
+        self.addCleanup(p_datasource.stop)
 
         p_audit_scope = mock.patch.object(
             strategies.WorkloadStabilization, "audit_scope",
@@ -65,10 +82,12 @@ class TestWorkloadStabilization(base.TestCase):
         self.addCleanup(p_audit_scope.stop)
 
         self.m_model.return_value = model_root.ModelRoot()
-        self.m_ceilometer.return_value = mock.Mock(
-            statistic_aggregation=self.fake_metrics.mock_get_statistics)
         self.m_audit_scope.return_value = mock.Mock()
-        self.strategy = strategies.WorkloadStabilization(config=mock.Mock())
+        self.m_datasource.return_value = mock.Mock(
+            statistic_aggregation=self.fake_metrics.mock_get_statistics)
+
+        self.strategy = strategies.WorkloadStabilization(
+            config=mock.Mock(datasource=self.datasource))
         self.strategy.input_parameters = utils.Struct()
         self.strategy.input_parameters.update(
             {'metrics': ["cpu_util", "memory.resident"],
@@ -109,17 +128,49 @@ class TestWorkloadStabilization(base.TestCase):
             strategies.WorkloadStabilization, "ceilometer")
         m_ceilometer = p_ceilometer.start()
         self.addCleanup(p_ceilometer.stop)
+        p_gnocchi = mock.patch.object(strategies.WorkloadStabilization,
+                                      "gnocchi")
+        m_gnocchi = p_gnocchi.start()
+        self.addCleanup(p_gnocchi.stop)
+        datetime_patcher = mock.patch.object(
+            datetime, 'datetime',
+            mock.Mock(wraps=datetime.datetime)
+        )
+        mocked_datetime = datetime_patcher.start()
+        mocked_datetime.utcnow.return_value = datetime.datetime(
+            2017, 3, 19, 18, 53, 11, 657417)
+        self.addCleanup(datetime_patcher.stop)
         m_ceilometer.return_value = mock.Mock(
             statistic_aggregation=self.fake_metrics.mock_get_statistics)
+        m_gnocchi.return_value = mock.Mock(
+            statistic_aggregation=self.fake_metrics.mock_get_statistics)
         instance0 = model.get_instance_by_uuid("INSTANCE_0")
         self.strategy.get_instance_load(instance0)
-        m_ceilometer.statistic_aggregation.assert_called_with(
+        if self.strategy.config.datasource == "ceilometer":
-            aggregate='min', meter_name='memory.resident',
+            m_ceilometer.statistic_aggregation.assert_called_with(
-            period=720, resource_id=instance0.uuid)
+                aggregate='min', meter_name='memory.resident',
+                period=720, resource_id=instance0.uuid)
+        elif self.strategy.config.datasource == "gnocchi":
+            stop_time = datetime.datetime.utcnow()
+            start_time = stop_time - datetime.timedelta(
+                seconds=int('720'))
+            m_gnocchi.statistic_aggregation.assert_called_with(
+                resource_id=instance0.uuid, metric='memory.resident',
+                granularity=300, start_time=start_time, stop_time=stop_time,
+                aggregation='mean')
         self.strategy.get_hosts_load()
-        m_ceilometer.statistic_aggregation.assert_called_with(
+        if self.strategy.config.datasource == "ceilometer":
-            aggregate='avg', meter_name='hardware.memory.used',
+            m_ceilometer.statistic_aggregation.assert_called_with(
-            period=600, resource_id=mock.ANY)
+                aggregate='avg', meter_name='hardware.memory.used',
+                period=600, resource_id=mock.ANY)
+        elif self.strategy.config.datasource == "gnocchi":
+            stop_time = datetime.datetime.utcnow()
+            start_time = stop_time - datetime.timedelta(
+                seconds=int('600'))
+            m_gnocchi.statistic_aggregation.assert_called_with(
+                resource_id=mock.ANY, metric='hardware.memory.used',
+                granularity=300, start_time=start_time, stop_time=stop_time,
+                aggregation='mean')
 
     def test_normalize_hosts_load(self):
         self.m_model.return_value = self.fake_cluster.generate_scenario_1()