Merge "Implemented GMR plugin to show CDM structures"

requirements.txt

@@ -7,6 +7,7 @@ enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' #
 jsonpatch>=1.1 # BSD
 keystoneauth1>=2.10.0 # Apache-2.0
 keystonemiddleware!=4.1.0,!=4.5.0,>=4.0.0 # Apache-2.0
+lxml>=2.3 # BSD
 oslo.concurrency>=3.8.0 # Apache-2.0
 oslo.cache>=1.5.0 # Apache-2.0
 oslo.config>=3.14.0 # Apache-2.0

watcher/cmd/decisionengine.py

@@ -25,6 +25,7 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as watcher_service
+from watcher.decision_engine import gmr
 from watcher.decision_engine import manager
 from watcher.decision_engine import scheduling
 from watcher.decision_engine import sync
@@ -35,6 +36,7 @@ CONF = cfg.CONF
 
 def main():
     watcher_service.prepare_service(sys.argv)
+    gmr.register_gmr_plugins()
 
     LOG.info(_LI('Starting Watcher Decision Engine service in PID %s'),
              os.getpid())

watcher/decision_engine/gmr.py

@@ -0,0 +1,48 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_reports import guru_meditation_report as gmr
+
+from watcher._i18n import _
+from watcher.decision_engine.model.collector import manager
+
+
+def register_gmr_plugins():
+    """Register GMR plugins that are specific to watcher-decision-engine."""
+    gmr.TextGuruMeditation.register_section(_('CDMCs'), show_models)
+
+
+def show_models():
+    """Create a formatted output of all the CDMs
+
+    Mainly used as a Guru Meditation Report (GMR) plugin
+    """
+    mgr = manager.CollectorManager()
+
+    output = []
+    for name, cdmc in mgr.get_collectors().items():
+        output.append("")
+        output.append("~" * len(name))
+        output.append(name)
+        output.append("~" * len(name))
+        output.append("")
+
+        cdmc_struct = cdmc.cluster_data_model.to_string()
+        output.append(cdmc_struct)
+
+    return "\n".join(output)

watcher/decision_engine/model/model_root.py

@@ -14,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
+
+from lxml import etree
 import six
 
 from watcher._i18n import _
@@ -159,3 +162,61 @@ class ModelRoot(object):
 
     def get_resource_from_id(self, resource_id):
         return self.resource[str(resource_id)]
+
+    def get_node_instances(self, node):
+        return self.mapping.get_node_instances(node)
+
+    def _build_compute_node_element(self, compute_node):
+        attrib = collections.OrderedDict(
+            uuid=compute_node.uuid, human_id=compute_node.human_id,
+            hostname=compute_node.hostname, state=compute_node.state,
+            status=compute_node.status)
+
+        for resource_name, resource in self.resource.items():
+            res_value = resource.get_capacity(compute_node)
+            if res_value is not None:
+                attrib[resource_name] = six.text_type(res_value)
+
+        compute_node_el = etree.Element("ComputeNode", attrib=attrib)
+
+        return compute_node_el
+
+    def _build_instance_element(self, instance):
+        attrib = collections.OrderedDict(
+            uuid=instance.uuid, human_id=instance.human_id,
+            hostname=instance.hostname, state=instance.state)
+
+        for resource_name, resource in self.resource.items():
+            res_value = resource.get_capacity(instance)
+            if res_value is not None:
+                attrib[resource_name] = six.text_type(res_value)
+
+        instance_el = etree.Element("Instance", attrib=attrib)
+
+        return instance_el
+
+    def to_string(self):
+        root = etree.Element("ModelRoot")
+        # Build compute node tree
+        for cn in sorted(self.get_all_compute_nodes().values(),
+                         key=lambda cn: cn.uuid):
+            compute_node_el = self._build_compute_node_element(cn)
+
+            # Build mapped instance tree
+            node_instance_uuids = self.get_node_instances(cn)
+            for instance_uuid in sorted(node_instance_uuids):
+                instance = self.get_instance_from_id(instance_uuid)
+                instance_el = self._build_instance_element(instance)
+                compute_node_el.append(instance_el)
+
+            root.append(compute_node_el)
+
+        # Build unmapped instance tree (i.e. not assigned to any compute node)
+        for instance in sorted(self.get_all_instances().values(),
+                               key=lambda inst: inst.uuid):
+            try:
+                self.get_node_from_instance_id(instance.uuid)
+            except exception.InstanceNotFound:
+                root.append(self._build_instance_element(instance))
+
+        return etree.tostring(root, pretty_print=True).decode('utf-8')

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -252,11 +252,11 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
         :rtype: float
         """
         resource_id = "%s_%s" % (node.uuid, node.hostname)
-        host_avg_cpu_util = self.ceilometer. \
+        host_avg_cpu_util = self.ceilometer.statistic_aggregation(
-            statistic_aggregation(resource_id=resource_id,
+            resource_id=resource_id,
-                                  meter_name=self.HOST_CPU_USAGE_METRIC_NAME,
+            meter_name=self.HOST_CPU_USAGE_METRIC_NAME,
-                                  period="7200",
+            period="7200",
-                                  aggregate='avg')
+            aggregate='avg')
 
         if host_avg_cpu_util is None:
             LOG.error(
@@ -269,7 +269,7 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
         cpu_capacity = self.compute_model.get_resource_from_id(
             element.ResourceType.cpu_cores).get_capacity(node)
 
-        total_cores_used = cpu_capacity * (host_avg_cpu_util / 100)
+        total_cores_used = cpu_capacity * (host_avg_cpu_util / 100.0)
 
         return self.calculate_weight(node, total_cores_used, 0, 0)
 

watcher/tests/decision_engine/model/test_model.py

@@ -15,9 +15,12 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 import uuid
 
+from lxml import etree
+import six
+
 from watcher.common import exception
 from watcher.decision_engine.model import element
 from watcher.decision_engine.model import model_root
@@ -27,7 +30,8 @@ from watcher.tests.decision_engine.strategy.strategies \
 
 
 class TestModel(base.TestCase):
-    def test_model(self):
+
+    def test_model_structure(self):
         fake_cluster = faker_cluster_state.FakerModelCollector()
         model = fake_cluster.generate_scenario_1()
 
@@ -35,6 +39,149 @@ class TestModel(base.TestCase):
         self.assertEqual(35, len(model._instances))
         self.assertEqual(5, len(model.mapping.get_mapping()))
 
+        expected_struct_str = """
+        <ModelRoot>
+          <ComputeNode ResourceType.cpu_cores="40" ResourceType.disk="250"
+            ResourceType.memory="132" hostname="hostname_0" human_id=""
+            state="up" status="enabled" uuid="Node_0">
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_0"/>
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_1"/>
+          </ComputeNode>
+          <ComputeNode ResourceType.cpu_cores="40" ResourceType.disk="250"
+            ResourceType.memory="132" hostname="hostname_1" human_id=""
+            state="up" status="enabled" uuid="Node_1">
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_2"/>
+          </ComputeNode>
+          <ComputeNode ResourceType.cpu_cores="40" ResourceType.disk="250"
+            ResourceType.memory="132" hostname="hostname_2" human_id=""
+            state="up" status="enabled" uuid="Node_2">
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_3"/>
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_4"/>
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_5"/>
+          </ComputeNode>
+          <ComputeNode ResourceType.cpu_cores="40" ResourceType.disk="250"
+            ResourceType.memory="132" hostname="hostname_3" human_id=""
+            state="up" status="enabled" uuid="Node_3">
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_6"/>
+          </ComputeNode>
+          <ComputeNode ResourceType.cpu_cores="40" ResourceType.disk="250"
+            ResourceType.memory="132" hostname="hostname_4" human_id=""
+            state="up" status="enabled" uuid="Node_4">
+            <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+              ResourceType.memory="2" hostname="" human_id=""
+              state="active" uuid="INSTANCE_7"/>
+          </ComputeNode>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_10"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_11"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_12"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_13"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_14"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_15"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_16"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_17"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_18"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_19"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_20"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_21"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_22"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_23"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_24"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_25"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_26"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_27"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_28"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_29"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_30"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_31"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_32"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_33"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_34"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_8"/>
+          <Instance ResourceType.cpu_cores="10" ResourceType.disk="20"
+            ResourceType.memory="2" hostname="" human_id=""
+            state="active" uuid="INSTANCE_9"/>
+        </ModelRoot>
+        """
+        parser = etree.XMLParser(remove_blank_text=True)
+        expected_struct = etree.fromstring(expected_struct_str, parser)
+        model_structure = etree.fromstring(model.to_string(), parser)
+
+        normalized_expected_output = six.BytesIO()
+        normalized_model_output = six.BytesIO()
+        expected_struct.getroottree().write_c14n(normalized_expected_output)
+        model_structure.getroottree().write_c14n(normalized_model_output)
+
+        normalized_expected_struct = normalized_expected_output.getvalue()
+        normalized_model_struct = normalized_model_output.getvalue()
+
+        self.assertEqual(normalized_expected_struct, normalized_model_struct)
+
     def test_add_node(self):
         model = model_root.ModelRoot()
         id_ = "{0}".format(uuid.uuid4())