From a2cb02a8614c8d758086caa1d737472bfc9d8f4a Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Singh Date: Mon, 27 Mar 2017 08:05:18 +0000 Subject: [PATCH] Prevent the migration of VM with 'optimize' False in VM metadata This patch adds the functionality to filter out VMs which have metadata field 'optimize' set to False. This patch implements the functionality for basic_consolidation strategy. Change-Id: Iaf7b63e09534e4a67406e7f092242558b78c0bde Partially-Implements: BP audit-tag-vm-metadata --- .../decision_engine/model/element/instance.py | 2 +- watcher/decision_engine/model/model_root.py | 7 ++ .../strategy/strategies/base.py | 18 +++++ .../strategies/basic_consolidation.py | 10 ++- watcher/objects/fields.py | 19 +++++ .../decision_engine/model/data/scenario_1.xml | 70 +++++++++---------- .../model/data/scenario_1_with_metrics.xml | 4 +- .../model/data/scenario_2_with_metrics.xml | 12 ++-- .../model/data/scenario_3_with_2_nodes.xml | 4 +- .../model/data/scenario_3_with_metrics.xml | 8 +-- .../data/scenario_5_with_instance_disk_0.xml | 2 +- .../model/data/scenario_6_with_2_nodes.xml | 8 +-- .../model/data/scenario_7_with_2_nodes.xml | 8 +-- .../model/data/scenario_8_with_4_nodes.xml | 12 ++-- ..._9_with_3_active_plus_1_disabled_nodes.xml | 12 ++-- .../model/faker_cluster_state.py | 2 + .../tests/decision_engine/model/test_model.py | 24 ++----- 17 files changed, 131 insertions(+), 91 deletions(-) diff --git a/watcher/decision_engine/model/element/instance.py b/watcher/decision_engine/model/element/instance.py index 451998356..ebdb16d7b 100644 --- a/watcher/decision_engine/model/element/instance.py +++ b/watcher/decision_engine/model/element/instance.py @@ -48,7 +48,7 @@ class Instance(compute_resource.ComputeResource): "disk": wfields.IntegerField(), "disk_capacity": wfields.NonNegativeIntegerField(), "vcpus": wfields.NonNegativeIntegerField(), - "metadata": wfields.DictField(), + "metadata": wfields.JsonField(), } def accept(self, visitor): diff --git a/watcher/decision_engine/model/model_root.py b/watcher/decision_engine/model/model_root.py index c1843bb90..4daeff5ee 100644 --- a/watcher/decision_engine/model/model_root.py +++ b/watcher/decision_engine/model/model_root.py @@ -235,3 +235,10 @@ class ModelRoot(nx.DiGraph, base.Model): model.add_instance(instance) return model + + @classmethod + def is_isomorphic(cls, G1, G2): + def node_match(node1, node2): + return node1.as_dict() == node2.as_dict() + return nx.algorithms.isomorphism.isomorph.is_isomorphic( + G1, G2, node_match=node_match) diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index 13c241032..effea1e5f 100644 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -39,6 +39,8 @@ which are dynamically loaded by Watcher at launch time. import abc import six +from oslo_utils import strutils + from watcher.common import clients from watcher.common import context from watcher.common import exception @@ -264,6 +266,22 @@ class BaseStrategy(loadable.Loadable): def state_collector(self, s): self._cluster_state_collector = s + def filter_instances_by_audit_tag(self, instances): + if not self.config.check_optimize_metadata: + return instances + instances_to_migrate = [] + for instance in instances: + optimize = True + if instance.metadata: + try: + optimize = strutils.bool_from_string( + instance.metadata.get('optimize')) + except ValueError: + optimize = False + if optimize: + instances_to_migrate.append(instance) + return instances_to_migrate + @six.add_metaclass(abc.ABCMeta) class DummyBaseStrategy(BaseStrategy): diff --git a/watcher/decision_engine/strategy/strategies/basic_consolidation.py b/watcher/decision_engine/strategy/strategies/basic_consolidation.py index 2bb776510..6d9067ff6 100644 --- a/watcher/decision_engine/strategy/strategies/basic_consolidation.py +++ b/watcher/decision_engine/strategy/strategies/basic_consolidation.py @@ -159,7 +159,12 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy): "datasource", help="Data source to use in order to query the needed metrics", default="ceilometer", - choices=["ceilometer", "monasca", "gnocchi"]) + choices=["ceilometer", "monasca", "gnocchi"]), + cfg.BoolOpt( + "check_optimize_metadata", + help="Check optimize metadata field in instance before " + "migration", + default=False), ] @property @@ -437,9 +442,10 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy): def node_and_instance_score(self, sorted_scores): """Get List of VMs from node""" node_to_release = sorted_scores[len(sorted_scores) - 1][0] - instances_to_migrate = self.compute_model.get_node_instances( + instances = self.compute_model.get_node_instances( self.compute_model.get_node_by_uuid(node_to_release)) + instances_to_migrate = self.filter_instances_by_audit_tag(instances) instance_score = [] for instance in instances_to_migrate: if instance.state == element.InstanceState.ACTIVE.value: diff --git a/watcher/objects/fields.py b/watcher/objects/fields.py index e05c7b870..d0df8548a 100644 --- a/watcher/objects/fields.py +++ b/watcher/objects/fields.py @@ -17,6 +17,7 @@ import ast import six +from oslo_serialization import jsonutils from oslo_versionedobjects import fields @@ -96,8 +97,26 @@ class FlexibleListOfDictField(fields.AutoTypedField): super(FlexibleListOfDictField, self)._null(obj, attr) +class Json(fields.FieldType): + def coerce(self, obj, attr, value): + if isinstance(value, six.string_types): + loaded = jsonutils.loads(value) + return loaded + return value + + def from_primitive(self, obj, attr, value): + return self.coerce(obj, attr, value) + + def to_primitive(self, obj, attr, value): + return jsonutils.dumps(value) + + +class JsonField(fields.AutoTypedField): + AUTO_TYPE = Json() + # ### Notification fields ### # + class BaseWatcherEnum(Enum): ALL = () diff --git a/watcher/tests/decision_engine/model/data/scenario_1.xml b/watcher/tests/decision_engine/model/data/scenario_1.xml index dcf08626d..7476af277 100644 --- a/watcher/tests/decision_engine/model/data/scenario_1.xml +++ b/watcher/tests/decision_engine/model/data/scenario_1.xml @@ -1,47 +1,47 @@ - - + + - + - - - + + + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/watcher/tests/decision_engine/model/data/scenario_1_with_metrics.xml b/watcher/tests/decision_engine/model/data/scenario_1_with_metrics.xml index 3edf6d17e..d8b80afca 100644 --- a/watcher/tests/decision_engine/model/data/scenario_1_with_metrics.xml +++ b/watcher/tests/decision_engine/model/data/scenario_1_with_metrics.xml @@ -1,8 +1,8 @@ - + - + diff --git a/watcher/tests/decision_engine/model/data/scenario_2_with_metrics.xml b/watcher/tests/decision_engine/model/data/scenario_2_with_metrics.xml index 6edba3e7d..668ef1985 100644 --- a/watcher/tests/decision_engine/model/data/scenario_2_with_metrics.xml +++ b/watcher/tests/decision_engine/model/data/scenario_2_with_metrics.xml @@ -1,11 +1,11 @@ - - - - - - + + + + + + diff --git a/watcher/tests/decision_engine/model/data/scenario_3_with_2_nodes.xml b/watcher/tests/decision_engine/model/data/scenario_3_with_2_nodes.xml index fee3a27eb..189d81ee9 100644 --- a/watcher/tests/decision_engine/model/data/scenario_3_with_2_nodes.xml +++ b/watcher/tests/decision_engine/model/data/scenario_3_with_2_nodes.xml @@ -1,8 +1,8 @@ - + - + diff --git a/watcher/tests/decision_engine/model/data/scenario_3_with_metrics.xml b/watcher/tests/decision_engine/model/data/scenario_3_with_metrics.xml index b8987ffd5..5c6079916 100644 --- a/watcher/tests/decision_engine/model/data/scenario_3_with_metrics.xml +++ b/watcher/tests/decision_engine/model/data/scenario_3_with_metrics.xml @@ -1,9 +1,9 @@ - - - - + + + + diff --git a/watcher/tests/decision_engine/model/data/scenario_5_with_instance_disk_0.xml b/watcher/tests/decision_engine/model/data/scenario_5_with_instance_disk_0.xml index 72caedbbb..a8bffab30 100644 --- a/watcher/tests/decision_engine/model/data/scenario_5_with_instance_disk_0.xml +++ b/watcher/tests/decision_engine/model/data/scenario_5_with_instance_disk_0.xml @@ -1,5 +1,5 @@ - + diff --git a/watcher/tests/decision_engine/model/data/scenario_6_with_2_nodes.xml b/watcher/tests/decision_engine/model/data/scenario_6_with_2_nodes.xml index 1ddcf7cab..c12eabaa5 100644 --- a/watcher/tests/decision_engine/model/data/scenario_6_with_2_nodes.xml +++ b/watcher/tests/decision_engine/model/data/scenario_6_with_2_nodes.xml @@ -1,10 +1,10 @@ - - + + - - + + diff --git a/watcher/tests/decision_engine/model/data/scenario_7_with_2_nodes.xml b/watcher/tests/decision_engine/model/data/scenario_7_with_2_nodes.xml index 1da834336..99ee529a8 100644 --- a/watcher/tests/decision_engine/model/data/scenario_7_with_2_nodes.xml +++ b/watcher/tests/decision_engine/model/data/scenario_7_with_2_nodes.xml @@ -1,10 +1,10 @@ - - + + - - + + diff --git a/watcher/tests/decision_engine/model/data/scenario_8_with_4_nodes.xml b/watcher/tests/decision_engine/model/data/scenario_8_with_4_nodes.xml index 6e619c4a7..a646c6e90 100644 --- a/watcher/tests/decision_engine/model/data/scenario_8_with_4_nodes.xml +++ b/watcher/tests/decision_engine/model/data/scenario_8_with_4_nodes.xml @@ -1,16 +1,16 @@ - - - + + + - + - + - + diff --git a/watcher/tests/decision_engine/model/data/scenario_9_with_3_active_plus_1_disabled_nodes.xml b/watcher/tests/decision_engine/model/data/scenario_9_with_3_active_plus_1_disabled_nodes.xml index 33396d824..d1d3f94eb 100644 --- a/watcher/tests/decision_engine/model/data/scenario_9_with_3_active_plus_1_disabled_nodes.xml +++ b/watcher/tests/decision_engine/model/data/scenario_9_with_3_active_plus_1_disabled_nodes.xml @@ -1,16 +1,16 @@ - - - + + + - + - + - + diff --git a/watcher/tests/decision_engine/model/faker_cluster_state.py b/watcher/tests/decision_engine/model/faker_cluster_state.py index 66dd5e1cd..2be8222ec 100644 --- a/watcher/tests/decision_engine/model/faker_cluster_state.py +++ b/watcher/tests/decision_engine/model/faker_cluster_state.py @@ -85,6 +85,8 @@ class FakerModelCollector(base.BaseClusterDataModelCollector): "disk": 20, "disk_capacity": 20, "vcpus": 10, + "metadata": + '{"optimize": true,"top": "floor","nested": {"x": "y"}}' } instance = element.Instance(**instance_attributes) diff --git a/watcher/tests/decision_engine/model/test_model.py b/watcher/tests/decision_engine/model/test_model.py index 65c364afe..d8438dc43 100644 --- a/watcher/tests/decision_engine/model/test_model.py +++ b/watcher/tests/decision_engine/model/test_model.py @@ -18,9 +18,7 @@ import os -from lxml import etree from oslo_utils import uuidutils -import six from watcher.common import exception from watcher.decision_engine.model import element @@ -45,26 +43,16 @@ class TestModel(base.TestCase): def test_model_structure(self): fake_cluster = faker_cluster_state.FakerModelCollector() - model = fake_cluster.build_scenario_1() + model1 = fake_cluster.build_scenario_1() - self.assertEqual(5, len(model.get_all_compute_nodes())) - self.assertEqual(35, len(model.get_all_instances())) - self.assertEqual(8, len(model.edges())) + self.assertEqual(5, len(model1.get_all_compute_nodes())) + self.assertEqual(35, len(model1.get_all_instances())) + self.assertEqual(8, len(model1.edges())) expected_struct_str = self.load_data('scenario_1.xml') - parser = etree.XMLParser(remove_blank_text=True) - expected_struct = etree.fromstring(expected_struct_str, parser) - model_structure = etree.fromstring(model.to_string(), parser) + model2 = model_root.ModelRoot.from_xml(expected_struct_str) - normalized_expected_output = six.BytesIO() - normalized_model_output = six.BytesIO() - expected_struct.getroottree().write_c14n(normalized_expected_output) - model_structure.getroottree().write_c14n(normalized_model_output) - - normalized_expected_struct = normalized_expected_output.getvalue() - normalized_model_struct = normalized_model_output.getvalue() - - self.assertEqual(normalized_expected_struct, normalized_model_struct) + self.assertTrue(model_root.ModelRoot.is_isomorphic(model2, model1)) def test_build_model_from_xml(self): fake_cluster = faker_cluster_state.FakerModelCollector()