From 0a899a2dc28575dc855899c6d2c57cd3218947b2 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Singh Date: Mon, 17 Apr 2017 07:46:16 +0000 Subject: [PATCH] Add host_aggregates in exclude rule of audit scope Currently if user wants to skip some host_aggregates from audit, it is not possible. This patch adds host_aggregates into the exclude rule of audit scope. This patch also implements audit-tag-vm-metadata using scopes. TODOs: 1. Add tests 2. Remove old implementation of audit-tag-vm-metadata Change-Id: Ie86378cb02145a660bbf446eedb29dc311fa29d7 Implements: BP audit-tag-vm-metadata --- watcher/api/controllers/v1/audit_template.py | 15 ++++ watcher/decision_engine/scope/base.py | 3 +- watcher/decision_engine/scope/default.py | 57 +++++++++++-- .../strategy/strategies/base.py | 2 +- watcher/tests/api/v1/test_audit_templates.py | 23 ++++++ .../decision_engine/scope/test_default.py | 81 ++++++++++++++----- .../strategies/test_basic_consolidation.py | 4 +- 7 files changed, 157 insertions(+), 28 deletions(-) diff --git a/watcher/api/controllers/v1/audit_template.py b/watcher/api/controllers/v1/audit_template.py index b03d4a535..b85e2f28a 100644 --- a/watcher/api/controllers/v1/audit_template.py +++ b/watcher/api/controllers/v1/audit_template.py @@ -109,6 +109,21 @@ class AuditTemplatePostType(wtypes.Base): common_utils.Draft4Validator( default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope) + include_host_aggregates = False + exclude_host_aggregates = False + for rule in audit_template.scope: + if 'host_aggregates' in rule: + include_host_aggregates = True + elif 'exclude' in rule: + for resource in rule['exclude']: + if 'host_aggregates' in resource: + exclude_host_aggregates = True + if include_host_aggregates and exclude_host_aggregates: + raise exception.Invalid( + message=_( + "host_aggregates can't be " + "included and excluded together")) + if audit_template.strategy: available_strategies = objects.Strategy.list( AuditTemplatePostType._ctx) diff --git a/watcher/decision_engine/scope/base.py b/watcher/decision_engine/scope/base.py index 69c14b8fc..76f17467f 100644 --- a/watcher/decision_engine/scope/base.py +++ b/watcher/decision_engine/scope/base.py @@ -29,9 +29,10 @@ class BaseScope(object): requires Cluster Data Model which can be segregated to achieve audit scope. """ - def __init__(self, scope): + def __init__(self, scope, config): self.ctx = context.make_context() self.scope = scope + self.config = config @abc.abstractmethod def get_scoped_model(self, cluster_model): diff --git a/watcher/decision_engine/scope/default.py b/watcher/decision_engine/scope/default.py index dfc0a2de8..4e74f6e2b 100644 --- a/watcher/decision_engine/scope/default.py +++ b/watcher/decision_engine/scope/default.py @@ -82,6 +82,23 @@ class DefaultScope(base.BaseScope): } } } + }, + "host_aggregates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "anyOf": [ + {"type": ["string", "number"]} + ] + }, + } + }, + "instance_metadata": { + "type": "array", + "items": { + "type": "object" + } } }, "additionalProperties": False @@ -92,8 +109,8 @@ class DefaultScope(base.BaseScope): } } - def __init__(self, scope, osc=None): - super(DefaultScope, self).__init__(scope) + def __init__(self, scope, config, osc=None): + super(DefaultScope, self).__init__(scope, config) self._osc = osc self.wrapper = nova_helper.NovaHelper(osc=self._osc) @@ -110,7 +127,7 @@ class DefaultScope(base.BaseScope): resource="host aggregates") return False - def _collect_aggregates(self, host_aggregates, allowed_nodes): + def _collect_aggregates(self, host_aggregates, compute_nodes): aggregate_list = self.wrapper.get_aggregate_list() aggregate_ids = [aggregate['id'] for aggregate in host_aggregates if 'id' in aggregate] @@ -125,7 +142,7 @@ class DefaultScope(base.BaseScope): if (detailed_aggregate.id in aggregate_ids or detailed_aggregate.name in aggregate_names or include_all_nodes): - allowed_nodes.extend(detailed_aggregate.hosts) + compute_nodes.extend(detailed_aggregate.hosts) def _collect_zones(self, availability_zones, allowed_nodes): zone_list = self.wrapper.get_availability_zone_list() @@ -145,6 +162,8 @@ class DefaultScope(base.BaseScope): def exclude_resources(self, resources, **kwargs): instances_to_exclude = kwargs.get('instances') nodes_to_exclude = kwargs.get('nodes') + instance_metadata = kwargs.get('instance_metadata') + for resource in resources: if 'instances' in resource: instances_to_exclude.extend( @@ -154,6 +173,14 @@ class DefaultScope(base.BaseScope): nodes_to_exclude.extend( [host['name'] for host in resource['compute_nodes']]) + elif 'host_aggregates' in resource: + prohibited_nodes = [] + self._collect_aggregates(resource['host_aggregates'], + prohibited_nodes) + nodes_to_exclude.extend(prohibited_nodes) + elif 'instance_metadata' in resource: + instance_metadata.extend( + [metadata for metadata in resource['instance_metadata']]) def remove_nodes_from_model(self, nodes_to_remove, cluster_model): for node_uuid in nodes_to_remove: @@ -179,6 +206,19 @@ class DefaultScope(base.BaseScope): cluster_model.get_instance_by_uuid(instance_uuid), node_name) + def exclude_instances_with_given_metadata( + self, instance_metadata, cluster_model, instances_to_remove): + metadata_dict = { + key: val for d in instance_metadata for key, val in d.items()} + instances = cluster_model.get_all_instances() + for uuid, instance in instances.items(): + metadata = instance.metadata + common_metadata = set(metadata_dict) & set(metadata) + if common_metadata and len(common_metadata) == len(metadata_dict): + for key, value in metadata_dict.items(): + if str(value).lower() == str(metadata.get(key)).lower(): + instances_to_remove.add(uuid) + def get_scoped_model(self, cluster_model): """Leave only nodes and instances proposed in the audit scope""" if not cluster_model: @@ -188,6 +228,7 @@ class DefaultScope(base.BaseScope): nodes_to_exclude = [] nodes_to_remove = set() instances_to_exclude = [] + instance_metadata = [] model_hosts = list(cluster_model.get_all_compute_nodes().keys()) if not self.scope: @@ -203,7 +244,8 @@ class DefaultScope(base.BaseScope): elif 'exclude' in rule: self.exclude_resources( rule['exclude'], instances=instances_to_exclude, - nodes=nodes_to_exclude) + nodes=nodes_to_exclude, + instance_metadata=instance_metadata) instances_to_remove = set(instances_to_exclude) if allowed_nodes: @@ -211,6 +253,11 @@ class DefaultScope(base.BaseScope): nodes_to_remove.update(nodes_to_exclude) self.remove_nodes_from_model(nodes_to_remove, cluster_model) + + if instance_metadata and self.config.check_optimize_metadata: + self.exclude_instances_with_given_metadata( + instance_metadata, cluster_model, instances_to_remove) + self.remove_instances_from_model(instances_to_remove, cluster_model) return cluster_model diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index effea1e5f..2ac2cf410 100644 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -235,7 +235,7 @@ class BaseStrategy(loadable.Loadable): def audit_scope_handler(self): if not self._audit_scope_handler: self._audit_scope_handler = default_scope.DefaultScope( - self.audit_scope) + self.audit_scope, self.config) return self._audit_scope_handler @property diff --git a/watcher/tests/api/v1/test_audit_templates.py b/watcher/tests/api/v1/test_audit_templates.py index 0f5bda059..6ce6451ba 100644 --- a/watcher/tests/api/v1/test_audit_templates.py +++ b/watcher/tests/api/v1/test_audit_templates.py @@ -13,6 +13,7 @@ import datetime import itertools import mock +from webtest.app import AppError from oslo_config import cfg from oslo_serialization import jsonutils @@ -36,6 +37,7 @@ def post_get_test_audit_template(**kw): strategy = db_utils.get_test_strategy(goal_id=goal['id']) kw['goal'] = kw.get('goal', goal['uuid']) kw['strategy'] = kw.get('strategy', strategy['uuid']) + kw['scope'] = kw.get('scope', []) audit_template = api_utils.audit_template_post_data(**kw) return audit_template @@ -510,6 +512,27 @@ class TestPost(FunctionalTestWithSetup): response.json['created_at']).replace(tzinfo=None) self.assertEqual(test_time, return_created_at) + def test_create_audit_template_vlidation_with_aggregates(self): + scope = [{'host_aggregates': [{'id': '*'}]}, + {'availability_zones': [{'name': 'AZ1'}, + {'name': 'AZ2'}]}, + {'exclude': [ + {'instances': [ + {'uuid': 'INSTANCE_1'}, + {'uuid': 'INSTANCE_2'}]}, + {'compute_nodes': [ + {'name': 'Node_1'}, + {'name': 'Node_2'}]}, + {'host_aggregates': [{'id': '*'}]} + ]} + ] + audit_template_dict = post_get_test_audit_template( + goal=self.fake_goal1.uuid, + strategy=self.fake_strategy1.uuid, scope=scope) + with self.assertRaisesRegexp(AppError, + "be included and excluded together"): + self.post_json('/audit_templates', audit_template_dict) + def test_create_audit_template_does_autogenerate_id(self): audit_template_dict = post_get_test_audit_template( goal=self.fake_goal1.uuid, strategy=None) diff --git a/watcher/tests/decision_engine/scope/test_default.py b/watcher/tests/decision_engine/scope/test_default.py index ab512ad82..06e7572d5 100644 --- a/watcher/tests/decision_engine/scope/test_default.py +++ b/watcher/tests/decision_engine/scope/test_default.py @@ -40,7 +40,7 @@ class TestDefaultScope(base.TestCase): mock.Mock(zoneName='AZ{0}'.format(i), hosts={'Node_{0}'.format(i): {}}) for i in range(2)] - model = default.DefaultScope(audit_scope, + model = default.DefaultScope(audit_scope, mock.Mock(), osc=mock.Mock()).get_scoped_model(cluster) expected_edges = [('INSTANCE_2', 'Node_1')] self.assertEqual(sorted(expected_edges), sorted(model.edges())) @@ -48,13 +48,13 @@ class TestDefaultScope(base.TestCase): @mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list') def test_get_scoped_model_without_scope(self, mock_zone_list): model = self.fake_cluster.generate_scenario_1() - default.DefaultScope([], + default.DefaultScope([], mock.Mock(), osc=mock.Mock()).get_scoped_model(model) assert not mock_zone_list.called def test_remove_instance(self): model = self.fake_cluster.generate_scenario_1() - default.DefaultScope([], osc=mock.Mock()).remove_instance( + default.DefaultScope([], mock.Mock(), osc=mock.Mock()).remove_instance( model, model.get_instance_by_uuid('INSTANCE_2'), 'Node_1') expected_edges = [ ('INSTANCE_0', 'Node_0'), @@ -75,7 +75,7 @@ class TestDefaultScope(base.TestCase): mock_detailed_aggregate.side_effect = [ mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)] default.DefaultScope([{'host_aggregates': [{'id': 1}, {'id': 2}]}], - osc=mock.Mock())._collect_aggregates( + mock.Mock(), osc=mock.Mock())._collect_aggregates( [{'id': 1}, {'id': 2}], allowed_nodes) self.assertEqual(['Node_1'], allowed_nodes) @@ -88,7 +88,7 @@ class TestDefaultScope(base.TestCase): mock_detailed_aggregate.side_effect = [ mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)] default.DefaultScope([{'host_aggregates': [{'id': '*'}]}], - osc=mock.Mock())._collect_aggregates( + mock.Mock(), osc=mock.Mock())._collect_aggregates( [{'id': '*'}], allowed_nodes) self.assertEqual(['Node_0', 'Node_1'], allowed_nodes) @@ -98,7 +98,7 @@ class TestDefaultScope(base.TestCase): mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)] scope_handler = default.DefaultScope( [{'host_aggregates': [{'id': '*'}, {'id': 1}]}], - osc=mock.Mock()) + mock.Mock(), osc=mock.Mock()) self.assertRaises(exception.WildcardCharacterIsUsed, scope_handler._collect_aggregates, [{'id': '*'}, {'id': 1}], @@ -121,7 +121,7 @@ class TestDefaultScope(base.TestCase): default.DefaultScope([{'host_aggregates': [{'name': 'HA_1'}, {'id': 0}]}], - osc=mock.Mock())._collect_aggregates( + mock.Mock(), osc=mock.Mock())._collect_aggregates( [{'name': 'HA_1'}, {'id': 0}], allowed_nodes) self.assertEqual(['Node_0', 'Node_1'], allowed_nodes) @@ -134,7 +134,7 @@ class TestDefaultScope(base.TestCase): 'Node_{0}'.format(2 * i + 1): 2}) for i in range(2)] default.DefaultScope([{'availability_zones': [{'name': "AZ1"}]}], - osc=mock.Mock())._collect_zones( + mock.Mock(), osc=mock.Mock())._collect_zones( [{'name': "AZ1"}], allowed_nodes) self.assertEqual(['Node_0', 'Node_1'], sorted(allowed_nodes)) @@ -147,7 +147,7 @@ class TestDefaultScope(base.TestCase): 'Node_{0}'.format(2 * i + 1): 2}) for i in range(2)] default.DefaultScope([{'availability_zones': [{'name': "*"}]}], - osc=mock.Mock())._collect_zones( + mock.Mock(), osc=mock.Mock())._collect_zones( [{'name': "*"}], allowed_nodes) self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'], sorted(allowed_nodes)) @@ -162,7 +162,7 @@ class TestDefaultScope(base.TestCase): for i in range(2)] scope_handler = default.DefaultScope( [{'availability_zones': [{'name': "*"}, {'name': 'AZ1'}]}], - osc=mock.Mock()) + mock.Mock(), osc=mock.Mock()) self.assertRaises(exception.WildcardCharacterIsUsed, scope_handler._collect_zones, [{'name': "*"}, {'name': 'AZ1'}], @@ -173,23 +173,65 @@ class TestDefaultScope(base.TestCase): validators.Draft4Validator( default.DefaultScope.DEFAULT_SCHEMA).validate(test_scope) - def test_exclude_resources(self): - resources_to_exclude = [{'instances': [{'uuid': 'INSTANCE_1'}, + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail') + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list') + def test_exclude_resource( + self, mock_aggregate, mock_detailed_aggregate): + mock_aggregate.return_value = [mock.Mock(id=i, + name="HA_{0}".format(i)) + for i in range(2)] + mock_collection = [mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) + for i in range(2)] + mock_collection[0].name = 'HA_0' + mock_collection[1].name = 'HA_1' + mock_detailed_aggregate.side_effect = mock_collection + + resources_to_exclude = [{'host_aggregates': [{'name': 'HA_1'}, + {'id': 0}]}, + {'instances': [{'uuid': 'INSTANCE_1'}, {'uuid': 'INSTANCE_2'}]}, - {'compute_nodes': [{'name': 'Node_1'}, - {'name': 'Node_2'}]}] + {'compute_nodes': [{'name': 'Node_2'}, + {'name': 'Node_3'}]}, + {'instance_metadata': [{'optimize': True}, + {'optimize1': False}]}] instances_to_exclude = [] nodes_to_exclude = [] - default.DefaultScope([], osc=mock.Mock()).exclude_resources( + instance_metadata = [] + default.DefaultScope([], mock.Mock(), + osc=mock.Mock()).exclude_resources( resources_to_exclude, instances=instances_to_exclude, - nodes=nodes_to_exclude) - self.assertEqual(['Node_1', 'Node_2'], sorted(nodes_to_exclude)) + nodes=nodes_to_exclude, instance_metadata=instance_metadata) + + self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'], + sorted(nodes_to_exclude)) self.assertEqual(['INSTANCE_1', 'INSTANCE_2'], sorted(instances_to_exclude)) + self.assertEqual([{'optimize': True}, {'optimize1': False}], + instance_metadata) + + def test_exclude_instances_with_given_metadata(self): + cluster = self.fake_cluster.generate_scenario_1() + instance_metadata = [{'optimize': True}] + instances_to_remove = set() + default.DefaultScope( + [], mock.Mock(), + osc=mock.Mock()).exclude_instances_with_given_metadata( + instance_metadata, cluster, instances_to_remove) + self.assertEqual(sorted(['INSTANCE_' + str(i) for i in range(35)]), + sorted(instances_to_remove)) + + instance_metadata = [{'optimize': False}] + instances_to_remove = set() + default.DefaultScope( + [], mock.Mock(), + osc=mock.Mock()).exclude_instances_with_given_metadata( + instance_metadata, cluster, instances_to_remove) + self.assertEqual(set(), instances_to_remove) def test_remove_nodes_from_model(self): model = self.fake_cluster.generate_scenario_1() - default.DefaultScope([], osc=mock.Mock()).remove_nodes_from_model( + default.DefaultScope([], mock.Mock(), + osc=mock.Mock()).remove_nodes_from_model( ['Node_1', 'Node_2'], model) expected_edges = [ ('INSTANCE_0', 'Node_0'), @@ -200,7 +242,8 @@ class TestDefaultScope(base.TestCase): def test_remove_instances_from_model(self): model = self.fake_cluster.generate_scenario_1() - default.DefaultScope([], osc=mock.Mock()).remove_instances_from_model( + default.DefaultScope([], mock.Mock(), + osc=mock.Mock()).remove_instances_from_model( ['INSTANCE_1', 'INSTANCE_2'], model) expected_edges = [ ('INSTANCE_0', 'Node_0'), diff --git a/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py b/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py index 5b525ee06..9f3cf424c 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py @@ -193,8 +193,8 @@ class TestBasicConsolidation(base.TestCase): model = self.fake_cluster.generate_scenario_3_with_2_nodes() self.m_model.return_value = copy.deepcopy(model) - self.assertEqual( - model.to_string(), self.strategy.compute_model.to_string()) + self.assertTrue(model_root.ModelRoot.is_isomorphic( + model, self.strategy.compute_model)) self.assertIsNot(model, self.strategy.compute_model) def test_basic_consolidation_migration(self):