Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add host_aggregates in exclude rule of audit scope

Browse Source 
Currently if user wants to skip some host_aggregates from audit,
it is not possible. This patch adds host_aggregates into the exclude
rule of audit scope. This patch also implements audit-tag-vm-metadata
using scopes.

TODOs:
1. Add tests
2. Remove old implementation of audit-tag-vm-metadata

Change-Id: Ie86378cb02145a660bbf446eedb29dc311fa29d7
Implements: BP audit-tag-vm-metadata
This commit is contained in:
Pradeep Kumar Singh
2017-04-17 07:46:16 +00:00
parent dbff4326e3
commit 0a899a2dc2
7 changed files with 157 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
watcher/api/controllers/v1/audit_template.py
View File

@@ -109,6 +109,21 @@ class AuditTemplatePostType(wtypes.Base):
common_utils.Draft4Validator(
default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope)
include_host_aggregates = False
exclude_host_aggregates = False
for rule in audit_template.scope:
if 'host_aggregates' in rule:
include_host_aggregates = True
elif 'exclude' in rule:
for resource in rule['exclude']:
if 'host_aggregates' in resource:
exclude_host_aggregates = True
if include_host_aggregates and exclude_host_aggregates:
raise exception.Invalid(
message=_(
"host_aggregates can't be "
"included and excluded together"))
if audit_template.strategy:
available_strategies = objects.Strategy.list(
AuditTemplatePostType._ctx)

3
watcher/decision_engine/scope/base.py
View File

@@ -29,9 +29,10 @@ class BaseScope(object):
requires Cluster Data Model which can be segregated to achieve audit scope.
"""
def __init__(self, scope):
def __init__(self, scope, config):
self.ctx = context.make_context()
self.scope = scope
self.config = config
@abc.abstractmethod
def get_scoped_model(self, cluster_model):

57
watcher/decision_engine/scope/default.py
View File

@@ -82,6 +82,23 @@ class DefaultScope(base.BaseScope):
}
}
}
},
"host_aggregates": {
"type": "array",
"items": {
"type": "object",
"properties": {
"anyOf": [
{"type": ["string", "number"]}
]
},
}
},
"instance_metadata": {
"type": "array",
"items": {
"type": "object"
}
}
},
"additionalProperties": False
@@ -92,8 +109,8 @@ class DefaultScope(base.BaseScope):
}
}
def __init__(self, scope, osc=None):
super(DefaultScope, self).__init__(scope)
def __init__(self, scope, config, osc=None):
super(DefaultScope, self).__init__(scope, config)
self._osc = osc
self.wrapper = nova_helper.NovaHelper(osc=self._osc)
@@ -110,7 +127,7 @@ class DefaultScope(base.BaseScope):
resource="host aggregates")
return False
def _collect_aggregates(self, host_aggregates, allowed_nodes):
def _collect_aggregates(self, host_aggregates, compute_nodes):
aggregate_list = self.wrapper.get_aggregate_list()
aggregate_ids = [aggregate['id'] for aggregate
in host_aggregates if 'id' in aggregate]
@@ -125,7 +142,7 @@ class DefaultScope(base.BaseScope):
if (detailed_aggregate.id in aggregate_ids or
detailed_aggregate.name in aggregate_names or
include_all_nodes):
allowed_nodes.extend(detailed_aggregate.hosts)
compute_nodes.extend(detailed_aggregate.hosts)
def _collect_zones(self, availability_zones, allowed_nodes):
zone_list = self.wrapper.get_availability_zone_list()
@@ -145,6 +162,8 @@ class DefaultScope(base.BaseScope):
def exclude_resources(self, resources, **kwargs):
instances_to_exclude = kwargs.get('instances')
nodes_to_exclude = kwargs.get('nodes')
instance_metadata = kwargs.get('instance_metadata')
for resource in resources:
if 'instances' in resource:
instances_to_exclude.extend(
@@ -154,6 +173,14 @@ class DefaultScope(base.BaseScope):
nodes_to_exclude.extend(
[host['name'] for host
in resource['compute_nodes']])
elif 'host_aggregates' in resource:
prohibited_nodes = []
self._collect_aggregates(resource['host_aggregates'],
prohibited_nodes)
nodes_to_exclude.extend(prohibited_nodes)
elif 'instance_metadata' in resource:
instance_metadata.extend(
[metadata for metadata in resource['instance_metadata']])
def remove_nodes_from_model(self, nodes_to_remove, cluster_model):
for node_uuid in nodes_to_remove:
@@ -179,6 +206,19 @@ class DefaultScope(base.BaseScope):
cluster_model.get_instance_by_uuid(instance_uuid),
node_name)
def exclude_instances_with_given_metadata(
self, instance_metadata, cluster_model, instances_to_remove):
metadata_dict = {
key: val for d in instance_metadata for key, val in d.items()}
instances = cluster_model.get_all_instances()
for uuid, instance in instances.items():
metadata = instance.metadata
common_metadata = set(metadata_dict) & set(metadata)
if common_metadata and len(common_metadata) == len(metadata_dict):
for key, value in metadata_dict.items():
if str(value).lower() == str(metadata.get(key)).lower():
instances_to_remove.add(uuid)
def get_scoped_model(self, cluster_model):
"""Leave only nodes and instances proposed in the audit scope"""
if not cluster_model:
@@ -188,6 +228,7 @@ class DefaultScope(base.BaseScope):
nodes_to_exclude = []
nodes_to_remove = set()
instances_to_exclude = []
instance_metadata = []
model_hosts = list(cluster_model.get_all_compute_nodes().keys())
if not self.scope:
@@ -203,7 +244,8 @@ class DefaultScope(base.BaseScope):
elif 'exclude' in rule:
self.exclude_resources(
rule['exclude'], instances=instances_to_exclude,
nodes=nodes_to_exclude)
nodes=nodes_to_exclude,
instance_metadata=instance_metadata)
instances_to_remove = set(instances_to_exclude)
if allowed_nodes:
@@ -211,6 +253,11 @@ class DefaultScope(base.BaseScope):
nodes_to_remove.update(nodes_to_exclude)
self.remove_nodes_from_model(nodes_to_remove, cluster_model)
if instance_metadata and self.config.check_optimize_metadata:
self.exclude_instances_with_given_metadata(
instance_metadata, cluster_model, instances_to_remove)
self.remove_instances_from_model(instances_to_remove, cluster_model)
return cluster_model

2
watcher/decision_engine/strategy/strategies/base.py
View File

@@ -235,7 +235,7 @@ class BaseStrategy(loadable.Loadable):
def audit_scope_handler(self):
if not self._audit_scope_handler:
self._audit_scope_handler = default_scope.DefaultScope(
self.audit_scope)
self.audit_scope, self.config)
return self._audit_scope_handler
@property

23
watcher/tests/api/v1/test_audit_templates.py
View File

@@ -13,6 +13,7 @@
import datetime
import itertools
import mock
from webtest.app import AppError
from oslo_config import cfg
from oslo_serialization import jsonutils
@@ -36,6 +37,7 @@ def post_get_test_audit_template(**kw):
strategy = db_utils.get_test_strategy(goal_id=goal['id'])
kw['goal'] = kw.get('goal', goal['uuid'])
kw['strategy'] = kw.get('strategy', strategy['uuid'])
kw['scope'] = kw.get('scope', [])
audit_template = api_utils.audit_template_post_data(**kw)
return audit_template
@@ -510,6 +512,27 @@ class TestPost(FunctionalTestWithSetup):
response.json['created_at']).replace(tzinfo=None)
self.assertEqual(test_time, return_created_at)
def test_create_audit_template_vlidation_with_aggregates(self):
scope = [{'host_aggregates': [{'id': '*'}]},
{'availability_zones': [{'name': 'AZ1'},
{'name': 'AZ2'}]},
{'exclude': [
{'instances': [
{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [
{'name': 'Node_1'},
{'name': 'Node_2'}]},
{'host_aggregates': [{'id': '*'}]}
]}
]
audit_template_dict = post_get_test_audit_template(
goal=self.fake_goal1.uuid,
strategy=self.fake_strategy1.uuid, scope=scope)
with self.assertRaisesRegexp(AppError,
"be included and excluded together"):
self.post_json('/audit_templates', audit_template_dict)
def test_create_audit_template_does_autogenerate_id(self):
audit_template_dict = post_get_test_audit_template(
goal=self.fake_goal1.uuid, strategy=None)

81
watcher/tests/decision_engine/scope/test_default.py
View File

@@ -40,7 +40,7 @@ class TestDefaultScope(base.TestCase):
mock.Mock(zoneName='AZ{0}'.format(i),
hosts={'Node_{0}'.format(i): {}})
for i in range(2)]
model = default.DefaultScope(audit_scope,
model = default.DefaultScope(audit_scope, mock.Mock(),
osc=mock.Mock()).get_scoped_model(cluster)
expected_edges = [('INSTANCE_2', 'Node_1')]
self.assertEqual(sorted(expected_edges), sorted(model.edges()))
@@ -48,13 +48,13 @@ class TestDefaultScope(base.TestCase):
@mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list')
def test_get_scoped_model_without_scope(self, mock_zone_list):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([],
default.DefaultScope([], mock.Mock(),
osc=mock.Mock()).get_scoped_model(model)
assert not mock_zone_list.called
def test_remove_instance(self):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], osc=mock.Mock()).remove_instance(
default.DefaultScope([], mock.Mock(), osc=mock.Mock()).remove_instance(
model, model.get_instance_by_uuid('INSTANCE_2'), 'Node_1')
expected_edges = [
('INSTANCE_0', 'Node_0'),
@@ -75,7 +75,7 @@ class TestDefaultScope(base.TestCase):
mock_detailed_aggregate.side_effect = [
mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)]
default.DefaultScope([{'host_aggregates': [{'id': 1}, {'id': 2}]}],
osc=mock.Mock())._collect_aggregates(
mock.Mock(), osc=mock.Mock())._collect_aggregates(
[{'id': 1}, {'id': 2}], allowed_nodes)
self.assertEqual(['Node_1'], allowed_nodes)
@@ -88,7 +88,7 @@ class TestDefaultScope(base.TestCase):
mock_detailed_aggregate.side_effect = [
mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)]
default.DefaultScope([{'host_aggregates': [{'id': '*'}]}],
osc=mock.Mock())._collect_aggregates(
mock.Mock(), osc=mock.Mock())._collect_aggregates(
[{'id': '*'}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], allowed_nodes)
@@ -98,7 +98,7 @@ class TestDefaultScope(base.TestCase):
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
scope_handler = default.DefaultScope(
[{'host_aggregates': [{'id': '*'}, {'id': 1}]}],
osc=mock.Mock())
mock.Mock(), osc=mock.Mock())
self.assertRaises(exception.WildcardCharacterIsUsed,
scope_handler._collect_aggregates,
[{'id': '*'}, {'id': 1}],
@@ -121,7 +121,7 @@ class TestDefaultScope(base.TestCase):
default.DefaultScope([{'host_aggregates': [{'name': 'HA_1'},
{'id': 0}]}],
osc=mock.Mock())._collect_aggregates(
mock.Mock(), osc=mock.Mock())._collect_aggregates(
[{'name': 'HA_1'}, {'id': 0}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], allowed_nodes)
@@ -134,7 +134,7 @@ class TestDefaultScope(base.TestCase):
'Node_{0}'.format(2 * i + 1): 2})
for i in range(2)]
default.DefaultScope([{'availability_zones': [{'name': "AZ1"}]}],
osc=mock.Mock())._collect_zones(
mock.Mock(), osc=mock.Mock())._collect_zones(
[{'name': "AZ1"}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], sorted(allowed_nodes))
@@ -147,7 +147,7 @@ class TestDefaultScope(base.TestCase):
'Node_{0}'.format(2 * i + 1): 2})
for i in range(2)]
default.DefaultScope([{'availability_zones': [{'name': "*"}]}],
osc=mock.Mock())._collect_zones(
mock.Mock(), osc=mock.Mock())._collect_zones(
[{'name': "*"}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'],
sorted(allowed_nodes))
@@ -162,7 +162,7 @@ class TestDefaultScope(base.TestCase):
for i in range(2)]
scope_handler = default.DefaultScope(
[{'availability_zones': [{'name': "*"}, {'name': 'AZ1'}]}],
osc=mock.Mock())
mock.Mock(), osc=mock.Mock())
self.assertRaises(exception.WildcardCharacterIsUsed,
scope_handler._collect_zones,
[{'name': "*"}, {'name': 'AZ1'}],
@@ -173,23 +173,65 @@ class TestDefaultScope(base.TestCase):
validators.Draft4Validator(
default.DefaultScope.DEFAULT_SCHEMA).validate(test_scope)
def test_exclude_resources(self):
resources_to_exclude = [{'instances': [{'uuid': 'INSTANCE_1'},
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail')
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list')
def test_exclude_resource(
self, mock_aggregate, mock_detailed_aggregate):
mock_aggregate.return_value = [mock.Mock(id=i,
name="HA_{0}".format(i))
for i in range(2)]
mock_collection = [mock.Mock(id=i, hosts=['Node_{0}'.format(i)])
for i in range(2)]
mock_collection[0].name = 'HA_0'
mock_collection[1].name = 'HA_1'
mock_detailed_aggregate.side_effect = mock_collection
resources_to_exclude = [{'host_aggregates': [{'name': 'HA_1'},
{'id': 0}]},
{'instances': [{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [{'name': 'Node_1'},
{'name': 'Node_2'}]}]
{'compute_nodes': [{'name': 'Node_2'},
{'name': 'Node_3'}]},
{'instance_metadata': [{'optimize': True},
{'optimize1': False}]}]
instances_to_exclude = []
nodes_to_exclude = []
default.DefaultScope([], osc=mock.Mock()).exclude_resources(
instance_metadata = []
default.DefaultScope([], mock.Mock(),
osc=mock.Mock()).exclude_resources(
resources_to_exclude, instances=instances_to_exclude,
nodes=nodes_to_exclude)
self.assertEqual(['Node_1', 'Node_2'], sorted(nodes_to_exclude))
nodes=nodes_to_exclude, instance_metadata=instance_metadata)
self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'],
sorted(nodes_to_exclude))
self.assertEqual(['INSTANCE_1', 'INSTANCE_2'],
sorted(instances_to_exclude))
self.assertEqual([{'optimize': True}, {'optimize1': False}],
instance_metadata)
def test_exclude_instances_with_given_metadata(self):
cluster = self.fake_cluster.generate_scenario_1()
instance_metadata = [{'optimize': True}]
instances_to_remove = set()
default.DefaultScope(
[], mock.Mock(),
osc=mock.Mock()).exclude_instances_with_given_metadata(
instance_metadata, cluster, instances_to_remove)
self.assertEqual(sorted(['INSTANCE_' + str(i) for i in range(35)]),
sorted(instances_to_remove))
instance_metadata = [{'optimize': False}]
instances_to_remove = set()
default.DefaultScope(
[], mock.Mock(),
osc=mock.Mock()).exclude_instances_with_given_metadata(
instance_metadata, cluster, instances_to_remove)
self.assertEqual(set(), instances_to_remove)
def test_remove_nodes_from_model(self):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], osc=mock.Mock()).remove_nodes_from_model(
default.DefaultScope([], mock.Mock(),
osc=mock.Mock()).remove_nodes_from_model(
['Node_1', 'Node_2'], model)
expected_edges = [
('INSTANCE_0', 'Node_0'),
@@ -200,7 +242,8 @@ class TestDefaultScope(base.TestCase):
def test_remove_instances_from_model(self):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], osc=mock.Mock()).remove_instances_from_model(
default.DefaultScope([], mock.Mock(),
osc=mock.Mock()).remove_instances_from_model(
['INSTANCE_1', 'INSTANCE_2'], model)
expected_edges = [
('INSTANCE_0', 'Node_0'),

4
watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py
View File

@@ -193,8 +193,8 @@ class TestBasicConsolidation(base.TestCase):
model = self.fake_cluster.generate_scenario_3_with_2_nodes()
self.m_model.return_value = copy.deepcopy(model)
self.assertEqual(
model.to_string(), self.strategy.compute_model.to_string())
self.assertTrue(model_root.ModelRoot.is_isomorphic(
model, self.strategy.compute_model))
self.assertIsNot(model, self.strategy.compute_model)
def test_basic_consolidation_migration(self):