Add Audit Scope Handler

This patch set adds audit scope mechanism.
It also removes host_aggregate field.

Change-Id: Ia98ed180a93fc8c19599735e2b41471d322bae9a
Partially-Implements: blueprint define-the-audit-scope
This commit is contained in:
Alexander Chadin
2016-08-24 18:28:19 +03:00
parent e7a1e148ca
commit 48cc6b2718
36 changed files with 673 additions and 101 deletions

View File

@@ -81,8 +81,7 @@ class TestListAuditTemplate(FunctionalTestWithSetup):
def _assert_audit_template_fields(self, audit_template):
audit_template_fields = ['name', 'goal_uuid', 'goal_name',
'strategy_uuid', 'strategy_name',
'host_aggregate']
'strategy_uuid', 'strategy_name']
for field in audit_template_fields:
self.assertIn(field, audit_template)

View File

@@ -88,7 +88,7 @@ class TestListAudit(api_base.FunctionalTest):
def _assert_audit_fields(self, audit):
audit_fields = ['audit_type', 'deadline', 'state', 'goal_uuid',
'strategy_uuid', 'host_aggregate']
'strategy_uuid']
for field in audit_fields:
self.assertIn(field, audit)
@@ -369,6 +369,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
response = self.post_json('/audits', audit_dict)
self.assertEqual('application/json', response.content_type)
@@ -410,6 +411,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
# Make the audit template UUID some garbage value
audit_dict['audit_template_uuid'] = (
'01234567-8910-1112-1314-151617181920')
@@ -431,6 +433,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
with mock.patch.object(self.dbapi, 'create_audit',
wraps=self.dbapi.create_audit) as cn_mock:
response = self.post_json('/audits', audit_dict)
@@ -447,6 +450,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
response = self.post_json('/audits', audit_dict)
self.assertEqual('application/json', response.content_type)
@@ -462,6 +466,7 @@ class TestPost(api_base.FunctionalTest):
audit_dict = post_get_test_audit()
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['scope']
audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
audit_dict['interval'] = 1200
@@ -482,6 +487,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['state']
audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
del audit_dict['interval']
del audit_dict['scope']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual(400, response.status_int)
@@ -500,6 +506,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['state']
audit_dict['audit_type'] = objects.audit.AuditType.ONESHOT.value
audit_dict['interval'] = 1200
del audit_dict['scope']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual(400, response.status_int)
@@ -515,6 +522,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
response = self.post_json('/audits', audit_dict)
de_mock.assert_called_once_with(mock.ANY, response.json['uuid'])
@@ -523,6 +531,7 @@ class TestPost(api_base.FunctionalTest):
mock_trigger_audit.return_value = mock.ANY
audit_dict = post_get_test_audit(state=objects.audit.State.PENDING)
del audit_dict['scope']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual('application/json', response.content_type)
self.assertEqual(400, response.status_int)
@@ -536,6 +545,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual('application/json', response.content_type)
@@ -556,6 +566,7 @@ class TestPost(api_base.FunctionalTest):
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
del audit_dict['scope']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual('application/json', response.content_type)
@@ -577,7 +588,8 @@ class TestPost(api_base.FunctionalTest):
parameters={'fake1': 1, 'fake2': "hello"})
audit_dict['audit_template_uuid'] = audit_template['uuid']
del_keys = ['uuid', 'goal_id', 'strategy_id', 'state', 'interval']
del_keys = ['uuid', 'goal_id', 'strategy_id', 'state', 'interval',
'scope']
for k in del_keys:
del audit_dict[k]
@@ -738,6 +750,7 @@ class TestAuaditPolicyEnforcement(api_base.FunctionalTest):
audit_dict = post_get_test_audit(state=objects.audit.State.PENDING)
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['scope']
self._common_policy_check(
"audit:create", self.post_json, '/audits', audit_dict,
expect_errors=True)

View File

@@ -247,7 +247,6 @@ class DbAuditTemplateTestCase(base.DbTestCase):
uuid=w_utils.generate_uuid(),
name='My Audit Template 1',
description='Description of my audit template 1',
host_aggregate=5,
goal='DUMMY',
extra={'automatic': True})
audit_template2 = self._create_test_audit_template(
@@ -255,18 +254,9 @@ class DbAuditTemplateTestCase(base.DbTestCase):
uuid=w_utils.generate_uuid(),
name='My Audit Template 2',
description='Description of my audit template 2',
host_aggregate=3,
goal='DUMMY',
extra={'automatic': True})
res = self.dbapi.get_audit_template_list(self.context,
filters={'host_aggregate': 5})
self.assertEqual([audit_template1['id']], [r.id for r in res])
res = self.dbapi.get_audit_template_list(self.context,
filters={'host_aggregate': 1})
self.assertEqual([], [r.id for r in res])
res = self.dbapi.get_audit_template_list(
self.context,
filters={'goal': 'DUMMY'})

View File

@@ -28,11 +28,11 @@ def get_test_audit_template(**kwargs):
'name': kwargs.get('name', 'My Audit Template'),
'description': kwargs.get('description', 'Desc. Of My Audit Template'),
'extra': kwargs.get('extra', {'automatic': False}),
'host_aggregate': kwargs.get('host_aggregate', 1),
'version': kwargs.get('version', 'v1'),
'created_at': kwargs.get('created_at'),
'updated_at': kwargs.get('updated_at'),
'deleted_at': kwargs.get('deleted_at'),
'scope': kwargs.get('scope', []),
}
@@ -66,7 +66,7 @@ def get_test_audit(**kwargs):
'interval': kwargs.get('period', 3600),
'goal_id': kwargs.get('goal_id', 1),
'strategy_id': kwargs.get('strategy_id', None),
'host_aggregate': kwargs.get('host_aggregate', 1),
'scope': kwargs.get('scope', []),
}

View File

@@ -148,7 +148,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
audit_handler.launch_audits_periodically()
mock_add_job.assert_has_calls(calls)
audit_handler.update_audit_state(self.context, audits[1],
audit_handler.update_audit_state(audits[1],
audit_objects.State.CANCELLED)
is_inactive = audit_handler._is_audit_inactive(audits[1])
self.assertTrue(is_inactive)

View File

@@ -0,0 +1,35 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 Servionica
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
fake_scope_1 = [{'availability_zones': [{'name': 'AZ1'}]},
{'exclude':
[{'instances':
[{'uuid': 'INSTANCE_6'}]}]
}
]
default_scope = [{'host_aggregates': [{'id': '*'}]},
{'availability_zones': [{'name': 'AZ1'},
{'name': 'AZ2'}]},
{'exclude': [
{'instances': [
{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [
{'name': 'Node_1'},
{'name': 'Node_2'}]}
]}
]

View File

@@ -0,0 +1,210 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 Servionica
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from jsonschema import validators
import mock
from watcher.common import exception
from watcher.common import nova_helper
from watcher.decision_engine.scope import default
from watcher.tests import base
from watcher.tests.decision_engine.model import faker_cluster_state
from watcher.tests.decision_engine.scope import fake_scopes
class TestDefaultScope(base.TestCase):
def setUp(self):
super(TestDefaultScope, self).setUp()
self.fake_cluster = faker_cluster_state.FakerModelCollector()
@mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list')
def test_get_scoped_model_with_zones_and_instances(self, mock_zone_list):
cluster = self.fake_cluster.generate_scenario_1()
audit_scope = fake_scopes.fake_scope_1
mock_zone_list.return_value = [
mock.Mock(zoneName='AZ{0}'.format(i),
hosts={'Node_{0}'.format(i): {}})
for i in range(2)]
model = default.DefaultScope(audit_scope,
osc=mock.Mock()).get_scoped_model(cluster)
nodes = {'Node_4': set([]), 'Node_0': set([]), 'Node_3': set([]),
'Node_1': set(['INSTANCE_2']), 'Node_2': set([])}
self.assertEqual(nodes, model.get_mapping().get_mapping())
@mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list')
def test_get_scoped_model_without_scope(self, mock_zone_list):
cluster = self.fake_cluster.generate_scenario_1()
default.DefaultScope([],
osc=mock.Mock()).get_scoped_model(cluster)
assert not mock_zone_list.called
def test__remove_instance(self):
cluster = self.fake_cluster.generate_scenario_1()
default.DefaultScope([],
osc=mock.Mock())._remove_instance(cluster,
'INSTANCE_2',
'Node_1')
expected_map = {'Node_4': set(['INSTANCE_7']), 'Node_1': set([]),
'Node_0': set(['INSTANCE_0', 'INSTANCE_1']),
'Node_3': set(['INSTANCE_6']),
'Node_2': set(['INSTANCE_4', 'INSTANCE_5',
'INSTANCE_3'])}
self.assertEqual(expected_map, cluster.get_mapping().get_mapping())
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail')
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list')
def test__collect_aggregates(self, mock_aggregate,
mock_detailed_aggregate):
allowed_nodes = []
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
mock_detailed_aggregate.side_effect = [
mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)]
default.DefaultScope([{'host_aggregates': [{'id': 1}, {'id': 2}]}],
osc=mock.Mock())._collect_aggregates(
[{'id': 1}, {'id': 2}], allowed_nodes)
self.assertEqual(['Node_1'], allowed_nodes)
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail')
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list')
def test_aggregates_wildcard_is_used(self, mock_aggregate,
mock_detailed_aggregate):
allowed_nodes = []
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
mock_detailed_aggregate.side_effect = [
mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)]
default.DefaultScope([{'host_aggregates': [{'id': '*'}]}],
osc=mock.Mock())._collect_aggregates(
[{'id': '*'}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], allowed_nodes)
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list')
def test_aggregates_wildcard_with_other_ids(self, mock_aggregate):
allowed_nodes = []
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
scope_handler = default.DefaultScope(
[{'host_aggregates': [{'id': '*'}, {'id': 1}]}],
osc=mock.Mock())
self.assertRaises(exception.WildcardCharacterIsUsed,
scope_handler._collect_aggregates,
[{'id': '*'}, {'id': 1}],
allowed_nodes)
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail')
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list')
def test_aggregates_with_names_and_ids(self, mock_aggregate,
mock_detailed_aggregate):
allowed_nodes = []
mock_aggregate.return_value = [mock.Mock(id=i,
name="HA_{0}".format(i))
for i in range(2)]
mock_collection = [mock.Mock(id=i, hosts=['Node_{0}'.format(i)])
for i in range(2)]
mock_collection[0].name = 'HA_0'
mock_collection[1].name = 'HA_1'
mock_detailed_aggregate.side_effect = mock_collection
default.DefaultScope([{'host_aggregates': [{'name': 'HA_1'},
{'id': 0}]}],
osc=mock.Mock())._collect_aggregates(
[{'name': 'HA_1'}, {'id': 0}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], allowed_nodes)
@mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list')
def test__collect_zones(self, mock_zone_list):
allowed_nodes = []
mock_zone_list.return_value = [
mock.Mock(zoneName="AZ{0}".format(i+1),
hosts={'Node_{0}'.format(2*i): 1,
'Node_{0}'.format(2*i+1): 2})
for i in range(2)]
default.DefaultScope([{'availability_zones': [{'name': "AZ1"}]}],
osc=mock.Mock())._collect_zones(
[{'name': "AZ1"}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], sorted(allowed_nodes))
@mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list')
def test_zones_wildcard_is_used(self, mock_zone_list):
allowed_nodes = []
mock_zone_list.return_value = [
mock.Mock(zoneName="AZ{0}".format(i+1),
hosts={'Node_{0}'.format(2*i): 1,
'Node_{0}'.format(2*i+1): 2})
for i in range(2)]
default.DefaultScope([{'availability_zones': [{'name': "*"}]}],
osc=mock.Mock())._collect_zones(
[{'name': "*"}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'],
sorted(allowed_nodes))
@mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list')
def test_zones_wildcard_with_other_ids(self, mock_zone_list):
allowed_nodes = []
mock_zone_list.return_value = [
mock.Mock(zoneName="AZ{0}".format(i+1),
hosts={'Node_{0}'.format(2*i): 1,
'Node_{0}'.format(2*i+1): 2})
for i in range(2)]
scope_handler = default.DefaultScope(
[{'availability_zones': [{'name': "*"}, {'name': 'AZ1'}]}],
osc=mock.Mock())
self.assertRaises(exception.WildcardCharacterIsUsed,
scope_handler._collect_zones,
[{'name': "*"}, {'name': 'AZ1'}],
allowed_nodes)
def test_default_schema(self):
test_scope = fake_scopes.default_scope
validators.Draft4Validator(
default.DefaultScope.DEFAULT_SCHEMA).validate(test_scope)
def test__exclude_resources(self):
resources_to_exclude = [{'instances': [{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [{'name': 'Node_1'},
{'name': 'Node_2'}]}
]
instances_to_exclude = []
nodes_to_exclude = []
default.DefaultScope([], osc=mock.Mock())._exclude_resources(
resources_to_exclude, instances=instances_to_exclude,
nodes=nodes_to_exclude)
self.assertEqual(['Node_1', 'Node_2'], sorted(nodes_to_exclude))
self.assertEqual(['INSTANCE_1', 'INSTANCE_2'],
sorted(instances_to_exclude))
def test__remove_node_from_model(self):
cluster = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], osc=mock.Mock())._remove_node_from_model(
['Node_1', 'Node_2'], cluster)
expected_cluster = {'Node_0': set(['INSTANCE_0', 'INSTANCE_1']),
'Node_1': set([]), 'Node_2': set([]),
'Node_3': set(['INSTANCE_6']),
'Node_4': set(['INSTANCE_7'])}
self.assertEqual(expected_cluster, cluster.get_mapping().get_mapping())
def test__remove_instances_from_model(self):
cluster = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], osc=mock.Mock())._remove_instances_from_model(
['INSTANCE_1', 'INSTANCE_2'], cluster)
expected_cluster = {'Node_0': set(['INSTANCE_0']), 'Node_1': set([]),
'Node_2': set(['INSTANCE_3', 'INSTANCE_4',
'INSTANCE_5']),
'Node_3': set(['INSTANCE_6']),
'Node_4': set(['INSTANCE_7'])}
self.assertEqual(expected_cluster, cluster.get_mapping().get_mapping())

View File

@@ -17,12 +17,12 @@
# limitations under the License.
#
import collections
import copy
import mock
from watcher.applier.loading import default
from watcher.common import clients
from watcher.common import exception
from watcher.decision_engine.model.collector import nova
from watcher.decision_engine.model import model_root
from watcher.decision_engine.strategy import strategies
from watcher.tests import base
@@ -45,7 +45,8 @@ class TestBasicConsolidation(base.TestCase):
self.addCleanup(p_osc.stop)
p_model = mock.patch.object(
nova.NovaClusterDataModelCollector, "execute")
strategies.BasicConsolidation, "compute_model",
new_callable=mock.PropertyMock)
self.m_model = p_model.start()
self.addCleanup(p_model.stop)
@@ -55,6 +56,15 @@ class TestBasicConsolidation(base.TestCase):
self.m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_audit_scope = mock.patch.object(
strategies.BasicConsolidation, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_audit_scope.return_value = mock.Mock()
self.m_model.return_value = model_root.ModelRoot()
self.m_ceilometer.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)
@@ -168,7 +178,7 @@ class TestBasicConsolidation(base.TestCase):
def test_basic_consolidation_works_on_model_copy(self):
model = self.fake_cluster.generate_scenario_3_with_2_nodes()
self.m_model.return_value = model
self.m_model.return_value = copy.deepcopy(model)
self.assertEqual(
model.to_string(), self.strategy.compute_model.to_string())

View File

@@ -37,6 +37,15 @@ class TestDummyStrategy(base.TestCase):
self.m_model = p_model.start()
self.addCleanup(p_model.stop)
p_audit_scope = mock.patch.object(
strategies.DummyStrategy, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_audit_scope.return_value = mock.Mock()
self.m_model.return_value = model_root.ModelRoot()
self.strategy = strategies.DummyStrategy(config=mock.Mock())

View File

@@ -51,6 +51,15 @@ class TestOutletTempControl(base.TestCase):
self.m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_audit_scope = mock.patch.object(
strategies.OutletTempControl, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_audit_scope.return_value = mock.Mock()
self.m_model.return_value = model_root.ModelRoot()
self.m_ceilometer.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)

View File

@@ -51,6 +51,15 @@ class TestUniformAirflow(base.TestCase):
self.m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_audit_scope = mock.patch.object(
strategies.UniformAirflow, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_audit_scope.return_value = mock.Mock()
self.m_model.return_value = model_root.ModelRoot()
self.m_ceilometer.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)

View File

@@ -47,6 +47,15 @@ class TestVMWorkloadConsolidation(base.TestCase):
self.m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_audit_scope = mock.patch.object(
strategies.VMWorkloadConsolidation, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_audit_scope.return_value = mock.Mock()
# fake metrics
self.fake_metrics = faker_cluster_and_metrics.FakeCeilometerMetrics(
self.m_model.return_value)

View File

@@ -51,6 +51,15 @@ class TestWorkloadBalance(base.TestCase):
self.m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_audit_scope = mock.patch.object(
strategies.WorkloadBalance, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_audit_scope.return_value = mock.Mock()
self.m_model.return_value = model_root.ModelRoot()
self.m_ceilometer.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics_wb)

View File

@@ -57,9 +57,17 @@ class TestWorkloadStabilization(base.TestCase):
self.m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_audit_scope = mock.patch.object(
strategies.WorkloadStabilization, "audit_scope",
new_callable=mock.PropertyMock
)
self.m_audit_scope = p_audit_scope.start()
self.addCleanup(p_audit_scope.stop)
self.m_model.return_value = model_root.ModelRoot()
self.m_ceilometer.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)
self.m_audit_scope.return_value = mock.Mock()
self.strategy = strategies.WorkloadStabilization(config=mock.Mock())
self.strategy.input_parameters = utils.Struct()
self.strategy.input_parameters.update(