Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Add cdm-scoping"

Browse Source
This commit is contained in:
Zuul
2017-10-25 13:52:10 +00:00
committed by Gerrit Code Review
parent deefc857ba 51b3a15c90
commit 7ed45e3ef1
13 changed files with 246 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
watcher/api/controllers/v1/audit_template.py
View File

@@ -51,6 +51,8 @@ import wsme
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from oslo_log import log
from watcher._i18n import _
from watcher.api.controllers import base
from watcher.api.controllers import link
@@ -61,9 +63,11 @@ from watcher.common import context as context_utils
from watcher.common import exception
from watcher.common import policy
from watcher.common import utils as common_utils
from watcher.decision_engine.scope import default
from watcher.decision_engine.loading import default as default_loading
from watcher import objects
LOG = log.getLogger(__name__)
class AuditTemplatePostType(wtypes.Base):
_ctx = context_utils.make_context()
@@ -94,6 +98,27 @@ class AuditTemplatePostType(wtypes.Base):
scope=self.scope,
)
@staticmethod
def _build_schema():
SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "array",
"items": {
"type": "object",
"properties": AuditTemplatePostType._get_schemas(),
"additionalProperties": False
}
}
return SCHEMA
@staticmethod
def _get_schemas():
collectors = default_loading.ClusterDataModelCollectorLoader(
).list_available()
schemas = {k: c.SCHEMA for k, c
in collectors.items() if hasattr(c, "SCHEMA")}
return schemas
@staticmethod
def validate(audit_template):
available_goals = objects.Goal.list(AuditTemplatePostType._ctx)
@@ -106,23 +131,25 @@ class AuditTemplatePostType(wtypes.Base):
else:
raise exception.InvalidGoal(goal=audit_template.goal)
common_utils.Draft4Validator(
default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope)
if audit_template.scope:
common_utils.Draft4Validator(
AuditTemplatePostType._build_schema()
).validate(audit_template.scope)
include_host_aggregates = False
exclude_host_aggregates = False
for rule in audit_template.scope:
if 'host_aggregates' in rule:
include_host_aggregates = True
elif 'exclude' in rule:
for resource in rule['exclude']:
if 'host_aggregates' in resource:
exclude_host_aggregates = True
if include_host_aggregates and exclude_host_aggregates:
raise exception.Invalid(
message=_(
"host_aggregates can't be "
"included and excluded together"))
include_host_aggregates = False
exclude_host_aggregates = False
for rule in audit_template.scope[0]['compute']:
if 'host_aggregates' in rule:
include_host_aggregates = True
elif 'exclude' in rule:
for resource in rule['exclude']:
if 'host_aggregates' in resource:
exclude_host_aggregates = True
if include_host_aggregates and exclude_host_aggregates:
raise exception.Invalid(
message=_(
"host_aggregates can't be "
"included and excluded together"))
if audit_template.strategy:
available_strategies = objects.Strategy.list(

9
watcher/decision_engine/model/collector/base.py
View File

@@ -123,11 +123,18 @@ class BaseClusterDataModelCollector(loadable.LoadableSingleton):
STALE_MODEL = model_root.ModelRoot(stale=True)
def __init__(self, config, osc=None):
def __init__(self, config, osc=None, audit_scope=None):
super(BaseClusterDataModelCollector, self).__init__(config)
self.osc = osc if osc else clients.OpenStackClients()
self._cluster_data_model = None
self.lock = threading.RLock()
self._audit_scope = audit_scope
self._audit_scope_handler = None
@abc.abstractproperty
def audit_scope_handler(self):
"""Get audit scope handler"""
raise NotImplementedError()
@property
def cluster_data_model(self):

4
watcher/decision_engine/model/collector/cinder.py
View File

@@ -37,6 +37,10 @@ class CinderClusterDataModelCollector(base.BaseClusterDataModelCollector):
def __init__(self, config, osc=None):
super(CinderClusterDataModelCollector, self).__init__(config, osc)
@property
def audit_scope_handler(self):
return None
@property
def notification_endpoints(self):
"""Associated notification endpoints

7
watcher/decision_engine/model/collector/manager.py
View File

@@ -50,14 +50,17 @@ class CollectorManager(object):
return self._notification_endpoints
def get_cluster_model_collector(self, name, osc=None):
def get_cluster_model_collector(self, name, osc=None, audit_scope=None):
"""Retrieve cluster data model collector
:param name: name of the cluster data model collector plugin
:type name: str
:param osc: an OpenStackClients instance
:type osc: :py:class:`~.OpenStackClients` instance
:param audit_scope: an BaseScope instance
:type audit_scope: :py:class:`~.BaseScope` instance
:returns: cluster data model collector plugin
:rtype: :py:class:`~.BaseClusterDataModelCollector`
"""
return self.collector_loader.load(name, osc=osc)
return self.collector_loader.load(
name, osc=osc, audit_scope=audit_scope)

111
watcher/decision_engine/model/collector/nova.py
View File

@@ -21,6 +21,7 @@ from watcher.decision_engine.model.collector import base
from watcher.decision_engine.model import element
from watcher.decision_engine.model import model_root
from watcher.decision_engine.model.notification import nova
from watcher.decision_engine.scope import compute as compute_scope
LOG = log.getLogger(__name__)
@@ -32,9 +33,119 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
representation of the resources exposed by the compute service.
"""
HOST_AGGREGATES = "#/items/properties/compute/host_aggregates/"
SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "array",
"items": {
"type": "object",
"properties": {
"host_aggregates": {
"type": "array",
"items": {
"anyOf": [
{"$ref": HOST_AGGREGATES + "id"},
{"$ref": HOST_AGGREGATES + "name"},
]
}
},
"availability_zones": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"additionalProperties": False
}
},
"exclude": {
"type": "array",
"items": {
"type": "object",
"properties": {
"instances": {
"type": "array",
"items": {
"type": "object",
"properties": {
"uuid": {
"type": "string"
}
},
"additionalProperties": False
}
},
"compute_nodes": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"additionalProperties": False
}
},
"host_aggregates": {
"type": "array",
"items": {
"anyOf": [
{"$ref": HOST_AGGREGATES + "id"},
{"$ref": HOST_AGGREGATES + "name"},
]
}
},
"instance_metadata": {
"type": "array",
"items": {
"type": "object"
}
}
},
"additionalProperties": False
}
}
},
"additionalProperties": False
},
"host_aggregates": {
"id": {
"properties": {
"id": {
"oneOf": [
{"type": "integer"},
{"enum": ["*"]}
]
}
},
"additionalProperties": False
},
"name": {
"properties": {
"name": {
"type": "string"
}
},
"additionalProperties": False
}
},
"additionalProperties": False
}
def __init__(self, config, osc=None):
super(NovaClusterDataModelCollector, self).__init__(config, osc)
@property
def audit_scope_handler(self):
if not self._audit_scope_handler:
self._audit_scope_handler = compute_scope.ComputeScope(
self._audit_scope, self.config)
return self._audit_scope_handler
@property
def notification_endpoints(self):
"""Associated notification endpoints

108
watcher/decision_engine/scope/default.py → watcher/decision_engine/scope/compute.py
View File

@@ -24,113 +24,11 @@ from watcher.decision_engine.scope import base
LOG = log.getLogger(__name__)
class DefaultScope(base.BaseScope):
"""Default Audit Scope Handler"""
DEFAULT_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "array",
"items": {
"type": "object",
"properties": {
"host_aggregates": {
"type": "array",
"items": {
"anyOf": [
{"$ref": "#/host_aggregates/id"},
{"$ref": "#/host_aggregates/name"},
]
}
},
"availability_zones": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"additionalProperties": False
}
},
"exclude": {
"type": "array",
"items": {
"type": "object",
"properties": {
"instances": {
"type": "array",
"items": {
"type": "object",
"properties": {
"uuid": {
"type": "string"
}
},
"additionalProperties": False
}
},
"compute_nodes": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"additionalProperties": False
}
},
"host_aggregates": {
"type": "array",
"items": {
"anyOf": [
{"$ref": "#/host_aggregates/id"},
{"$ref": "#/host_aggregates/name"},
]
}
},
"instance_metadata": {
"type": "array",
"items": {
"type": "object"
}
}
},
"additionalProperties": False
}
}
},
"additionalProperties": False
},
"host_aggregates": {
"id": {
"properties": {
"id": {
"oneOf": [
{"type": "integer"},
{"enum": ["*"]}
]
}
},
"additionalProperties": False
},
"name": {
"properties": {
"name": {
"type": "string"
}
},
"additionalProperties": False
}
},
"additionalProperties": False
}
class ComputeScope(base.BaseScope):
"""Compute Audit Scope Handler"""
def __init__(self, scope, config, osc=None):
super(DefaultScope, self).__init__(scope, config)
super(ComputeScope, self).__init__(scope, config)
self._osc = osc
self.wrapper = nova_helper.NovaHelper(osc=self._osc)

14
watcher/decision_engine/strategy/strategies/base.py
View File

@@ -48,7 +48,6 @@ from watcher.common.loader import loadable
from watcher.common import utils
from watcher.decision_engine.loading import default as loading
from watcher.decision_engine.model.collector import manager
from watcher.decision_engine.scope import default as default_scope
from watcher.decision_engine.solution import default
from watcher.decision_engine.strategy.common import level
@@ -85,7 +84,6 @@ class BaseStrategy(loadable.Loadable):
self._storage_model = None
self._input_parameters = utils.Struct()
self._audit_scope = None
self._audit_scope_handler = None
@classmethod
@abc.abstractmethod
@@ -181,8 +179,9 @@ class BaseStrategy(loadable.Loadable):
"""
if self._compute_model is None:
collector = self.collector_manager.get_cluster_model_collector(
'compute', osc=self.osc)
self._compute_model = self.audit_scope_handler.get_scoped_model(
'compute', osc=self.osc, audit_scope=self.audit_scope)
audit_scope_handler = collector.audit_scope_handler
self._compute_model = audit_scope_handler.get_scoped_model(
collector.get_latest_cluster_data_model())
if not self._compute_model:
@@ -253,13 +252,6 @@ class BaseStrategy(loadable.Loadable):
def audit_scope(self, s):
self._audit_scope = s
@property
def audit_scope_handler(self):
if not self._audit_scope_handler:
self._audit_scope_handler = default_scope.DefaultScope(
self.audit_scope, self.config)
return self._audit_scope_handler
@property
def name(self):
return self._name

26
watcher/tests/api/v1/test_audit_templates.py
View File

@@ -512,18 +512,20 @@ class TestPost(FunctionalTestWithSetup):
self.assertEqual(test_time, return_created_at)
def test_create_audit_template_validation_with_aggregates(self):
scope = [{'host_aggregates': [{'id': '*'}]},
{'availability_zones': [{'name': 'AZ1'},
{'name': 'AZ2'}]},
{'exclude': [
{'instances': [
{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [
{'name': 'Node_1'},
{'name': 'Node_2'}]},
{'host_aggregates': [{'id': '*'}]}
]}
scope = [{'compute': [{'host_aggregates': [{'id': '*'}]},
{'availability_zones': [{'name': 'AZ1'},
{'name': 'AZ2'}]},
{'exclude': [
{'instances': [
{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [
{'name': 'Node_1'},
{'name': 'Node_2'}]},
{'host_aggregates': [{'id': '*'}]}
]}
]
}
]
audit_template_dict = post_get_test_audit_template(
goal=self.fake_goal1.uuid,

4
watcher/tests/decision_engine/cluster/test_cluster_data_model_collector.py
View File

@@ -23,6 +23,10 @@ from watcher.tests import base as test_base
class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
@property
def audit_scope_handler(self):
return None
@property
def notification_endpoints(self):
return []

6
watcher/tests/decision_engine/model/faker_cluster_and_metrics.py
View File

@@ -27,11 +27,15 @@ from watcher.decision_engine.model import model_root as modelroot
class FakerModelCollector(base.BaseClusterDataModelCollector):
def __init__(self, config=None, osc=None):
def __init__(self, config=None, osc=None, audit_scope=None):
if config is None:
config = mock.Mock()
super(FakerModelCollector, self).__init__(config)
@property
def audit_scope_handler(self):
return None
@property
def notification_endpoints(self):
return []

12
watcher/tests/decision_engine/model/faker_cluster_state.py
View File

@@ -27,11 +27,15 @@ from watcher.decision_engine.model import model_root as modelroot
class FakerModelCollector(base.BaseClusterDataModelCollector):
def __init__(self, config=None, osc=None):
def __init__(self, config=None, osc=None, audit_scope=None):
if config is None:
config = mock.Mock(period=777)
super(FakerModelCollector, self).__init__(config)
@property
def audit_scope_handler(self):
return None
@property
def notification_endpoints(self):
return []
@@ -139,11 +143,15 @@ class FakerModelCollector(base.BaseClusterDataModelCollector):
class FakerStorageModelCollector(base.BaseClusterDataModelCollector):
def __init__(self, config=None, osc=None):
def __init__(self, config=None, osc=None, audit_scope=None):
if config is None:
config = mock.Mock(period=777)
super(FakerStorageModelCollector, self).__init__(config)
@property
def audit_scope_handler(self):
return None
@property
def notification_endpoints(self):
return []

23
watcher/tests/decision_engine/scope/fake_scopes.py
View File

@@ -21,15 +21,16 @@ fake_scope_1 = [{'availability_zones': [{'name': 'AZ1'}]},
}
]
default_scope = [{'host_aggregates': [{'id': '*'}]},
{'availability_zones': [{'name': 'AZ1'},
{'name': 'AZ2'}]},
{'exclude': [
{'instances': [
{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [
{'name': 'Node_1'},
{'name': 'Node_2'}]}
]}
compute_scope = [{'compute': [{'host_aggregates': [{'id': '*'}]},
{'availability_zones': [{'name': 'AZ1'},
{'name': 'AZ2'}]},
{'exclude': [
{'instances': [
{'uuid': 'INSTANCE_1'},
{'uuid': 'INSTANCE_2'}]},
{'compute_nodes': [
{'name': 'Node_1'},
{'name': 'Node_2'}]}
]}]
}
]

44
watcher/tests/decision_engine/scope/test_default.py → watcher/tests/decision_engine/scope/test_compute.py
View File

@@ -18,18 +18,19 @@
from jsonschema import validators
import mock
from watcher.api.controllers.v1 import audit_template
from watcher.common import exception
from watcher.common import nova_helper
from watcher.decision_engine.scope import default
from watcher.decision_engine.scope import compute
from watcher.tests import base
from watcher.tests.decision_engine.model import faker_cluster_state
from watcher.tests.decision_engine.scope import fake_scopes
class TestDefaultScope(base.TestCase):
class TestComputeScope(base.TestCase):
def setUp(self):
super(TestDefaultScope, self).setUp()
super(TestComputeScope, self).setUp()
self.fake_cluster = faker_cluster_state.FakerModelCollector()
@mock.patch.object(nova_helper.NovaHelper, 'get_service_list')
@@ -40,7 +41,7 @@ class TestDefaultScope(base.TestCase):
mock.Mock(zone='AZ{0}'.format(i),
host={'Node_{0}'.format(i): {}})
for i in range(2)]
model = default.DefaultScope(audit_scope, mock.Mock(),
model = compute.ComputeScope(audit_scope, mock.Mock(),
osc=mock.Mock()).get_scoped_model(cluster)
expected_edges = [('INSTANCE_2', 'Node_1')]
self.assertEqual(sorted(expected_edges), sorted(model.edges()))
@@ -48,13 +49,13 @@ class TestDefaultScope(base.TestCase):
@mock.patch.object(nova_helper.NovaHelper, 'get_service_list')
def test_get_scoped_model_without_scope(self, mock_zone_list):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], mock.Mock(),
compute.ComputeScope([], mock.Mock(),
osc=mock.Mock()).get_scoped_model(model)
assert not mock_zone_list.called
def test_remove_instance(self):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], mock.Mock(), osc=mock.Mock()).remove_instance(
compute.ComputeScope([], mock.Mock(), osc=mock.Mock()).remove_instance(
model, model.get_instance_by_uuid('INSTANCE_2'), 'Node_1')
expected_edges = [
('INSTANCE_0', 'Node_0'),
@@ -74,7 +75,7 @@ class TestDefaultScope(base.TestCase):
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
mock_detailed_aggregate.side_effect = [
mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)]
default.DefaultScope([{'host_aggregates': [{'id': 1}, {'id': 2}]}],
compute.ComputeScope([{'host_aggregates': [{'id': 1}, {'id': 2}]}],
mock.Mock(), osc=mock.Mock())._collect_aggregates(
[{'id': 1}, {'id': 2}], allowed_nodes)
self.assertEqual(['Node_1'], allowed_nodes)
@@ -87,7 +88,7 @@ class TestDefaultScope(base.TestCase):
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
mock_detailed_aggregate.side_effect = [
mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)]
default.DefaultScope([{'host_aggregates': [{'id': '*'}]}],
compute.ComputeScope([{'host_aggregates': [{'id': '*'}]}],
mock.Mock(), osc=mock.Mock())._collect_aggregates(
[{'id': '*'}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], allowed_nodes)
@@ -96,7 +97,7 @@ class TestDefaultScope(base.TestCase):
def test_aggregates_wildcard_with_other_ids(self, mock_aggregate):
allowed_nodes = []
mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)]
scope_handler = default.DefaultScope(
scope_handler = compute.ComputeScope(
[{'host_aggregates': [{'id': '*'}, {'id': 1}]}],
mock.Mock(), osc=mock.Mock())
self.assertRaises(exception.WildcardCharacterIsUsed,
@@ -119,7 +120,7 @@ class TestDefaultScope(base.TestCase):
mock_detailed_aggregate.side_effect = mock_collection
default.DefaultScope([{'host_aggregates': [{'name': 'HA_1'},
compute.ComputeScope([{'host_aggregates': [{'name': 'HA_1'},
{'id': 0}]}],
mock.Mock(), osc=mock.Mock())._collect_aggregates(
[{'name': 'HA_1'}, {'id': 0}], allowed_nodes)
@@ -133,7 +134,7 @@ class TestDefaultScope(base.TestCase):
host={'Node_{0}'.format(2 * i): 1,
'Node_{0}'.format(2 * i + 1): 2})
for i in range(2)]
default.DefaultScope([{'availability_zones': [{'name': "AZ1"}]}],
compute.ComputeScope([{'availability_zones': [{'name': "AZ1"}]}],
mock.Mock(), osc=mock.Mock())._collect_zones(
[{'name': "AZ1"}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1'], sorted(allowed_nodes))
@@ -146,7 +147,7 @@ class TestDefaultScope(base.TestCase):
host={'Node_{0}'.format(2 * i): 1,
'Node_{0}'.format(2 * i + 1): 2})
for i in range(2)]
default.DefaultScope([{'availability_zones': [{'name': "*"}]}],
compute.ComputeScope([{'availability_zones': [{'name': "*"}]}],
mock.Mock(), osc=mock.Mock())._collect_zones(
[{'name': "*"}], allowed_nodes)
self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'],
@@ -160,7 +161,7 @@ class TestDefaultScope(base.TestCase):
host={'Node_{0}'.format(2 * i): 1,
'Node_{0}'.format(2 * i + 1): 2})
for i in range(2)]
scope_handler = default.DefaultScope(
scope_handler = compute.ComputeScope(
[{'availability_zones': [{'name': "*"}, {'name': 'AZ1'}]}],
mock.Mock(), osc=mock.Mock())
self.assertRaises(exception.WildcardCharacterIsUsed,
@@ -168,10 +169,11 @@ class TestDefaultScope(base.TestCase):
[{'name': "*"}, {'name': 'AZ1'}],
allowed_nodes)
def test_default_schema(self):
test_scope = fake_scopes.default_scope
def test_compute_schema(self):
test_scope = fake_scopes.compute_scope
validators.Draft4Validator(
default.DefaultScope.DEFAULT_SCHEMA).validate(test_scope)
audit_template.AuditTemplatePostType._build_schema()
).validate(test_scope)
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail')
@mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list')
@@ -197,7 +199,7 @@ class TestDefaultScope(base.TestCase):
instances_to_exclude = []
nodes_to_exclude = []
instance_metadata = []
default.DefaultScope([], mock.Mock(),
compute.ComputeScope([], mock.Mock(),
osc=mock.Mock()).exclude_resources(
resources_to_exclude, instances=instances_to_exclude,
nodes=nodes_to_exclude, instance_metadata=instance_metadata)
@@ -213,7 +215,7 @@ class TestDefaultScope(base.TestCase):
cluster = self.fake_cluster.generate_scenario_1()
instance_metadata = [{'optimize': True}]
instances_to_remove = set()
default.DefaultScope(
compute.ComputeScope(
[], mock.Mock(),
osc=mock.Mock()).exclude_instances_with_given_metadata(
instance_metadata, cluster, instances_to_remove)
@@ -222,7 +224,7 @@ class TestDefaultScope(base.TestCase):
instance_metadata = [{'optimize': False}]
instances_to_remove = set()
default.DefaultScope(
compute.ComputeScope(
[], mock.Mock(),
osc=mock.Mock()).exclude_instances_with_given_metadata(
instance_metadata, cluster, instances_to_remove)
@@ -230,7 +232,7 @@ class TestDefaultScope(base.TestCase):
def test_remove_nodes_from_model(self):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], mock.Mock(),
compute.ComputeScope([], mock.Mock(),
osc=mock.Mock()).remove_nodes_from_model(
['Node_1', 'Node_2'], model)
expected_edges = [
@@ -242,7 +244,7 @@ class TestDefaultScope(base.TestCase):
def test_remove_instances_from_model(self):
model = self.fake_cluster.generate_scenario_1()
default.DefaultScope([], mock.Mock(),
compute.ComputeScope([], mock.Mock(),
osc=mock.Mock()).remove_instances_from_model(
['INSTANCE_1', 'INSTANCE_2'], model)
expected_edges = [