Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Added efficacy specification to /goals"

Browse Source
This commit is contained in:
Jenkins
2016-06-15 08:59:46 +00:00
committed by Gerrit Code Review
parent 28a2422ceb f665d83657
commit 488c6e3f4e
16 changed files with 450 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
watcher/api/controllers/v1/goal.py
View File

@@ -68,24 +68,28 @@ class Goal(base.APIBase):
display_name = wtypes.text
"""Localized name of the goal"""
efficacy_specification = wtypes.wsattr(types.jsontype, readonly=True)
"""Efficacy specification for this goal"""
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link and associated audit template links"""
def __init__(self, **kwargs):
super(Goal, self).__init__()
self.fields = []
self.fields.append('uuid')
self.fields.append('name')
self.fields.append('display_name')
setattr(self, 'uuid', kwargs.get('uuid', wtypes.Unset))
setattr(self, 'name', kwargs.get('name', wtypes.Unset))
setattr(self, 'display_name', kwargs.get('display_name', wtypes.Unset))
fields = list(objects.Goal.fields)
for k in fields:
# Skip fields we do not expose.
if not hasattr(self, k):
continue
self.fields.append(k)
setattr(self, k, kwargs.get(k, wtypes.Unset))
@staticmethod
def _convert_with_links(goal, url, expand=True):
if not expand:
goal.unset_fields_except(['uuid', 'name', 'display_name'])
goal.unset_fields_except(['uuid', 'name', 'display_name',
'efficacy_specification'])
goal.links = [link.Link.make_link('self', url,
'goals', goal.uuid),
@@ -101,9 +105,16 @@ class Goal(base.APIBase):
@classmethod
def sample(cls, expand=True):
sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
name='DUMMY',
display_name='Dummy strategy')
sample = cls(
uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
name='DUMMY',
display_name='Dummy strategy',
efficacy_specification=[
{'description': 'Dummy indicator', 'name': 'dummy',
'schema': 'Range(min=0, max=100, min_included=True, '
'max_included=True, msg=None)',
'unit': '%'}
])
return cls._convert_with_links(sample, 'http://localhost:9322', expand)

5
watcher/common/exception.py
View File

@@ -306,6 +306,11 @@ class NoAvailableStrategyForGoal(WatcherException):
msg_fmt = _("No strategy could be found to achieve the '%(goal)s' goal.")
class InvalidIndicatorValue(WatcherException):
msg_fmt = _("The indicator '%(name)s' with value '%(value)s' "
"and spec type '%(spec_type)s' is invalid.")
class NoMetricValuesForVM(WatcherException):
msg_fmt = _("No values returned by %(resource_id)s for %(metric_name)s.")

1
watcher/db/sqlalchemy/models.py
View File

@@ -138,6 +138,7 @@ class Goal(Base):
uuid = Column(String(36))
name = Column(String(63), nullable=False)
display_name = Column(String(63), nullable=False)
efficacy_specification = Column(JSONEncodedList, nullable=False)
class AuditTemplate(Base):

6
watcher/decision_engine/goal/base.py
View File

@@ -31,6 +31,7 @@ class Goal(loadable.Loadable):
super(Goal, self).__init__(config)
self.name = self.get_name()
self.display_name = self.get_display_name()
self.efficacy_specification = self.get_efficacy_specification()
@classmethod
@abc.abstractmethod
@@ -60,3 +61,8 @@ class Goal(loadable.Loadable):
:rtype: list of :class:`oslo_config.cfg.Opt` instances
"""
return []
@abc.abstractmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
raise NotImplementedError()

0
watcher/decision_engine/goal/efficacy/__init__.py Normal file
View File

75
watcher/decision_engine/goal/efficacy/base.py Normal file
View File

@@ -0,0 +1,75 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import json
import six
import voluptuous
@six.add_metaclass(abc.ABCMeta)
class EfficacySpecification(object):
def __init__(self):
self._indicators_specs = self.get_indicators_specifications()
@property
def indicators_specs(self):
return self._indicators_specs
@abc.abstractmethod
def get_indicators_specifications(self):
"""List the specifications of the indicator for this efficacy spec
:return: Tuple of indicator specifications
:rtype: Tuple of :py:class:`~.IndicatorSpecification` instances
"""
raise NotImplementedError()
@abc.abstractmethod
def get_global_efficacy_indicator(self, indicators_map):
"""Compute the global efficacy for the goal it achieves
:param indicators_map: dict-like object containing the
efficacy indicators related to this spec
:type indicators_map: :py:class:`~.IndicatorsMap` instance
:raises: NotImplementedError
:returns: :py:class:`~.Indicator` instance
"""
raise NotImplementedError()
@property
def schema(self):
"""Combined schema from the schema of the indicators"""
schema = voluptuous.Schema({}, required=True)
for indicator in self.indicators_specs:
key_constraint = (voluptuous.Required
if indicator.required else voluptuous.Optional)
schema = schema.extend(
{key_constraint(indicator.name): indicator.schema.schema})
return schema
def validate_efficacy_indicators(self, indicators_map):
return self.schema(indicators_map)
def get_indicators_specs_dicts(self):
return [indicator.to_dict()
for indicator in self.indicators_specs]
def serialize_indicators_specs(self):
return json.dumps(self.get_indicators_specs_dicts())

132
watcher/decision_engine/goal/efficacy/indicators.py Normal file
View File

@@ -0,0 +1,132 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
from oslo_log import log
import voluptuous
from watcher._i18n import _
from watcher.common import exception
LOG = log.getLogger(__name__)
@six.add_metaclass(abc.ABCMeta)
class IndicatorSpecification(object):
def __init__(self, name=None, description=None, unit=None, required=True):
self.name = name
self.description = description
self.unit = unit
self.required = required
@abc.abstractproperty
def schema(self):
"""Schema used to validate the indicator value
:return: A Voplutuous Schema
:rtype: :py:class:`.voluptuous.Schema` instance
"""
raise NotImplementedError()
@classmethod
def validate(cls, solution):
"""Validate the given solution
:raises: :py:class:`~.InvalidIndicatorValue` when the validation fails
"""
indicator = cls()
value = None
try:
value = getattr(solution, indicator.name)
indicator.schema(value)
except Exception as exc:
LOG.exception(exc)
raise exception.InvalidIndicatorValue(
name=indicator.name, value=value, spec_type=type(indicator))
def to_dict(self):
return {
"name": self.name,
"description": self.description,
"unit": self.unit,
"schema": str(self.schema.schema) if self.schema else None,
}
def __str__(self):
return str(self.to_dict())
class AverageCpuLoad(IndicatorSpecification):
def __init__(self):
super(AverageCpuLoad, self).__init__(
name="avg_cpu_percent",
description=_("Average CPU load as a percentage of the CPU time."),
unit="%",
)
@property
def schema(self):
return voluptuous.Schema(
voluptuous.Range(min=0, max=100), required=True)
class MigrationEfficacy(IndicatorSpecification):
def __init__(self):
super(MigrationEfficacy, self).__init__(
name="migration_efficacy",
description=_("Represents the percentage of released nodes out of "
"the total number of migrations."),
unit="%",
required=True
)
@property
def schema(self):
return voluptuous.Schema(
voluptuous.Range(min=0, max=100), required=True)
class ReleasedComputeNodesCount(IndicatorSpecification):
def __init__(self):
super(ReleasedComputeNodesCount, self).__init__(
name="released_compute_nodes_count",
description=_("The number of compute nodes to be released."),
unit=None,
)
@property
def schema(self):
return voluptuous.Schema(
voluptuous.Range(min=0), required=True)
class VmMigrationsCount(IndicatorSpecification):
def __init__(self):
super(VmMigrationsCount, self).__init__(
name="vm_migrations_count",
description=_("The number of migrations to be performed."),
unit=None,
)
@property
def schema(self):
return voluptuous.Schema(
voluptuous.Range(min=0), required=True)

52
watcher/decision_engine/goal/efficacy/specs.py Normal file
View File

@@ -0,0 +1,52 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from watcher._i18n import _
from watcher.decision_engine.goal.efficacy import base
from watcher.decision_engine.goal.efficacy import indicators
from watcher.decision_engine.solution import efficacy
class Unclassified(base.EfficacySpecification):
def get_indicators_specifications(self):
return ()
def get_global_efficacy_indicator(self, indicators_map):
return None
class ServerConsolidation(base.EfficacySpecification):
def get_indicators_specifications(self):
return [
indicators.ReleasedComputeNodesCount(),
indicators.VmMigrationsCount(),
]
def get_global_efficacy_indicator(self, indicators_map):
value = 0
if indicators_map.vm_migrations_count > 0:
value = (float(indicators_map.released_compute_nodes_count) /
float(indicators_map.vm_migrations_count)) * 100
return efficacy.Indicator(
name="released_nodes_ratio",
description=_("Ratio of released compute nodes divided by the "
"number of VM migrations."),
unit='%',
value=value,
)

38
watcher/decision_engine/goal/goals.py
View File

@@ -16,9 +16,14 @@
from watcher._i18n import _
from watcher.decision_engine.goal import base
from watcher.decision_engine.goal.efficacy import specs
class Dummy(base.Goal):
"""Dummy
Reserved goal that is used for testing purposes.
"""
@classmethod
def get_name(cls):
@@ -32,8 +37,21 @@ class Dummy(base.Goal):
def get_translatable_display_name(cls):
return "Dummy goal"
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return specs.Unclassified()
class Unclassified(base.Goal):
"""Unclassified
This goal is used to ease the development process of a strategy. Containing
no actual indicator specification, this goal can be used whenever a
strategy has yet to be formally associated with an existing goal. If the
goal achieve has been identified but there is no available implementation,
this Goal can also be used as a transitional stage.
"""
@classmethod
def get_name(cls):
@@ -47,6 +65,11 @@ class Unclassified(base.Goal):
def get_translatable_display_name(cls):
return "Unclassified"
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return specs.Unclassified()
class ServerConsolidation(base.Goal):
@@ -62,6 +85,11 @@ class ServerConsolidation(base.Goal):
def get_translatable_display_name(cls):
return "Server consolidation"
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return specs.ServerConsolidation()
class ThermalOptimization(base.Goal):
@@ -77,6 +105,11 @@ class ThermalOptimization(base.Goal):
def get_translatable_display_name(cls):
return "Thermal optimization"
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return specs.Unclassified()
class WorkloadBalancing(base.Goal):
@@ -91,3 +124,8 @@ class WorkloadBalancing(base.Goal):
@classmethod
def get_translatable_display_name(cls):
return "Workload balancing"
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return specs.Unclassified()

27
watcher/decision_engine/solution/efficacy.py Normal file
View File

@@ -0,0 +1,27 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from watcher.common import utils
class Indicator(utils.Struct):
def __init__(self, name, description, unit, value):
super(Indicator, self).__init__()
self.name = name
self.description = description
self.unit = unit
self.value = value

23
watcher/decision_engine/sync.py
View File

@@ -26,7 +26,7 @@ from watcher import objects
LOG = log.getLogger(__name__)
GoalMapping = collections.namedtuple(
'GoalMapping', ['name', 'display_name'])
'GoalMapping', ['name', 'display_name', 'efficacy_specification'])
StrategyMapping = collections.namedtuple(
'StrategyMapping', ['name', 'goal_name', 'display_name'])
@@ -75,7 +75,10 @@ class Syncer(object):
self._available_goals_map = {
GoalMapping(
name=g.name,
display_name=g.display_name): g
display_name=g.display_name,
efficacy_specification=tuple(
IndicatorSpec(**item)
for item in g.efficacy_specification)): g
for g in self.available_goals
}
return self._available_goals_map
@@ -127,6 +130,9 @@ class Syncer(object):
goal = objects.Goal(self.ctx)
goal.name = goal_name
goal.display_name = goal_map.display_name
goal.efficacy_specification = [
indicator._asdict()
for indicator in goal_map.efficacy_specification]
goal.create()
LOG.info(_LI("Goal %s created"), goal_name)
@@ -140,6 +146,7 @@ class Syncer(object):
def _sync_strategy(self, strategy_map):
strategy_name = strategy_map.name
strategy_display_name = strategy_map.display_name
goal_name = strategy_map.goal_name
strategy_mapping = dict()
@@ -153,7 +160,7 @@ class Syncer(object):
if stale_strategies or not matching_strategies:
strategy = objects.Strategy(self.ctx)
strategy.name = strategy_name
strategy.display_name = strategy_map.display_name
strategy.display_name = strategy_display_name
strategy.goal_id = objects.Goal.get_by_name(self.ctx, goal_name).id
strategy.create()
LOG.info(_LI("Strategy %s created"), strategy_name)
@@ -267,7 +274,11 @@ class Syncer(object):
for _, goal_cls in implemented_goals.items():
goals_map[goal_cls.get_name()] = GoalMapping(
name=goal_cls.get_name(),
display_name=goal_cls.get_translatable_display_name())
display_name=goal_cls.get_translatable_display_name(),
efficacy_specification=tuple(
IndicatorSpec(**indicator.to_dict())
for indicator in goal_cls.get_efficacy_specification()
.get_indicators_specifications()))
for _, strategy_cls in implemented_strategies.items():
strategies_map[strategy_cls.get_name()] = StrategyMapping(
@@ -289,10 +300,12 @@ class Syncer(object):
"""
goal_display_name = goal_map.display_name
goal_name = goal_map.name
goal_efficacy_spec = goal_map.efficacy_specification
stale_goals = []
for matching_goal in matching_goals:
if matching_goal.display_name == goal_display_name:
if (matching_goal.efficacy_specification == goal_efficacy_spec and
matching_goal.display_name == goal_display_name):
LOG.info(_LI("Goal %s unchanged"), goal_name)
else:
LOG.info(_LI("Goal %s modified"), goal_name)

1
watcher/objects/goal.py
View File

@@ -32,6 +32,7 @@ class Goal(base.WatcherObject):
'uuid': obj_utils.str_or_none,
'name': obj_utils.str_or_none,
'display_name': obj_utils.str_or_none,
'efficacy_specification': obj_utils.list_or_none,
}
@staticmethod

3
watcher/tests/api/v1/test_goals.py
View File

@@ -21,7 +21,8 @@ from watcher.tests.objects import utils as obj_utils
class TestListGoal(api_base.FunctionalTest):
def _assert_goal_fields(self, goal):
goal_fields = ['uuid', 'name', 'display_name']
goal_fields = ['uuid', 'name', 'display_name',
'efficacy_specification']
for field in goal_fields:
self.assertIn(field, goal)

1
watcher/tests/db/utils.py
View File

@@ -151,6 +151,7 @@ def get_test_goal(**kwargs):
'created_at': kwargs.get('created_at'),
'updated_at': kwargs.get('updated_at'),
'deleted_at': kwargs.get('deleted_at'),
'efficacy_specification': kwargs.get('efficacy_specification', []),
}

38
watcher/tests/decision_engine/fake_goals.py
View File

@@ -14,7 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import voluptuous
from watcher.decision_engine.goal import base as base_goal
from watcher.decision_engine.goal.efficacy import base as efficacy_base
from watcher.decision_engine.goal.efficacy import indicators
from watcher.decision_engine.goal.efficacy import specs
class FakeGoal(base_goal.Goal):
@@ -34,11 +39,44 @@ class FakeGoal(base_goal.Goal):
def get_translatable_display_name(cls):
return cls.DISPLAY_NAME
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return specs.Unclassified()
class DummyIndicator(indicators.IndicatorSpecification):
def __init__(self):
super(DummyIndicator, self).__init__(
name="dummy",
description="Dummy indicator",
unit="%",
)
@property
def schema(self):
return voluptuous.Schema(
voluptuous.Range(min=0, max=100), required=True)
class DummySpec1(efficacy_base.EfficacySpecification):
def get_indicators_specifications(self):
return [DummyIndicator()]
def get_global_efficacy_indicator(self, indicators_map):
return None
class FakeDummy1(FakeGoal):
NAME = "dummy_1"
DISPLAY_NAME = "Dummy 1"
@classmethod
def get_efficacy_specification(cls):
"""The efficacy spec for the current goal"""
return DummySpec1()
class FakeDummy2(FakeGoal):
NAME = "dummy_2"

35
watcher/tests/decision_engine/test_sync.py
View File

@@ -49,6 +49,11 @@ class TestSyncer(base.DbTestCase):
fake_goals.FakeDummy2.get_name(): fake_goals.FakeDummy2,
})
self.goal1_spec = fake_goals.FakeDummy1(
config=mock.Mock()).get_efficacy_specification()
self.goal2_spec = fake_goals.FakeDummy2(
config=mock.Mock()).get_efficacy_specification()
p_goals_load = mock.patch.object(
default.DefaultGoalLoader, 'load',
side_effect=lambda goal: self.m_available_goals()[goal]())
@@ -111,7 +116,9 @@ class TestSyncer(base.DbTestCase):
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="dummy_1", display_name="Dummy 1",)
name="dummy_1", display_name="Dummy 1",
efficacy_specification=(
self.goal1_spec.serialize_indicators_specs()))
]
m_s_list.return_value = []
@@ -141,7 +148,9 @@ class TestSyncer(base.DbTestCase):
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="dummy_1", display_name="Dummy 1",)
name="dummy_1", display_name="Dummy 1",
efficacy_specification=(
self.goal1_spec.serialize_indicators_specs()))
]
m_s_list.return_value = [
objects.Strategy(self.ctx, id=1, name="strategy_1",
@@ -174,6 +183,7 @@ class TestSyncer(base.DbTestCase):
m_g_list.return_value = [objects.Goal(
self.ctx, id=1, uuid=utils.generate_uuid(),
name="dummy_2", display_name="original",
efficacy_specification=self.goal2_spec.serialize_indicators_specs()
)]
m_s_list.return_value = []
self.syncer.sync()
@@ -202,7 +212,9 @@ class TestSyncer(base.DbTestCase):
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="dummy_1", display_name="Dummy 1",)
name="dummy_1", display_name="Dummy 1",
efficacy_specification=(
self.goal1_spec.serialize_indicators_specs()))
]
m_s_list.return_value = [
objects.Strategy(self.ctx, id=1, name="strategy_1",
@@ -227,11 +239,14 @@ class TestSyncer(base.DbTestCase):
# Should stay unmodified after sync()
goal1 = objects.Goal(
self.ctx, id=1, uuid=utils.generate_uuid(),
name="dummy_1", display_name="Dummy 1",)
name="dummy_1", display_name="Dummy 1",
efficacy_specification=(
self.goal1_spec.serialize_indicators_specs()))
# Should be modified by the sync()
goal2 = objects.Goal(
self.ctx, id=2, uuid=utils.generate_uuid(),
name="dummy_2", display_name="Original",
efficacy_specification=self.goal2_spec.serialize_indicators_specs()
)
goal1.create()
goal2.create()
@@ -322,6 +337,16 @@ class TestSyncer(base.DbTestCase):
if a_s.uuid not in [b_s.uuid for b_s in before_strategies]
}
dummy_1_spec = [
{'description': 'Dummy indicator', 'name': 'dummy',
'schema': 'Range(min=0, max=100, min_included=True, '
'max_included=True, msg=None)',
'unit': '%'}]
dummy_2_spec = []
self.assertEqual(
[dummy_1_spec, dummy_2_spec],
[g.efficacy_specification for g in after_goals])
self.assertEqual(1, len(created_goals))
self.assertEqual(3, len(created_strategies))
@@ -374,11 +399,13 @@ class TestSyncer(base.DbTestCase):
goal1 = objects.Goal(
self.ctx, id=1, uuid=utils.generate_uuid(),
name="dummy_1", display_name="Dummy 1",
efficacy_specification=self.goal1_spec.serialize_indicators_specs()
)
# To be removed by the sync()
goal2 = objects.Goal(
self.ctx, id=2, uuid=utils.generate_uuid(),
name="dummy_2", display_name="Dummy 2",
efficacy_specification=self.goal2_spec.serialize_indicators_specs()
)
goal1.create()
goal2.create()