From f8a80938ef54a260f08b7fd75a50988e40456be6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Fran=C3=A7oise?= Date: Mon, 23 Nov 2015 17:35:41 +0100 Subject: [PATCH] Made Decision Engine extensible via stevedore The objective for this is to give the ability to extend the default set of placement algorithms (i.e. strategies) currently available in Watcher using Stevedore. Now, you can add your new strategy as an entry point under the '[watcher_strategies]' section. Change-Id: I4aecf629015e41b0389d07e47220333e50bbbe1a --- .../decision_engine/api/strategy/strategy.py | 8 +-- .../framework/strategy/strategy_loader.py | 54 +++++++------------ .../strategies/basic_consolidation.py | 11 ++-- .../strategies/dummy_strategy.py | 11 +++- watcher/opts.py | 2 - ...nager_impl.py => test_strategy_context.py} | 2 +- .../strategy/test_strategy_loader.py | 6 +-- .../decision_engine/test_strategy_loader.py | 49 +++++++++++++++++ 8 files changed, 93 insertions(+), 50 deletions(-) rename watcher/tests/decision_engine/framework/strategy/{test_strategy_manager_impl.py => test_strategy_context.py} (95%) create mode 100644 watcher/tests/decision_engine/test_strategy_loader.py diff --git a/watcher/decision_engine/api/strategy/strategy.py b/watcher/decision_engine/api/strategy/strategy.py index 3faa1c383..410db6ec4 100644 --- a/watcher/decision_engine/api/strategy/strategy.py +++ b/watcher/decision_engine/api/strategy/strategy.py @@ -26,7 +26,7 @@ LOG = log.getLogger(__name__) @six.add_metaclass(abc.ABCMeta) -class Strategy(object): +class BaseStrategy(object): def __init__(self, name=None, description=None): self._name = name self.description = description @@ -40,8 +40,10 @@ class Strategy(object): def execute(self, model): """Execute a strategy - :param model: - :return: + :param model: The name of the strategy to execute (loaded dynamically) + :type model: str + :return: A computed solution (via a placement algorithm) + :rtype: :class:`watcher.decision_engine.api.strategy.solution.Solution` """ @property diff --git a/watcher/decision_engine/framework/strategy/strategy_loader.py b/watcher/decision_engine/framework/strategy/strategy_loader.py index e20f56f04..5e5064dc5 100644 --- a/watcher/decision_engine/framework/strategy/strategy_loader.py +++ b/watcher/decision_engine/framework/strategy/strategy_loader.py @@ -16,53 +16,37 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from oslo_config import cfg + +from __future__ import unicode_literals + from oslo_log import log -from stevedore import driver +from stevedore import ExtensionManager from watcher.decision_engine.strategies.basic_consolidation import \ BasicConsolidation LOG = log.getLogger(__name__) -CONF = cfg.CONF - -strategies = { - 'basic': 'watcher.decision_engine.strategies.' - 'basic_consolidation::BasicConsolidation' -} -WATCHER_STRATEGY_OPTS = [ - cfg.DictOpt('strategies', - default=strategies, - help='Strategies used for the optimization ') -] -strategies_opt_group = cfg.OptGroup( - name='watcher_strategies', - title='Defines strategies available for the optimization') -CONF.register_group(strategies_opt_group) -CONF.register_opts(WATCHER_STRATEGY_OPTS, strategies_opt_group) class StrategyLoader(object): - def __init__(self): - '''Stevedor loader + default_strategy_cls = BasicConsolidation - :return: - ''' - - self.strategies = { - None: BasicConsolidation("basic", "Basic offline consolidation"), - "basic": BasicConsolidation( - "basic", - "Basic offline consolidation") - } - - def load_driver(self, algo): - _algo = driver.DriverManager( + def load_strategies(self): + extension_manager = ExtensionManager( namespace='watcher_strategies', - name=algo, invoke_on_load=True, ) - return _algo + return {ext.name: ext.plugin for ext in extension_manager.extensions} def load(self, model): - return self.strategies[model] + strategy = None + try: + available_strategies = self.load_strategies() + strategy_cls = available_strategies.get( + model, self.default_strategy_cls + ) + strategy = strategy_cls() + except Exception as exc: + LOG.exception(exc) + + return strategy diff --git a/watcher/decision_engine/strategies/basic_consolidation.py b/watcher/decision_engine/strategies/basic_consolidation.py index 8817fa2bc..aa95feb77 100644 --- a/watcher/decision_engine/strategies/basic_consolidation.py +++ b/watcher/decision_engine/strategies/basic_consolidation.py @@ -20,7 +20,7 @@ from oslo_log import log from watcher.common.exception import ClusterEmpty from watcher.common.exception import ClusteStateNotDefined -from watcher.decision_engine.api.strategy.strategy import Strategy +from watcher.decision_engine.api.strategy.strategy import BaseStrategy from watcher.decision_engine.api.strategy.strategy import StrategyLevel from watcher.decision_engine.framework.meta_actions.hypervisor_state import \ @@ -42,8 +42,12 @@ from watcher.metrics_engine.cluster_history.ceilometer import \ LOG = log.getLogger(__name__) -class BasicConsolidation(Strategy): - def __init__(self, name=None, description=None): +class BasicConsolidation(BaseStrategy): + + DEFAULT_NAME = "basic" + DEFAULT_DESCRIPTION = "Basic offline consolidation" + + def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION): """Basic offline Consolidation using live migration The basic consolidation algorithm has several limitations. @@ -217,6 +221,7 @@ class BasicConsolidation(Strategy): def calculate_weight(self, model, element, total_cores_used, total_disk_used, total_memory_used): """Calculate weight of every + :param model: :param element: :param total_cores_used: diff --git a/watcher/decision_engine/strategies/dummy_strategy.py b/watcher/decision_engine/strategies/dummy_strategy.py index 085d7b95e..b693d9d9d 100644 --- a/watcher/decision_engine/strategies/dummy_strategy.py +++ b/watcher/decision_engine/strategies/dummy_strategy.py @@ -18,13 +18,20 @@ # from oslo_log import log -from watcher.decision_engine.api.strategy.strategy import Strategy +from watcher.decision_engine.api.strategy.strategy import BaseStrategy from watcher.decision_engine.framework.meta_actions.nop import Nop LOG = log.getLogger(__name__) -class DummyStrategy(Strategy): +class DummyStrategy(BaseStrategy): + + DEFAULT_NAME = "dummy" + DEFAULT_DESCRIPTION = "Dummy Strategy" + + def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION): + super(DummyStrategy, self).__init__(name, description) + def execute(self, model): n = Nop() self.solution.add_change_request(n) diff --git a/watcher/opts.py b/watcher/opts.py index 75bc62eac..885e997c9 100644 --- a/watcher/opts.py +++ b/watcher/opts.py @@ -23,7 +23,6 @@ from watcher.applier.framework import manager_applier import watcher.common.messaging.messaging_core from watcher.decision_engine.framework import manager -from watcher.decision_engine.framework.strategy import strategy_loader from watcher.decision_engine.framework.strategy import strategy_selector @@ -38,7 +37,6 @@ def list_opts(): ('api', watcher.api.app.API_SERVICE_OPTS), ('watcher_messaging', watcher.common.messaging.messaging_core.WATCHER_MESSAGING_OPTS), - ('watcher_strategies', strategy_loader.WATCHER_STRATEGY_OPTS), ('watcher_goals', strategy_selector.WATCHER_GOALS_OPTS), ('watcher_decision_engine', manager.WATCHER_DECISION_ENGINE_OPTS), diff --git a/watcher/tests/decision_engine/framework/strategy/test_strategy_manager_impl.py b/watcher/tests/decision_engine/framework/strategy/test_strategy_context.py similarity index 95% rename from watcher/tests/decision_engine/framework/strategy/test_strategy_manager_impl.py rename to watcher/tests/decision_engine/framework/strategy/test_strategy_context.py index 34534899d..a9b66d651 100644 --- a/watcher/tests/decision_engine/framework/strategy/test_strategy_manager_impl.py +++ b/watcher/tests/decision_engine/framework/strategy/test_strategy_context.py @@ -24,7 +24,7 @@ class FakeStrategy(object): self.name = "BALANCE_LOAD" -class TestStrategyContextImpl(base.BaseTestCase): +class TestStrategyContext(base.BaseTestCase): def test_add_remove_strategy(self): strategy = FakeStrategy() strategy_context = StrategyContext() diff --git a/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py b/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py index 159bd3e61..eb63c6992 100644 --- a/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py +++ b/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py @@ -13,6 +13,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from watcher.decision_engine.api.strategy.strategy import BaseStrategy from watcher.decision_engine.framework.strategy.strategy_loader import \ StrategyLoader from watcher.tests import base @@ -26,6 +27,7 @@ class TestStrategySelector(base.BaseTestCase): selected_strategy = self.strategy_loader.load(None) self.assertIsNotNone(selected_strategy, 'The default strategy be must not none') + self.assertIsInstance(selected_strategy, BaseStrategy) def test_load_strategy_is_basic(self): exptected_strategy = 'basic' @@ -34,7 +36,3 @@ class TestStrategySelector(base.BaseTestCase): selected_strategy.name, exptected_strategy, 'The default strategy should be basic') - - def test_load_driver(self): - algo = self.strategy_loader.load_driver("basic") - self.assertEqual(algo._names[0], "basic") diff --git a/watcher/tests/decision_engine/test_strategy_loader.py b/watcher/tests/decision_engine/test_strategy_loader.py new file mode 100644 index 000000000..673ab791b --- /dev/null +++ b/watcher/tests/decision_engine/test_strategy_loader.py @@ -0,0 +1,49 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import unicode_literals + +from mock import patch +from stevedore.extension import Extension +from stevedore.extension import ExtensionManager +from watcher.decision_engine.framework.strategy.strategy_loader import \ + StrategyLoader +from watcher.decision_engine.strategies.dummy_strategy import DummyStrategy +from watcher.tests import base + + +class TestLoader(base.BaseTestCase): + + @patch("watcher.decision_engine.framework.strategy." + "strategy_loader.ExtensionManager") + def test_strategy_loader(self, m_extension_manager): + dummy_strategy_name = "dummy" + m_extension_manager.return_value = ExtensionManager.make_test_instance( + extensions=[Extension( + name=dummy_strategy_name, + entry_point="%s:%s" % (DummyStrategy.__module__, + DummyStrategy.__name__), + plugin=DummyStrategy, + obj=None, + )], + namespace="watcher_strategies", + ) + # Set up the fake Stevedore extensions + strategy_loader = StrategyLoader() + loaded_strategy = strategy_loader.load("dummy") + + self.assertEqual("dummy", loaded_strategy.name) + self.assertEqual("Dummy Strategy", loaded_strategy.description)