From c0306ea8f49e3209fc6acab3ce231d7bf41db268 Mon Sep 17 00:00:00 2001 From: Jean-Emile DARTOIS Date: Thu, 7 Jan 2016 14:47:44 +0100 Subject: [PATCH] Add a common generic dynamic loader for watcher In watcher, an audit generates a set of actions which aims at achieving a given goal (lower energy consumption, ...). It is possible to configure different strategies in order to achieve each goal. Each strategy is written as a Python class which produces a set of actions. Today, the set of possible actions is fixed for a given version of Watcher and enables optimization algorithms to include actions such as instance migration, changing hypervisor state, changing power state (ACPI level, ...). This patchset add a common generic dynamic loader for plugins, such as for custom Actions, Strategies, Planners, etc. Partially implements: blueprint watcher-add-actions-via-conf Change-Id: I59d031b93865fff2540e3973921e1bdafa95f88e --- .../applier/primitives/loading/__init__.py | 0 watcher/applier/primitives/loading/default.py | 29 ++++++++++ watcher/common/exception.py | 4 ++ watcher/common/loader/__init__.py | 0 watcher/common/loader/base.py | 32 +++++++++++ watcher/common/loader/default.py | 48 +++++++++++++++++ .../planner/loading/__init__.py | 0 .../planner/loading/default.py | 30 +++++++++++ .../decision_engine/strategy/loading/base.py | 49 ----------------- .../strategy/loading/default.py | 16 ++---- watcher/locale/fr/LC_MESSAGES/watcher.po | 9 +++- watcher/locale/watcher.pot | 11 ++-- watcher/tests/common/loader/FakeLoadable.py | 28 ++++++++++ watcher/tests/common/loader/__init__.py | 0 watcher/tests/common/loader/test_loader.py | 53 +++++++++++++++++++ 15 files changed, 244 insertions(+), 65 deletions(-) create mode 100644 watcher/applier/primitives/loading/__init__.py create mode 100644 watcher/applier/primitives/loading/default.py create mode 100644 watcher/common/loader/__init__.py create mode 100644 watcher/common/loader/base.py create mode 100644 watcher/common/loader/default.py create mode 100644 watcher/decision_engine/planner/loading/__init__.py create mode 100644 watcher/decision_engine/planner/loading/default.py delete mode 100644 watcher/decision_engine/strategy/loading/base.py create mode 100644 watcher/tests/common/loader/FakeLoadable.py create mode 100644 watcher/tests/common/loader/__init__.py create mode 100644 watcher/tests/common/loader/test_loader.py diff --git a/watcher/applier/primitives/loading/__init__.py b/watcher/applier/primitives/loading/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/applier/primitives/loading/default.py b/watcher/applier/primitives/loading/default.py new file mode 100644 index 000000000..0540feb76 --- /dev/null +++ b/watcher/applier/primitives/loading/default.py @@ -0,0 +1,29 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import unicode_literals + +from oslo_log import log + +from watcher.common.loader.default import DefaultLoader + +LOG = log.getLogger(__name__) + + +class DefaultActionLoader(DefaultLoader): + def __init__(self): + super(DefaultActionLoader, self).__init__(namespace='watcher_actions') diff --git a/watcher/common/exception.py b/watcher/common/exception.py index ee8c5df87..03ffa54a2 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -297,3 +297,7 @@ class VMNotFound(WatcherException): class HypervisorNotFound(WatcherException): message = _("The hypervisor could not be found") + + +class LoadingError(WatcherException): + message = _("Error loading plugin '%(name)s'") diff --git a/watcher/common/loader/__init__.py b/watcher/common/loader/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/common/loader/base.py b/watcher/common/loader/base.py new file mode 100644 index 000000000..322cb4335 --- /dev/null +++ b/watcher/common/loader/base.py @@ -0,0 +1,32 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import unicode_literals + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseLoader(object): + + @abc.abstractmethod + def list_available(self): + raise NotImplementedError() + + @abc.abstractmethod + def load(self, name): + raise NotImplementedError() diff --git a/watcher/common/loader/default.py b/watcher/common/loader/default.py new file mode 100644 index 000000000..34dac2f53 --- /dev/null +++ b/watcher/common/loader/default.py @@ -0,0 +1,48 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import unicode_literals + +from oslo_log import log +from stevedore.driver import DriverManager +from stevedore import ExtensionManager + +from watcher.common import exception +from watcher.common.loader.base import BaseLoader + +LOG = log.getLogger(__name__) + + +class DefaultLoader(BaseLoader): + def __init__(self, namespace): + super(DefaultLoader, self).__init__() + self.namespace = namespace + + def load(self, name): + try: + LOG.debug("Loading in namespace %s => %s ", self.namespace, name) + driver_manager = DriverManager(namespace=self.namespace, + name=name) + loaded = driver_manager.driver + except Exception as exc: + LOG.exception(exc) + raise exception.LoadingError(name=name) + + return loaded() + + def list_available(self): + extension_manager = ExtensionManager(namespace=self.namespace) + return {ext.name: ext.plugin for ext in extension_manager.extensions} diff --git a/watcher/decision_engine/planner/loading/__init__.py b/watcher/decision_engine/planner/loading/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/decision_engine/planner/loading/default.py b/watcher/decision_engine/planner/loading/default.py new file mode 100644 index 000000000..010d8341c --- /dev/null +++ b/watcher/decision_engine/planner/loading/default.py @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import unicode_literals + +from oslo_log import log + +from watcher.common.loader.default import DefaultLoader + +LOG = log.getLogger(__name__) + + +class DefaultPlannerLoader(DefaultLoader): + def __init__(self): + super(DefaultPlannerLoader, self).__init__( + namespace='watcher_planners') diff --git a/watcher/decision_engine/strategy/loading/base.py b/watcher/decision_engine/strategy/loading/base.py deleted file mode 100644 index dc668e9df..000000000 --- a/watcher/decision_engine/strategy/loading/base.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Authors: Jean-Emile DARTOIS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc -from oslo_log import log -import six - -from watcher.decision_engine.strategy.strategies.dummy_strategy import \ - DummyStrategy - -LOG = log.getLogger(__name__) - - -@six.add_metaclass(abc.ABCMeta) -class BaseStrategyLoader(object): - default_strategy_cls = DummyStrategy - - @abc.abstractmethod - def load_available_strategies(self): - raise NotImplementedError() - - def load(self, strategy_to_load=None): - strategy_selected = None - try: - available_strategies = self.load_available_strategies() - LOG.debug("Available strategies: %s ", available_strategies) - strategy_cls = available_strategies.get( - strategy_to_load, self.default_strategy_cls - ) - strategy_selected = strategy_cls() - except Exception as exc: - LOG.exception(exc) - - return strategy_selected diff --git a/watcher/decision_engine/strategy/loading/default.py b/watcher/decision_engine/strategy/loading/default.py index df91d50d1..b1812940a 100644 --- a/watcher/decision_engine/strategy/loading/default.py +++ b/watcher/decision_engine/strategy/loading/default.py @@ -20,19 +20,13 @@ from __future__ import unicode_literals from oslo_log import log -from stevedore import ExtensionManager - -from watcher.decision_engine.strategy.loading.base import BaseStrategyLoader +from watcher.common.loader.default import DefaultLoader LOG = log.getLogger(__name__) -class DefaultStrategyLoader(BaseStrategyLoader): - - def load_available_strategies(self): - extension_manager = ExtensionManager( - namespace='watcher_strategies', - invoke_on_load=False, - ) - return {ext.name: ext.plugin for ext in extension_manager.extensions} +class DefaultStrategyLoader(DefaultLoader): + def __init__(self): + super(DefaultStrategyLoader, self).__init__( + namespace='watcher_strategies') diff --git a/watcher/locale/fr/LC_MESSAGES/watcher.po b/watcher/locale/fr/LC_MESSAGES/watcher.po index fe5261395..75a7cda29 100644 --- a/watcher/locale/fr/LC_MESSAGES/watcher.po +++ b/watcher/locale/fr/LC_MESSAGES/watcher.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: python-watcher 0.21.1.dev32\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2016-01-05 14:22+0100\n" +"POT-Creation-Date: 2016-01-12 18:12+0100\n" "PO-Revision-Date: 2015-12-11 15:42+0100\n" "Last-Translator: FULL NAME \n" "Language: fr\n" @@ -258,6 +258,11 @@ msgstr "" msgid "The hypervisor could not be found" msgstr "" +#: watcher/common/exception.py:303 +#, fuzzy, python-format +msgid "Error loading plugin '%(name)s'" +msgstr "Erreur lors du chargement du module '%(name)s'" + #: watcher/common/keystone.py:59 msgid "No Keystone service catalog loaded" msgstr "" @@ -291,7 +296,7 @@ msgstr "" msgid "'obj' argument type is not valid" msgstr "" -#: watcher/decision_engine/strategy/selection/default.py:58 +#: watcher/decision_engine/strategy/selection/default.py:59 #, python-format msgid "Incorrect mapping: could not find associated strategy for '%s'" msgstr "" diff --git a/watcher/locale/watcher.pot b/watcher/locale/watcher.pot index 856af3f9b..52cd84280 100644 --- a/watcher/locale/watcher.pot +++ b/watcher/locale/watcher.pot @@ -7,9 +7,9 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: python-watcher 0.21.1.dev79\n" +"Project-Id-Version: python-watcher 0.22.1.dev9\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2016-01-05 14:22+0100\n" +"POT-Creation-Date: 2016-01-12 18:12+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -256,6 +256,11 @@ msgstr "" msgid "The hypervisor could not be found" msgstr "" +#: watcher/common/exception.py:303 +#, python-format +msgid "Error loading plugin '%(name)s'" +msgstr "" + #: watcher/common/keystone.py:59 msgid "No Keystone service catalog loaded" msgstr "" @@ -289,7 +294,7 @@ msgstr "" msgid "'obj' argument type is not valid" msgstr "" -#: watcher/decision_engine/strategy/selection/default.py:58 +#: watcher/decision_engine/strategy/selection/default.py:59 #, python-format msgid "Incorrect mapping: could not find associated strategy for '%s'" msgstr "" diff --git a/watcher/tests/common/loader/FakeLoadable.py b/watcher/tests/common/loader/FakeLoadable.py new file mode 100644 index 000000000..34b1a07f5 --- /dev/null +++ b/watcher/tests/common/loader/FakeLoadable.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class FakeLoadable(object): + @classmethod + def namespace(cls): + return "TESTING" + + @classmethod + def get_name(cls): + return 'fake' diff --git a/watcher/tests/common/loader/__init__.py b/watcher/tests/common/loader/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/common/loader/test_loader.py b/watcher/tests/common/loader/test_loader.py new file mode 100644 index 000000000..79c45b89b --- /dev/null +++ b/watcher/tests/common/loader/test_loader.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import unicode_literals + +import mock + +from oslotest.base import BaseTestCase +from stevedore.driver import DriverManager +from stevedore.extension import Extension + +from watcher.common import exception +from watcher.common.loader.default import DefaultLoader +from watcher.tests.common.loader.FakeLoadable import FakeLoadable + + +class TestLoader(BaseTestCase): + @mock.patch("watcher.common.loader.default.DriverManager") + def test_load_driver_no_opt(self, m_driver_manager): + m_driver_manager.return_value = DriverManager.make_test_instance( + extension=Extension(name=FakeLoadable.get_name(), + entry_point="%s:%s" % ( + FakeLoadable.__module__, + FakeLoadable.__name__), + plugin=FakeLoadable, + obj=None), + namespace=FakeLoadable.namespace()) + + loader_manager = DefaultLoader(namespace='TESTING') + loaded_driver = loader_manager.load(name='fake') + + self.assertEqual(loaded_driver.get_name(), FakeLoadable.get_name()) + + @mock.patch("watcher.common.loader.default.DriverManager") + def test_load_driver_bad_plugin(self, m_driver_manager): + m_driver_manager.side_effect = Exception() + + loader_manager = DefaultLoader(namespace='TESTING') + self.assertRaises(exception.LoadingError, loader_manager.load, + name='bad_driver')