From dd924dd9d5cbf1260553189c4bf3effe5f8da703 Mon Sep 17 00:00:00 2001 From: Tomasz Kaczynski Date: Tue, 30 Aug 2016 11:46:29 +0000 Subject: [PATCH] Add documentation for Scoring Module This change is updating the glossary with Scoring Module terms and adding an information page about implementing scoring engine plugin. Partially-Implements: blueprint scoring-module Change-Id: I411370dcc003ed837d8ce67659ecbfef3548ee11 --- .../dev/plugin/scoring-engine-plugin.rst | 210 ++++++++++++++++++ doc/source/dev/plugins.rst | 16 ++ doc/source/glossary.rst | 6 + doc/source/index.rst | 1 + watcher/api/controllers/v1/scoring_engine.py | 6 +- .../decision_engine/scoring/dummy_scorer.py | 2 +- 6 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 doc/source/dev/plugin/scoring-engine-plugin.rst diff --git a/doc/source/dev/plugin/scoring-engine-plugin.rst b/doc/source/dev/plugin/scoring-engine-plugin.rst new file mode 100644 index 000000000..728fbcfb4 --- /dev/null +++ b/doc/source/dev/plugin/scoring-engine-plugin.rst @@ -0,0 +1,210 @@ +.. + Except where otherwise noted, this document is licensed under Creative + Commons Attribution 3.0 License. You can view the license at: + + https://creativecommons.org/licenses/by/3.0/ + +.. _implement_scoring_engine_plugin: + +========================== +Build a new scoring engine +========================== + +Watcher Decision Engine has an external :ref:`scoring engine +` plugin interface which gives anyone the ability +to integrate an external scoring engine in order to make use of it in a +:ref:`strategy `. + +This section gives some guidelines on how to implement and integrate custom +scoring engines with Watcher. If you wish to create a third-party package for +your plugin, you can refer to our :ref:`documentation for third-party package +creation `. + + +Pre-requisites +============== + +Because scoring engines execute a purely mathematical tasks, they typically do +not have any additional dependencies. Additional requirements might be defined +by specific scoring engine implementations. For example, some scoring engines +might require to prepare learning data, which has to be loaded during the +scoring engine startup. Some other might require some external services to be +available (e.g. if the scoring infrastructure is running in the cloud). + + +Create a new scoring engine plugin +================================== + +In order to create a new scoring engine you have to: + +- Extend the :py:class:`~.ScoringEngine` class +- Implement its :py:meth:`~.ScoringEngine.get_name` method to return the + **unique** ID of the new scoring engine you want to create. This unique ID + should be the same as the name of :ref:`the entry point we will declare later + on `. +- Implement its :py:meth:`~.ScoringEngine.get_description` method to return the + user-friendly description of the implemented scoring engine. It might contain + information about algorithm used, learning data etc. +- Implement its :py:meth:`~.ScoringEngine.get_metainfo` method to return the + machine-friendly metadata about this scoring engine. For example, it could be + a JSON formatted text with information about the data model used, its input + and output data format, column names, etc. +- Implement its :py:meth:`~.ScoringEngine.calculate_score` method to return the + result calculated by this scoring engine. + +Here is an example showing how you can write a plugin called ``NewScorer``: + +.. code-block:: python + + # filepath: thirdparty/new.py + # import path: thirdparty.new + from watcher.decision_engine.scoring import base + + + class NewScorer(base.ScoringEngine): + + def get_name(self): + return 'new_scorer' + + def get_description(self): + return '' + + def get_metainfo(self): + return """{ + "feature_columns": [ + "column1", + "column2", + "column3"], + "result_columns": [ + "value", + "probability"] + }""" + + def calculate_score(self, features): + return '[12, 0.83]' + +As you can see in the above example, the +:py:meth:`~.ScoringEngine.calculate_score` method returns a string. Both this +class and the client (caller) should perform all the necessary serialization +or deserialization. + + +(Optional) Create a new scoring engine container plugin +======================================================= + +Optionally, it's possible to implement a container plugin, which can return a +list of scoring engines. This list can be re-evaluated multiple times during +the lifecycle of :ref:`Watcher Decision Engine +` and synchronized with :ref:`Watcher +Database ` using the ``watcher-sync`` command line +tool. + +Below is an example of a container using some scoring engine implementation +that is simply made of a client responsible for communicating with a real +scoring engine deployed as a web service on external servers: + +.. code-block:: python + + class NewScoringContainer(base.ScoringEngineContainer): + + @classmethod + def get_scoring_engine_list(self): + return [ + RemoteScoringEngine( + name='scoring_engine1', + description='Some remote Scoring Engine 1', + remote_url='http://engine1.example.com/score'), + RemoteScoringEngine( + name='scoring_engine2', + description='Some remote Scoring Engine 2', + remote_url='http://engine2.example.com/score'), + ] + + +Abstract Plugin Class +===================== + +Here below is the abstract :py:class:`~.ScoringEngine` class: + +.. autoclass:: watcher.decision_engine.scoring.base.ScoringEngine + :members: + :special-members: __init__ + :noindex: + + +Abstract Plugin Container Class +=============================== + +Here below is the abstract :py:class:`~.ScoringContainer` class: + +.. autoclass:: watcher.decision_engine.scoring.base.ScoringEngineContainer + :members: + :special-members: __init__ + :noindex: + + +.. _scoring_engine_plugin_add_entrypoint: + +Add a new entry point +===================== + +In order for the Watcher Decision Engine to load your new scoring engine, it +must be registered as a named entry point under the ``watcher_scoring_engines`` +entry point of your ``setup.py`` file. If you are using pbr_, this entry point +should be placed in your ``setup.cfg`` file. + +The name you give to your entry point has to be unique and should be the same +as the value returned by the :py:meth:`~.ScoringEngine.get_name` method of your +strategy. + +Here below is how you would proceed to register ``NewScorer`` using pbr_: + +.. code-block:: ini + + [entry_points] + watcher_scoring_engines = + new_scorer = thirdparty.new:NewScorer + + +To get a better understanding on how to implement a more advanced scoring +engine, have a look at the :py:class:`~.DummyScorer` class. This implementation +is not really using machine learning, but other than that it contains all the +pieces which the "real" implementation would have. + +In addition, for some use cases there is a need to register a list (possibly +dynamic, depending on the implementation and configuration) of scoring engines +in a single plugin, so there is no need to restart :ref:`Watcher Decision +Engine ` every time such list changes. For +these cases, an additional ``watcher_scoring_engine_containers`` entry point +can be used. + +For the example how to use scoring engine containers, please have a look at +the :py:class:`~.DummyScoringContainer` and the way it is configured in +``setup.cfg``. For new containers it could be done like this: + +.. code-block:: ini + + [entry_points] + watcher_scoring_engine_containers = + new_scoring_container = thirdparty.new:NewContainer + +.. _pbr: http://docs.openstack.org/developer/pbr/ + + +Using scoring engine plugins +============================ + +The Watcher Decision Engine service will automatically discover any installed +plugins when it is restarted. If a Python package containing a custom plugin is +installed within the same environment as Watcher, Watcher will automatically +make that plugin available for use. + +At this point, Watcher will scan and register inside the :ref:`Watcher Database +` all the scoring engines you implemented upon +restarting the :ref:`Watcher Decision Engine +`. + +In addition, ``watcher-sync`` tool can be used to trigger :ref:`Watcher +Database ` synchronization. This might be used for +"dynamic" scoring containers, which can return different scoring engines based +on some external configuration (if they support that). diff --git a/doc/source/dev/plugins.rst b/doc/source/dev/plugins.rst index 422e84431..6eeb7a1ff 100644 --- a/doc/source/dev/plugins.rst +++ b/doc/source/dev/plugins.rst @@ -21,6 +21,22 @@ Goals .. list-plugins:: watcher_goals :detailed: +.. _watcher_scoring_engines: + +Scoring Engines +=============== + +.. list-plugins:: watcher_scoring_engines + :detailed: + +.. _watcher_scoring_engine_containers: + +Scoring Engine Containers +========================= + +.. list-plugins:: watcher_scoring_engine_containers + :detailed: + .. _watcher_strategies: Strategies diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 48f5094e6..986770e88 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -280,6 +280,12 @@ specific domain. Please, read `the official OpenStack definition of a Project `_. +.. _scoring_engine_definition: + +Scoring Engine +============== + +.. watcher-term:: watcher.api.controllers.v1.scoring_engine .. _sla_definition: diff --git a/doc/source/index.rst b/doc/source/index.rst index 53677447f..885d0e2c7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -74,6 +74,7 @@ Plugins dev/plugin/base-setup dev/plugin/goal-plugin + dev/plugin/scoring-engine-plugin dev/plugin/strategy-plugin dev/plugin/cdmc-plugin dev/plugin/action-plugin diff --git a/watcher/api/controllers/v1/scoring_engine.py b/watcher/api/controllers/v1/scoring_engine.py index 960aab34e..0e13d3831 100644 --- a/watcher/api/controllers/v1/scoring_engine.py +++ b/watcher/api/controllers/v1/scoring_engine.py @@ -16,8 +16,10 @@ # limitations under the License. """ -A :ref:`Scoring Engine ` is an instance of a data -model, to which a learning data was applied. +A :ref:`Scoring Engine ` is an executable that has +a well-defined input, a well-defined output, and performs a purely mathematical +task. That is, the calculation does not depend on the environment in which it +is running - it would produce the same result anywhere. Because there might be multiple algorithms used to build a particular data model (and therefore a scoring engine), the usage of scoring engine might diff --git a/watcher/decision_engine/scoring/dummy_scorer.py b/watcher/decision_engine/scoring/dummy_scorer.py index cb0d87692..735dbac65 100644 --- a/watcher/decision_engine/scoring/dummy_scorer.py +++ b/watcher/decision_engine/scoring/dummy_scorer.py @@ -15,7 +15,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -# from oslo_log import log from oslo_serialization import jsonutils @@ -33,6 +32,7 @@ class DummyScorer(base.ScoringEngine): Typically a scoring engine would be implemented using machine learning techniques. For example, for workload classification problem the solution could consist of the following steps: + 1. Define a problem to solve: we want to detect the workload on the machine based on the collected metrics like power consumption, temperature, CPU load, memory usage, disk usage, network usage, etc.