Merge "Cluster data model collector plugin documentation"

watcher/decision_engine/goal/goals.py

@@ -72,7 +72,7 @@ class Unclassified(base.Goal):
 
 
 class ServerConsolidation(base.Goal):
-    """ServerConsolidation
+    """Server Consolidation
 
     This goal is for efficient usage of compute server resources in order to
     reduce the total number of servers.
@@ -97,7 +97,7 @@ class ServerConsolidation(base.Goal):
 
 
 class ThermalOptimization(base.Goal):
-    """ThermalOptimization
+    """Thermal Optimization
 
     This goal is used to balance the temperature across different servers.
     """
@@ -121,7 +121,7 @@ class ThermalOptimization(base.Goal):
 
 
 class WorkloadBalancing(base.Goal):
-    """WorkloadBalancing
+    """Workload Balancing
 
     This goal is used to evenly distribute workloads across different servers.
     """
@@ -145,6 +145,10 @@ class WorkloadBalancing(base.Goal):
 
 
 class AirflowOptimization(base.Goal):
+    """Workload Balancing
+
+    This goal is used to optimize the air flow within a cloud infrastructure.
+    """
 
     @classmethod
     def get_name(cls):

watcher/decision_engine/model/mapping.py

@@ -30,7 +30,7 @@ class Mapping(object):
         self.lock = threading.Lock()
 
     def map(self, hypervisor, vm):
-        """Select the hypervisor where the instance are launched
+        """Select the hypervisor where the instance is launched
 
         :param hypervisor: the hypervisor
         :param vm: the virtual machine or instance

watcher/doc.py

@@ -22,7 +22,6 @@ import inspect
 from docutils import nodes
 from docutils.parsers import rst
 from docutils import statemachine
-from stevedore import extension
 
 from watcher.version import version_info
 
@@ -98,74 +97,6 @@ class WatcherTerm(BaseWatcherDirective):
         return node.children
 
 
-class DriversDoc(BaseWatcherDirective):
-    """Directive to import an RST formatted docstring into the Watcher doc
-
-    This directive imports the RST formatted docstring of every driver declared
-    within an entry point namespace provided as argument
-
-    **How to use it**
-
-    # inside your .py file
-    class DocumentedClassReferencedInEntrypoint(object):
-        '''My *.rst* docstring'''
-
-        def foo(self):
-            '''Foo docstring'''
-
-    # Inside your .rst file
-    .. drivers-doc:: entrypoint_namespace
-       :append_methods_doc: foo
-
-    This directive will then import the docstring and then interprete it.
-
-    Note that no section/sub-section can be imported via this directive as it
-    is a Sphinx restriction.
-    """
-
-    # You need to put an import path as an argument for this directive to work
-    required_arguments = 1
-    optional_arguments = 0
-    final_argument_whitespace = True
-    has_content = False
-
-    option_spec = dict(
-        # CSV formatted list of method names whose return values will be zipped
-        # together in the given order
-        append_methods_doc=lambda opts: [
-            opt.strip() for opt in opts.split(",") if opt.strip()],
-        # By default, we always start by adding the driver object docstring
-        exclude_driver_docstring=rst.directives.flag,
-    )
-
-    def run(self):
-        ext_manager = extension.ExtensionManager(namespace=self.arguments[0])
-        extensions = ext_manager.extensions
-        # Aggregates drivers based on their module name (i.e import path)
-        classes = [(ext.name, ext.plugin) for ext in extensions]
-
-        for name, cls in classes:
-            self.add_line(".. rubric:: %s" % name)
-            self.add_line("")
-
-            if "exclude_driver_docstring" not in self.options:
-                self.add_object_docstring(cls)
-                self.add_line("")
-
-            for method_name in self.options.get("append_methods_doc", []):
-                if hasattr(cls, method_name):
-                    method = getattr(cls, method_name)
-                    method_result = inspect.cleandoc(method)
-                    self.add_textblock(method_result())
-                    self.add_line("")
-
-        node = nodes.paragraph()
-        node.document = self.state.document
-        self.state.nested_parse(self.result, 0, node)
-        return node.children
-
-
 def setup(app):
-    app.add_directive('drivers-doc', DriversDoc)
     app.add_directive('watcher-term', WatcherTerm)
     return {'version': version_info.version_string()}

watcher/metrics_engine/cluster_model_collector/base.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2015 b<>com
 #
 # Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#          Vincent FRANCOISE <vincent.francoise@b-com.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,93 +20,85 @@
 
 """
 A :ref:`Cluster Data Model <cluster_data_model_definition>` is a logical
-representation of the current state and topology of the
-:ref:`Cluster <cluster_definition>`
-:ref:`Managed resources <managed_resource_definition>`.
+representation of the current state and topology of the :ref:`Cluster
+<cluster_definition>` :ref:`Managed resources <managed_resource_definition>`.
 
-It is represented as a set of
-:ref:`Managed resources <managed_resource_definition>`
-(which may be a simple tree or a flat list of key-value pairs)
-which enables Watcher :ref:`Strategies <strategy_definition>` to know the
-current relationships between the different
-:ref:`resources <managed_resource_definition>`) of the
-:ref:`Cluster <cluster_definition>` during an :ref:`Audit <audit_definition>`
-and enables the :ref:`Strategy <strategy_definition>` to request information
-such as:
+It is represented as a set of :ref:`Managed resources
+<managed_resource_definition>` (which may be a simple tree or a flat list of
+key-value pairs) which enables Watcher :ref:`Strategies <strategy_definition>`
+to know the current relationships between the different :ref:`resources
+<managed_resource_definition>`) of the :ref:`Cluster <cluster_definition>`
+during an :ref:`Audit <audit_definition>` and enables the :ref:`Strategy
+<strategy_definition>` to request information such as:
 
--  What compute nodes are in a given
-   :ref:`Availability Zone <availability_zone_definition>`
-   or a given :ref:`Host Aggregate <host_aggregates_definition>` ?
--  What :ref:`Instances <instance_definition>` are hosted on a given compute
-   node ?
--  What is the current load of a compute node ?
--  What is the current free memory of a compute node ?
--  What is the network link between two compute nodes ?
--  What is the available bandwidth on a given network link ?
--  What is the current space available on a given virtual disk of a given
-   :ref:`Instance <instance_definition>` ?
--  What is the current state of a given :ref:`Instance <instance_definition>`?
--  ...
+- What compute nodes are in a given :ref:`Availability Zone
+  <availability_zone_definition>` or a given :ref:`Host Aggregate
+  <host_aggregates_definition>`?
+- What :ref:`Instances <instance_definition>` are hosted on a given compute
+  node?
+- What is the current load of a compute node?
+- What is the current free memory of a compute node?
+- What is the network link between two compute nodes?
+- What is the available bandwidth on a given network link?
+- What is the current space available on a given virtual disk of a given
+  :ref:`Instance <instance_definition>` ?
+- What is the current state of a given :ref:`Instance <instance_definition>`?
+- ...
 
 In a word, this data model enables the :ref:`Strategy <strategy_definition>`
 to know:
 
--  the current topology of the :ref:`Cluster <cluster_definition>`
--  the current capacity for each
-   :ref:`Managed resource <managed_resource_definition>`
--  the current amount of used/free space for each
-   :ref:`Managed resource <managed_resource_definition>`
--  the current state of each
-   :ref:`Managed resources <managed_resource_definition>`
+- the current topology of the :ref:`Cluster <cluster_definition>`
+- the current capacity for each :ref:`Managed resource
+  <managed_resource_definition>`
+- the current amount of used/free space for each :ref:`Managed resource
+  <managed_resource_definition>`
+- the current state of each :ref:`Managed resources
+  <managed_resource_definition>`
 
-In the Watcher project, we aim at providing a generic and very basic
-:ref:`Cluster Data Model <cluster_data_model_definition>` for each
-:ref:`Goal <goal_definition>`, usable in the associated
-:ref:`Strategies <strategy_definition>` through some helper classes in order
-to:
+In the Watcher project, we aim at providing a some generic and basic
+:ref:`Cluster Data Model <cluster_data_model_definition>` for each :ref:`Goal
+<goal_definition>`, usable in the associated :ref:`Strategies
+<strategy_definition>` through a plugin-based mechanism that are directly
+accessible from the strategies classes in order to:
 
--  simplify the development of a new
-   :ref:`Strategy <strategy_definition>` for a given
-   :ref:`Goal <goal_definition>` when there already are some existing
-   :ref:`Strategies <strategy_definition>` associated to the same
-   :ref:`Goal <goal_definition>`
--  avoid duplicating the same code in several
-   :ref:`Strategies <strategy_definition>` associated to the same
-   :ref:`Goal <goal_definition>`
--  have a better consistency between the different
-   :ref:`Strategies <strategy_definition>` for a given
-   :ref:`Goal <goal_definition>`
--  avoid any strong coupling with any external
-   :ref:`Cluster Data Model <cluster_data_model_definition>`
-   (the proposed data model acts as a pivot data model)
+- simplify the development of a new :ref:`Strategy <strategy_definition>` for a
+  given :ref:`Goal <goal_definition>` when there already are some existing
+  :ref:`Strategies <strategy_definition>` associated to the same :ref:`Goal
+  <goal_definition>`
+- avoid duplicating the same code in several :ref:`Strategies
+  <strategy_definition>` associated to the same :ref:`Goal <goal_definition>`
+- have a better consistency between the different :ref:`Strategies
+  <strategy_definition>` for a given :ref:`Goal <goal_definition>`
+- avoid any strong coupling with any external :ref:`Cluster Data Model
+  <cluster_data_model_definition>` (the proposed data model acts as a pivot
+  data model)
 
-There may be various
-:ref:`generic and basic Cluster Data Models <cluster_data_model_definition>`
-proposed in Watcher helpers, each of them being adapted to achieving a given
-:ref:`Goal <goal_definition>`:
+There may be various :ref:`generic and basic Cluster Data Models
+<cluster_data_model_definition>` proposed in Watcher helpers, each of them
+being adapted to achieving a given :ref:`Goal <goal_definition>`:
 
--  For example, for a
-   :ref:`Goal <goal_definition>` which aims at optimizing the network
-   :ref:`resources <managed_resource_definition>` the
-   :ref:`Strategy <strategy_definition>` may need to know which
-   :ref:`resources <managed_resource_definition>` are communicating together.
--  Whereas for a :ref:`Goal <goal_definition>` which aims at optimizing thermal
-   and power conditions, the :ref:`Strategy <strategy_definition>` may need to
-   know the location of each compute node in the racks and the location of each
-   rack in the room.
+- For example, for a :ref:`Goal <goal_definition>` which aims at optimizing
+  the network :ref:`resources <managed_resource_definition>` the :ref:`Strategy
+  <strategy_definition>` may need to know which :ref:`resources
+  <managed_resource_definition>` are communicating together.
+- Whereas for a :ref:`Goal <goal_definition>` which aims at optimizing thermal
+  and power conditions, the :ref:`Strategy <strategy_definition>` may need to
+  know the location of each compute node in the racks and the location of each
+  rack in the room.
 
-Note however that a developer can use his/her own
-:ref:`Cluster Data Model <cluster_data_model_definition>` if the proposed data
-model does not fit his/her needs as long as the
-:ref:`Strategy <strategy_definition>` is able to produce a
-:ref:`Solution <solution_definition>` for the requested
-:ref:`Goal <goal_definition>`.
-For example, a developer could rely on the Nova Data Model to optimize some
-compute resources.
+Note however that a developer can use his/her own :ref:`Cluster Data Model
+<cluster_data_model_definition>` if the proposed data model does not fit
+his/her needs as long as the :ref:`Strategy <strategy_definition>` is able to
+produce a :ref:`Solution <solution_definition>` for the requested :ref:`Goal
+<goal_definition>`. For example, a developer could rely on the Nova Data Model
+to optimize some compute resources.
 
 The :ref:`Cluster Data Model <cluster_data_model_definition>` may be persisted
 in any appropriate storage system (SQL database, NoSQL database, JSON file,
-XML File, In Memory Database, ...).
+XML File, In Memory Database, ...). As of now, an in-memory model is built and
+maintained in the background in order to accelerate the execution of
+strategies.
 """
 
 import abc

watcher/opts.py

@@ -26,6 +26,7 @@ from watcher.common import utils
 from watcher.decision_engine.loading import default as decision_engine_loader
 from watcher.decision_engine import manager as decision_engine_manger
 from watcher.decision_engine.planner import manager as planner_manager
+from watcher.metrics_engine.loading import default as cdm_loader
 
 
 PLUGIN_LOADERS = (
@@ -33,6 +34,7 @@ PLUGIN_LOADERS = (
     decision_engine_loader.DefaultPlannerLoader,
     decision_engine_loader.DefaultStrategyLoader,
     applier_loader.DefaultWorkFlowEngineLoader,
+    cdm_loader.ClusterDataModelCollectorLoader,
 )