Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cluster data model collector plugin documentation

Browse Source 
In this changeset, I wrote down the documentation on how to implement
a cluster data model collector plugin for Watcher.

This documentation corresponds to part 1 of the associated
specification.

Change-Id: Iac72b933df95252163033cd559d13348075a9b16
Partially-Implements: blueprint cluster-model-objects-wrapper
This commit is contained in:
Vincent Françoise
2016-06-27 15:06:06 +02:00
parent 5a2a94fbec
commit b94677c3ef
11 changed files with 253 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
watcher/decision_engine/goal/goals.py
View File

@@ -72,7 +72,7 @@ class Unclassified(base.Goal):
class ServerConsolidation(base.Goal):
"""ServerConsolidation
"""Server Consolidation
This goal is for efficient usage of compute server resources in order to
reduce the total number of servers.
@@ -97,7 +97,7 @@ class ServerConsolidation(base.Goal):
class ThermalOptimization(base.Goal):
"""ThermalOptimization
"""Thermal Optimization
This goal is used to balance the temperature across different servers.
"""
@@ -121,7 +121,7 @@ class ThermalOptimization(base.Goal):
class WorkloadBalancing(base.Goal):
"""WorkloadBalancing
"""Workload Balancing
This goal is used to evenly distribute workloads across different servers.
"""
@@ -145,6 +145,10 @@ class WorkloadBalancing(base.Goal):
class AirflowOptimization(base.Goal):
"""Workload Balancing
This goal is used to optimize the air flow within a cloud infrastructure.
"""
@classmethod
def get_name(cls):

2
watcher/decision_engine/model/mapping.py
View File

@@ -30,7 +30,7 @@ class Mapping(object):
self.lock = threading.Lock()
def map(self, hypervisor, vm):
"""Select the hypervisor where the instance are launched
"""Select the hypervisor where the instance is launched
:param hypervisor: the hypervisor
:param vm: the virtual machine or instance

69
watcher/doc.py
View File

@@ -22,7 +22,6 @@ import inspect
from docutils import nodes
from docutils.parsers import rst
from docutils import statemachine
from stevedore import extension
from watcher.version import version_info
@@ -98,74 +97,6 @@ class WatcherTerm(BaseWatcherDirective):
return node.children
class DriversDoc(BaseWatcherDirective):
"""Directive to import an RST formatted docstring into the Watcher doc
This directive imports the RST formatted docstring of every driver declared
within an entry point namespace provided as argument
**How to use it**
# inside your .py file
class DocumentedClassReferencedInEntrypoint(object):
'''My *.rst* docstring'''
def foo(self):
'''Foo docstring'''
# Inside your .rst file
.. drivers-doc:: entrypoint_namespace
:append_methods_doc: foo
This directive will then import the docstring and then interprete it.
Note that no section/sub-section can be imported via this directive as it
is a Sphinx restriction.
"""
# You need to put an import path as an argument for this directive to work
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
has_content = False
option_spec = dict(
# CSV formatted list of method names whose return values will be zipped
# together in the given order
append_methods_doc=lambda opts: [
opt.strip() for opt in opts.split(",") if opt.strip()],
# By default, we always start by adding the driver object docstring
exclude_driver_docstring=rst.directives.flag,
)
def run(self):
ext_manager = extension.ExtensionManager(namespace=self.arguments[0])
extensions = ext_manager.extensions
# Aggregates drivers based on their module name (i.e import path)
classes = [(ext.name, ext.plugin) for ext in extensions]
for name, cls in classes:
self.add_line(".. rubric:: %s" % name)
self.add_line("")
if "exclude_driver_docstring" not in self.options:
self.add_object_docstring(cls)
self.add_line("")
for method_name in self.options.get("append_methods_doc", []):
if hasattr(cls, method_name):
method = getattr(cls, method_name)
method_result = inspect.cleandoc(method)
self.add_textblock(method_result())
self.add_line("")
node = nodes.paragraph()
node.document = self.state.document
self.state.nested_parse(self.result, 0, node)
return node.children
def setup(app):
app.add_directive('drivers-doc', DriversDoc)
app.add_directive('watcher-term', WatcherTerm)
return {'version': version_info.version_string()}

139
watcher/metrics_engine/cluster_model_collector/base.py
View File

@@ -2,6 +2,7 @@
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
# Vincent FRANCOISE <vincent.francoise@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -19,93 +20,85 @@
"""
A :ref:`Cluster Data Model <cluster_data_model_definition>` is a logical
representation of the current state and topology of the
:ref:`Cluster <cluster_definition>`
:ref:`Managed resources <managed_resource_definition>`.
representation of the current state and topology of the :ref:`Cluster
<cluster_definition>` :ref:`Managed resources <managed_resource_definition>`.
It is represented as a set of
:ref:`Managed resources <managed_resource_definition>`
(which may be a simple tree or a flat list of key-value pairs)
which enables Watcher :ref:`Strategies <strategy_definition>` to know the
current relationships between the different
:ref:`resources <managed_resource_definition>`) of the
:ref:`Cluster <cluster_definition>` during an :ref:`Audit <audit_definition>`
and enables the :ref:`Strategy <strategy_definition>` to request information
such as:
It is represented as a set of :ref:`Managed resources
<managed_resource_definition>` (which may be a simple tree or a flat list of
key-value pairs) which enables Watcher :ref:`Strategies <strategy_definition>`
to know the current relationships between the different :ref:`resources
<managed_resource_definition>`) of the :ref:`Cluster <cluster_definition>`
during an :ref:`Audit <audit_definition>` and enables the :ref:`Strategy
<strategy_definition>` to request information such as:
- What compute nodes are in a given
:ref:`Availability Zone <availability_zone_definition>`
or a given :ref:`Host Aggregate <host_aggregates_definition>` ?
- What :ref:`Instances <instance_definition>` are hosted on a given compute
node ?
- What is the current load of a compute node ?
- What is the current free memory of a compute node ?
- What is the network link between two compute nodes ?
- What is the available bandwidth on a given network link ?
- What is the current space available on a given virtual disk of a given
:ref:`Instance <instance_definition>` ?
- What is the current state of a given :ref:`Instance <instance_definition>`?
- ...
- What compute nodes are in a given :ref:`Availability Zone
<availability_zone_definition>` or a given :ref:`Host Aggregate
<host_aggregates_definition>`?
- What :ref:`Instances <instance_definition>` are hosted on a given compute
node?
- What is the current load of a compute node?
- What is the current free memory of a compute node?
- What is the network link between two compute nodes?
- What is the available bandwidth on a given network link?
- What is the current space available on a given virtual disk of a given
:ref:`Instance <instance_definition>` ?
- What is the current state of a given :ref:`Instance <instance_definition>`?
- ...
In a word, this data model enables the :ref:`Strategy <strategy_definition>`
to know:
- the current topology of the :ref:`Cluster <cluster_definition>`
- the current capacity for each
:ref:`Managed resource <managed_resource_definition>`
- the current amount of used/free space for each
:ref:`Managed resource <managed_resource_definition>`
- the current state of each
:ref:`Managed resources <managed_resource_definition>`
- the current topology of the :ref:`Cluster <cluster_definition>`
- the current capacity for each :ref:`Managed resource
<managed_resource_definition>`
- the current amount of used/free space for each :ref:`Managed resource
<managed_resource_definition>`
- the current state of each :ref:`Managed resources
<managed_resource_definition>`
In the Watcher project, we aim at providing a generic and very basic
:ref:`Cluster Data Model <cluster_data_model_definition>` for each
:ref:`Goal <goal_definition>`, usable in the associated
:ref:`Strategies <strategy_definition>` through some helper classes in order
to:
In the Watcher project, we aim at providing a some generic and basic
:ref:`Cluster Data Model <cluster_data_model_definition>` for each :ref:`Goal
<goal_definition>`, usable in the associated :ref:`Strategies
<strategy_definition>` through a plugin-based mechanism that are directly
accessible from the strategies classes in order to:
- simplify the development of a new
:ref:`Strategy <strategy_definition>` for a given
:ref:`Goal <goal_definition>` when there already are some existing
:ref:`Strategies <strategy_definition>` associated to the same
:ref:`Goal <goal_definition>`
- avoid duplicating the same code in several
:ref:`Strategies <strategy_definition>` associated to the same
:ref:`Goal <goal_definition>`
- have a better consistency between the different
:ref:`Strategies <strategy_definition>` for a given
:ref:`Goal <goal_definition>`
- avoid any strong coupling with any external
:ref:`Cluster Data Model <cluster_data_model_definition>`
(the proposed data model acts as a pivot data model)
- simplify the development of a new :ref:`Strategy <strategy_definition>` for a
given :ref:`Goal <goal_definition>` when there already are some existing
:ref:`Strategies <strategy_definition>` associated to the same :ref:`Goal
<goal_definition>`
- avoid duplicating the same code in several :ref:`Strategies
<strategy_definition>` associated to the same :ref:`Goal <goal_definition>`
- have a better consistency between the different :ref:`Strategies
<strategy_definition>` for a given :ref:`Goal <goal_definition>`
- avoid any strong coupling with any external :ref:`Cluster Data Model
<cluster_data_model_definition>` (the proposed data model acts as a pivot
data model)
There may be various
:ref:`generic and basic Cluster Data Models <cluster_data_model_definition>`
proposed in Watcher helpers, each of them being adapted to achieving a given
:ref:`Goal <goal_definition>`:
There may be various :ref:`generic and basic Cluster Data Models
<cluster_data_model_definition>` proposed in Watcher helpers, each of them
being adapted to achieving a given :ref:`Goal <goal_definition>`:
- For example, for a
:ref:`Goal <goal_definition>` which aims at optimizing the network
:ref:`resources <managed_resource_definition>` the
:ref:`Strategy <strategy_definition>` may need to know which
:ref:`resources <managed_resource_definition>` are communicating together.
- Whereas for a :ref:`Goal <goal_definition>` which aims at optimizing thermal
and power conditions, the :ref:`Strategy <strategy_definition>` may need to
know the location of each compute node in the racks and the location of each
rack in the room.
- For example, for a :ref:`Goal <goal_definition>` which aims at optimizing
the network :ref:`resources <managed_resource_definition>` the :ref:`Strategy
<strategy_definition>` may need to know which :ref:`resources
<managed_resource_definition>` are communicating together.
- Whereas for a :ref:`Goal <goal_definition>` which aims at optimizing thermal
and power conditions, the :ref:`Strategy <strategy_definition>` may need to
know the location of each compute node in the racks and the location of each
rack in the room.
Note however that a developer can use his/her own
:ref:`Cluster Data Model <cluster_data_model_definition>` if the proposed data
model does not fit his/her needs as long as the
:ref:`Strategy <strategy_definition>` is able to produce a
:ref:`Solution <solution_definition>` for the requested
:ref:`Goal <goal_definition>`.
For example, a developer could rely on the Nova Data Model to optimize some
compute resources.
Note however that a developer can use his/her own :ref:`Cluster Data Model
<cluster_data_model_definition>` if the proposed data model does not fit
his/her needs as long as the :ref:`Strategy <strategy_definition>` is able to
produce a :ref:`Solution <solution_definition>` for the requested :ref:`Goal
<goal_definition>`. For example, a developer could rely on the Nova Data Model
to optimize some compute resources.
The :ref:`Cluster Data Model <cluster_data_model_definition>` may be persisted
in any appropriate storage system (SQL database, NoSQL database, JSON file,
XML File, In Memory Database, ...).
XML File, In Memory Database, ...). As of now, an in-memory model is built and
maintained in the background in order to accelerate the execution of
strategies.
"""
import abc

2
watcher/opts.py
View File

@@ -26,6 +26,7 @@ from watcher.common import utils
from watcher.decision_engine.loading import default as decision_engine_loader
from watcher.decision_engine import manager as decision_engine_manger
from watcher.decision_engine.planner import manager as planner_manager
from watcher.metrics_engine.loading import default as cdm_loader
PLUGIN_LOADERS = (
@@ -33,6 +34,7 @@ PLUGIN_LOADERS = (
decision_engine_loader.DefaultPlannerLoader,
decision_engine_loader.DefaultStrategyLoader,
applier_loader.DefaultWorkFlowEngineLoader,
cdm_loader.ClusterDataModelCollectorLoader,
)