Merged metrics_engine package into decision_engine

In this changeset, I merged the metrics_engine package into
the decision_engine one alongside the required changes to make
the tests pass.

Change-Id: Iac1cd266a854212bf4fa8b21c744b076c3b834a8
Partially-Implements: blueprint cluster-model-objects-wrapper

watcher/decision_engine/cluster/history/base.py

@@ -0,0 +1,80 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+The :ref:`Cluster History <cluster_history_definition>` contains all the
+previously collected timestamped data such as metrics and events associated
+to any :ref:`managed resource <managed_resource_definition>` of the
+:ref:`Cluster <cluster_definition>`.
+
+Just like the :ref:`Cluster Data Model <cluster_data_model_definition>`, this
+history may be used by any :ref:`Strategy <strategy_definition>` in order to
+find the most optimal :ref:`Solution <solution_definition>` during an
+:ref:`Audit <audit_definition>`.
+
+In the Watcher project, a generic
+:ref:`Cluster History <cluster_history_definition>`
+API is proposed with some helper classes in order to :
+
+-  share a common measurement (events or metrics) naming based on what is
+   defined in Ceilometer.
+   See `the full list of available measurements <http://docs.openstack.org/admin-guide-cloud/telemetry-measurements.html>`_
+-  share common meter types (Cumulative, Delta, Gauge) based on what is
+   defined in Ceilometer.
+   See `the full list of meter types <http://docs.openstack.org/admin-guide-cloud/telemetry-measurements.html>`_
+-  simplify the development of a new :ref:`Strategy <strategy_definition>`
+-  avoid duplicating the same code in several
+   :ref:`Strategies <strategy_definition>`
+-  have a better consistency between the different
+   :ref:`Strategies <strategy_definition>`
+-  avoid any strong coupling with any external metrics/events storage system
+   (the proposed API and measurement naming system acts as a pivot format)
+
+Note however that a developer can use his/her own history management system if
+the Ceilometer system does not fit his/her needs as long as the
+:ref:`Strategy <strategy_definition>` is able to produce a
+:ref:`Solution <solution_definition>` for the requested
+:ref:`Goal <goal_definition>`.
+
+The :ref:`Cluster History <cluster_history_definition>` data may be persisted
+in any appropriate storage system (InfluxDB, OpenTSDB, MongoDB,...).
+"""  # noqa
+
+import abc
+import six
+
+""" Work in progress Helper to query metrics """
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseClusterHistory(object):
+    @abc.abstractmethod
+    def statistic_aggregation(self, resource_id, meter_name, period,
+                              aggregate='avg'):
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_last_sample_values(self, resource_id, meter_name, limit=1):
+        raise NotImplementedError()
+
+    def query_sample(self, meter_name, query, limit=1):
+        raise NotImplementedError()
+
+    def statistic_list(self, meter_name, query=None, period=None):
+        raise NotImplementedError()

watcher/decision_engine/cluster/history/ceilometer.py

@@ -0,0 +1,49 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+from oslo_config import cfg
+from watcher.common import ceilometer_helper
+
+from watcher.decision_engine.cluster.history import base
+
+CONF = cfg.CONF
+
+
+class CeilometerClusterHistory(base.BaseClusterHistory):
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        super(CeilometerClusterHistory, self).__init__()
+        self.ceilometer = ceilometer_helper.CeilometerHelper(osc=osc)
+
+    def statistic_list(self, meter_name, query=None, period=None):
+        return self.ceilometer.statistic_list(meter_name, query, period)
+
+    def query_sample(self, meter_name, query, limit=1):
+        return self.ceilometer.query_sample(meter_name, query, limit)
+
+    def get_last_sample_values(self, resource_id, meter_name, limit=1):
+        return self.ceilometer.get_last_sample_values(resource_id, meter_name,
+                                                      limit)
+
+    def statistic_aggregation(self, resource_id, meter_name, period,
+                              aggregate='avg'):
+        return self.ceilometer.statistic_aggregation(resource_id, meter_name,
+                                                     period,
+                                                     aggregate)

watcher/decision_engine/loading/default.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2015 b<>com
 #
 # Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#          Vincent FRANCOISE <vincent.francoise@b-com.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,3 +40,9 @@ class DefaultPlannerLoader(default.DefaultLoader):
     def __init__(self):
         super(DefaultPlannerLoader, self).__init__(
             namespace='watcher_planners')
+
+
+class ClusterDataModelCollectorLoader(default.DefaultLoader):
+    def __init__(self):
+        super(ClusterDataModelCollectorLoader, self).__init__(
+            namespace='watcher_cluster_data_model_collectors')

watcher/decision_engine/model/collector/base.py

@@ -0,0 +1,169 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#          Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A :ref:`Cluster Data Model <cluster_data_model_definition>` is a logical
+representation of the current state and topology of the :ref:`Cluster
+<cluster_definition>` :ref:`Managed resources <managed_resource_definition>`.
+
+It is represented as a set of :ref:`Managed resources
+<managed_resource_definition>` (which may be a simple tree or a flat list of
+key-value pairs) which enables Watcher :ref:`Strategies <strategy_definition>`
+to know the current relationships between the different :ref:`resources
+<managed_resource_definition>`) of the :ref:`Cluster <cluster_definition>`
+during an :ref:`Audit <audit_definition>` and enables the :ref:`Strategy
+<strategy_definition>` to request information such as:
+
+- What compute nodes are in a given :ref:`Availability Zone
+  <availability_zone_definition>` or a given :ref:`Host Aggregate
+  <host_aggregates_definition>`?
+- What :ref:`Instances <instance_definition>` are hosted on a given compute
+  node?
+- What is the current load of a compute node?
+- What is the current free memory of a compute node?
+- What is the network link between two compute nodes?
+- What is the available bandwidth on a given network link?
+- What is the current space available on a given virtual disk of a given
+  :ref:`Instance <instance_definition>` ?
+- What is the current state of a given :ref:`Instance <instance_definition>`?
+- ...
+
+In a word, this data model enables the :ref:`Strategy <strategy_definition>`
+to know:
+
+- the current topology of the :ref:`Cluster <cluster_definition>`
+- the current capacity for each :ref:`Managed resource
+  <managed_resource_definition>`
+- the current amount of used/free space for each :ref:`Managed resource
+  <managed_resource_definition>`
+- the current state of each :ref:`Managed resources
+  <managed_resource_definition>`
+
+In the Watcher project, we aim at providing a some generic and basic
+:ref:`Cluster Data Model <cluster_data_model_definition>` for each :ref:`Goal
+<goal_definition>`, usable in the associated :ref:`Strategies
+<strategy_definition>` through a plugin-based mechanism that are directly
+accessible from the strategies classes in order to:
+
+- simplify the development of a new :ref:`Strategy <strategy_definition>` for a
+  given :ref:`Goal <goal_definition>` when there already are some existing
+  :ref:`Strategies <strategy_definition>` associated to the same :ref:`Goal
+  <goal_definition>`
+- avoid duplicating the same code in several :ref:`Strategies
+  <strategy_definition>` associated to the same :ref:`Goal <goal_definition>`
+- have a better consistency between the different :ref:`Strategies
+  <strategy_definition>` for a given :ref:`Goal <goal_definition>`
+- avoid any strong coupling with any external :ref:`Cluster Data Model
+  <cluster_data_model_definition>` (the proposed data model acts as a pivot
+  data model)
+
+There may be various :ref:`generic and basic Cluster Data Models
+<cluster_data_model_definition>` proposed in Watcher helpers, each of them
+being adapted to achieving a given :ref:`Goal <goal_definition>`:
+
+- For example, for a :ref:`Goal <goal_definition>` which aims at optimizing
+  the network :ref:`resources <managed_resource_definition>` the :ref:`Strategy
+  <strategy_definition>` may need to know which :ref:`resources
+  <managed_resource_definition>` are communicating together.
+- Whereas for a :ref:`Goal <goal_definition>` which aims at optimizing thermal
+  and power conditions, the :ref:`Strategy <strategy_definition>` may need to
+  know the location of each compute node in the racks and the location of each
+  rack in the room.
+
+Note however that a developer can use his/her own :ref:`Cluster Data Model
+<cluster_data_model_definition>` if the proposed data model does not fit
+his/her needs as long as the :ref:`Strategy <strategy_definition>` is able to
+produce a :ref:`Solution <solution_definition>` for the requested :ref:`Goal
+<goal_definition>`. For example, a developer could rely on the Nova Data Model
+to optimize some compute resources.
+
+The :ref:`Cluster Data Model <cluster_data_model_definition>` may be persisted
+in any appropriate storage system (SQL database, NoSQL database, JSON file,
+XML File, In Memory Database, ...). As of now, an in-memory model is built and
+maintained in the background in order to accelerate the execution of
+strategies.
+"""
+
+import abc
+import copy
+import threading
+
+from oslo_config import cfg
+import six
+
+from watcher.common import clients
+from watcher.common.loader import loadable
+from watcher.decision_engine.model import model_root
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseClusterDataModelCollector(loadable.LoadableSingleton):
+
+    STALE_MODEL = model_root.ModelRoot(stale=True)
+
+    def __init__(self, config, osc=None):
+        super(BaseClusterDataModelCollector, self).__init__(config)
+        self.osc = osc if osc else clients.OpenStackClients()
+        self._cluster_data_model = None
+        self.lock = threading.RLock()
+
+    @property
+    def cluster_data_model(self):
+        if self._cluster_data_model is None:
+            self.lock.acquire()
+            self._cluster_data_model = self.execute()
+            self.lock.release()
+
+        return self._cluster_data_model
+
+    @cluster_data_model.setter
+    def cluster_data_model(self, model):
+        self.lock.acquire()
+        self._cluster_data_model = model
+        self.lock.release()
+
+    def set_cluster_data_model_as_stale(self):
+        self.cluster_data_model = self.STALE_MODEL
+
+    @abc.abstractmethod
+    def execute(self):
+        """Build a cluster data model"""
+        raise NotImplementedError()
+
+    @classmethod
+    def get_config_opts(cls):
+        return [
+            cfg.IntOpt(
+                'period',
+                default=3600,
+                help='The time interval (in seconds) between each '
+                     'synchronization of the model'),
+        ]
+
+    def get_latest_cluster_data_model(self):
+        return copy.deepcopy(self.cluster_data_model)
+
+    def synchronize(self):
+        """Synchronize the cluster data model
+
+        Whenever called this synchronization will perform a drop-in replacement
+        with the existing cluster data model
+        """
+        self.cluster_data_model = self.execute()

watcher/decision_engine/model/collector/manager.py

@@ -0,0 +1,57 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#          Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from oslo_config import cfg
+
+from watcher.common import utils
+from watcher.decision_engine.loading import default
+
+
+CONF = cfg.CONF
+
+
+class CollectorManager(object):
+
+    def __init__(self):
+        self.collector_loader = default.ClusterDataModelCollectorLoader()
+        self._collectors = None
+
+    def get_collectors(self):
+        if self._collectors is None:
+            collectors = utils.Struct()
+            available_collectors = self.collector_loader.list_available()
+            for collector_name in available_collectors:
+                collector = self.collector_loader.load(collector_name)
+                collectors[collector_name] = collector
+                self._collectors = collectors
+
+        return self._collectors
+
+    def get_cluster_model_collector(self, name, osc=None):
+        """Retrieve cluster data model collector
+
+        :param name: name of the cluster data model collector plugin
+        :type name: str
+        :param osc: an OpenStackClients instance
+        :type osc: :py:class:`~.OpenStackClients` instance
+        :returns: cluster data model collector plugin
+        :rtype: :py:class:`~.BaseClusterDataModelCollector`
+        """
+        return self.collector_loader.load(name, osc=osc)

watcher/decision_engine/model/collector/nova.py

@@ -0,0 +1,94 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_log import log
+
+from watcher.common import nova_helper
+from watcher.decision_engine.model.collector import base
+from watcher.decision_engine.model import hypervisor as obj_hypervisor
+from watcher.decision_engine.model import model_root
+from watcher.decision_engine.model import resource
+from watcher.decision_engine.model import vm as obj_vm
+
+LOG = log.getLogger(__name__)
+
+
+class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
+    """nova
+
+    *Description*
+
+    This Nova cluster data model collector creates an in-memory representation
+    of the resources exposed by the compute service.
+
+    *Spec URL*
+
+    <None>
+    """
+
+    def __init__(self, config, osc=None):
+        super(NovaClusterDataModelCollector, self).__init__(config, osc)
+        self.wrapper = nova_helper.NovaHelper(osc=self.osc)
+
+    def execute(self):
+        """Build the compute cluster data model"""
+        LOG.debug("Building latest Nova cluster data model")
+
+        model = model_root.ModelRoot()
+        mem = resource.Resource(resource.ResourceType.memory)
+        num_cores = resource.Resource(resource.ResourceType.cpu_cores)
+        disk = resource.Resource(resource.ResourceType.disk)
+        disk_capacity = resource.Resource(resource.ResourceType.disk_capacity)
+        model.create_resource(mem)
+        model.create_resource(num_cores)
+        model.create_resource(disk)
+        model.create_resource(disk_capacity)
+
+        flavor_cache = {}
+        hypervisors = self.wrapper.get_hypervisors_list()
+        for h in hypervisors:
+            service = self.wrapper.nova.services.find(id=h.service['id'])
+            # create hypervisor in cluster_model_collector
+            hypervisor = obj_hypervisor.Hypervisor()
+            hypervisor.uuid = service.host
+            hypervisor.hostname = h.hypervisor_hostname
+            # set capacity
+            mem.set_capacity(hypervisor, h.memory_mb)
+            disk.set_capacity(hypervisor, h.free_disk_gb)
+            disk_capacity.set_capacity(hypervisor, h.local_gb)
+            num_cores.set_capacity(hypervisor, h.vcpus)
+            hypervisor.state = h.state
+            hypervisor.status = h.status
+            model.add_hypervisor(hypervisor)
+            vms = self.wrapper.get_vms_by_hypervisor(str(service.host))
+            for v in vms:
+                # create VM in cluster_model_collector
+                vm = obj_vm.VM()
+                vm.uuid = v.id
+                # nova/nova/compute/vm_states.py
+                vm.state = getattr(v, 'OS-EXT-STS:vm_state')
+
+                # set capacity
+                self.wrapper.get_flavor_instance(v, flavor_cache)
+                mem.set_capacity(vm, v.flavor['ram'])
+                disk.set_capacity(vm, v.flavor['disk'])
+                num_cores.set_capacity(vm, v.flavor['vcpus'])
+
+                model.get_mapping().map(hypervisor, vm)
+                model.add_vm(vm)
+        return model

watcher/decision_engine/scheduling.py

@@ -21,7 +21,8 @@ from oslo_log import log
 
 from watcher.common import exception
 from watcher.common import scheduling
-from watcher.metrics_engine.cluster_model_collector import manager
+
+from watcher.decision_engine.model.collector import manager
 
 LOG = log.getLogger(__name__)
 

watcher/decision_engine/strategy/strategies/__init__.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from watcher.decision_engine.strategy.strategies import basic_consolidation
 from watcher.decision_engine.strategy.strategies import dummy_strategy
 from watcher.decision_engine.strategy.strategies import outlet_temp_control

watcher/decision_engine/strategy/strategies/base.py

@@ -44,9 +44,9 @@ from watcher.common import exception
 from watcher.common.loader import loadable
 from watcher.common import utils
 from watcher.decision_engine.loading import default as loading
+from watcher.decision_engine.model.collector import manager
 from watcher.decision_engine.solution import default
 from watcher.decision_engine.strategy.common import level
-from watcher.metrics_engine.cluster_model_collector import manager
 
 
 @six.add_metaclass(abc.ABCMeta)

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -31,12 +31,11 @@ from oslo_log import log
 
 from watcher._i18n import _, _LE, _LI, _LW
 from watcher.common import exception
+from watcher.decision_engine.cluster.history import ceilometer as cch
 from watcher.decision_engine.model import hypervisor_state as hyper_state
 from watcher.decision_engine.model import resource
 from watcher.decision_engine.model import vm_state
 from watcher.decision_engine.strategy.strategies import base
-from watcher.metrics_engine.cluster_history import ceilometer as \
-    ceilometer_cluster_history
 
 LOG = log.getLogger(__name__)
 
@@ -123,8 +122,7 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
     @property
     def ceilometer(self):
         if self._ceilometer is None:
-            self._ceilometer = (ceilometer_cluster_history.
-                                CeilometerClusterHistory(osc=self.osc))
+            self._ceilometer = cch.CeilometerClusterHistory(osc=self.osc)
         return self._ceilometer
 
     @ceilometer.setter

watcher/decision_engine/strategy/strategies/outlet_temp_control.py

@@ -32,10 +32,10 @@ from oslo_log import log
 
 from watcher._i18n import _, _LI, _LW
 from watcher.common import exception as wexc
+from watcher.decision_engine.cluster.history import ceilometer as ceil
 from watcher.decision_engine.model import resource
 from watcher.decision_engine.model import vm_state
 from watcher.decision_engine.strategy.strategies import base
-from watcher.metrics_engine.cluster_history import ceilometer as ceil
 
 
 LOG = log.getLogger(__name__)

watcher/decision_engine/strategy/strategies/uniform_airflow.py

@@ -20,10 +20,10 @@ from oslo_log import log
 
 from watcher._i18n import _, _LE, _LI, _LW
 from watcher.common import exception as wexc
+from watcher.decision_engine.cluster.history import ceilometer as ceil
 from watcher.decision_engine.model import resource
 from watcher.decision_engine.model import vm_state
 from watcher.decision_engine.strategy.strategies import base
-from watcher.metrics_engine.cluster_history import ceilometer as ceil
 
 LOG = log.getLogger(__name__)
 

watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py

@@ -22,12 +22,12 @@ import six
 
 from watcher._i18n import _, _LE, _LI
 from watcher.common import exception
+from watcher.decision_engine.cluster.history import ceilometer \
+    as ceilometer_cluster_history
 from watcher.decision_engine.model import hypervisor_state as hyper_state
 from watcher.decision_engine.model import resource
 from watcher.decision_engine.model import vm_state
 from watcher.decision_engine.strategy.strategies import base
-from watcher.metrics_engine.cluster_history import ceilometer \
-    as ceilometer_cluster_history
 
 LOG = log.getLogger(__name__)
 

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -20,10 +20,10 @@ from oslo_log import log
 
 from watcher._i18n import _, _LE, _LI, _LW
 from watcher.common import exception as wexc
+from watcher.decision_engine.cluster.history import ceilometer as ceil
 from watcher.decision_engine.model import resource
 from watcher.decision_engine.model import vm_state
 from watcher.decision_engine.strategy.strategies import base
-from watcher.metrics_engine.cluster_history import ceilometer as ceil
 
 LOG = log.getLogger(__name__)
 

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -28,11 +28,11 @@ from oslo_log import log
 
 from watcher._i18n import _LI, _
 from watcher.common import exception
+from watcher.decision_engine.cluster.history import ceilometer as \
+    ceilometer_cluster_history
 from watcher.decision_engine.model import resource
 from watcher.decision_engine.model import vm_state
 from watcher.decision_engine.strategy.strategies import base
-from watcher.metrics_engine.cluster_history import ceilometer as \
-    ceilometer_cluster_history
 
 LOG = log.getLogger(__name__)