Added DE Background Scheduler w/ model sync jobs

In this changeset, I implemented a background scheduler service
for Watcher and more particularly for the Decision Engine where
I made it create 2 types of job per cluster data model collector
plugin:

- An initial job that is asynchronously executed upon starting the
  Decision Engine
- A periodical job that gets triggered every configurable interval
  of time

Change-Id: I3f5442f81933a19565217b894bd86c186e339762
Partially-Implements: bluprint cluster-model-objects-wrapper

watcher/decision_engine/model/model_root.py

@@ -23,11 +23,18 @@ from watcher.decision_engine.model import vm
 
 
 class ModelRoot(object):
-    def __init__(self):
+
+    def __init__(self, stale=False):
         self._hypervisors = utils.Struct()
         self._vms = utils.Struct()
         self.mapping = mapping.Mapping(self)
         self.resource = utils.Struct()
+        self.stale = stale
+
+    def __nonzero__(self):
+        return not self.stale
+
+    __bool__ = __nonzero__
 
     def assert_hypervisor(self, obj):
         if not isinstance(obj, hypervisor.Hypervisor):

watcher/decision_engine/scheduling.py

@@ -0,0 +1,91 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+import eventlet
+from oslo_log import log
+
+from watcher.common import exception
+from watcher.common import scheduling
+from watcher.metrics_engine.cluster_model_collector import manager
+
+LOG = log.getLogger(__name__)
+
+
+class DecisionEngineSchedulingService(scheduling.BackgroundSchedulerService):
+
+    def __init__(self, gconfig=None, **options):
+        gconfig = None or {}
+        super(DecisionEngineSchedulingService, self).__init__(
+            gconfig, **options)
+        self.collector_manager = manager.CollectorManager()
+
+    @property
+    def collectors(self):
+        return self.collector_manager.get_collectors()
+
+    def add_sync_jobs(self):
+        for name, collector in self.collectors.items():
+            timed_task = self._wrap_collector_sync_with_timeout(
+                collector, name)
+            self.add_job(timed_task,
+                         trigger='interval',
+                         seconds=collector.config.period,
+                         next_run_time=datetime.datetime.now())
+
+    def _as_timed_sync_func(self, sync_func, name, timeout):
+        def _timed_sync():
+            with eventlet.Timeout(
+                timeout,
+                exception=exception.ClusterDataModelCollectionError(cdm=name)
+            ):
+                sync_func()
+
+        return _timed_sync
+
+    def _wrap_collector_sync_with_timeout(self, collector, name):
+        """Add an execution timeout constraint on a function"""
+        timeout = collector.config.period
+
+        def _sync():
+            try:
+                timed_sync = self._as_timed_sync_func(
+                    collector.synchronize, name, timeout)
+                timed_sync()
+            except Exception as exc:
+                LOG.exception(exc)
+                collector.set_cluster_data_model_as_stale()
+
+        return _sync
+
+    def start(self):
+        """Start service."""
+        self.add_sync_jobs()
+        super(DecisionEngineSchedulingService, self).start()
+
+    def stop(self):
+        """Stop service."""
+        self.shutdown()
+
+    def wait(self):
+        """Wait for service to complete."""
+
+    def reset(self):
+        """Reset service.
+
+        Called in case service running in daemon mode receives SIGHUP.
+        """

watcher/decision_engine/strategy/strategies/base.py

@@ -40,6 +40,7 @@ import abc
 import six
 
 from watcher.common import clients
+from watcher.common import exception
 from watcher.common.loader import loadable
 from watcher.common import utils
 from watcher.decision_engine.loading import default as loading
@@ -176,6 +177,9 @@ class BaseStrategy(loadable.Loadable):
                 'compute', osc=self.osc)
             self._compute_model = collector.get_latest_cluster_data_model()
 
+        if not self._compute_model:
+            raise exception.ClusterStateNotDefined()
+
         return self._compute_model
 
     @classmethod

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -413,7 +413,7 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
 
     def pre_execute(self):
         LOG.info(_LI("Initializing Sercon Consolidation"))
-        if self.compute_model is None:
+        if not self.compute_model:
             raise exception.ClusterStateNotDefined()
 
     def do_execute(self):

watcher/decision_engine/strategy/strategies/outlet_temp_control.py

@@ -226,7 +226,7 @@ class OutletTempControl(base.ThermalOptimizationBaseStrategy):
     def pre_execute(self):
         LOG.debug("Initializing Outlet temperature strategy")
 
-        if self.compute_model is None:
+        if not self.compute_model:
             raise wexc.ClusterStateNotDefined()
 
     def do_execute(self):

watcher/decision_engine/strategy/strategies/uniform_airflow.py

@@ -272,7 +272,7 @@ class UniformAirflow(base.BaseStrategy):
     def pre_execute(self):
         LOG.debug("Initializing Uniform Airflow Strategy")
 
-        if self.compute_model is None:
+        if not self.compute_model:
             raise wexc.ClusterStateNotDefined()
 
     def do_execute(self):

watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py

@@ -491,7 +491,7 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
             asc += 1
 
     def pre_execute(self):
-        if self.compute_model is None:
+        if not self.compute_model:
             raise exception.ClusterStateNotDefined()
 
     def do_execute(self):

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -265,7 +265,7 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
         """
         LOG.info(_LI("Initializing Workload Balance Strategy"))
 
-        if self.compute_model is None:
+        if not self.compute_model:
             raise wexc.ClusterStateNotDefined()
 
     def do_execute(self):

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -369,7 +369,7 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
     def pre_execute(self):
         LOG.info(_LI("Initializing Workload Stabilization"))
 
-        if self.compute_model is None:
+        if not self.compute_model:
             raise exception.ClusterStateNotDefined()
 
     def do_execute(self):