Added DE Background Scheduler w/ model sync jobs

In this changeset, I implemented a background scheduler service
for Watcher and more particularly for the Decision Engine where
I made it create 2 types of job per cluster data model collector
plugin:

- An initial job that is asynchronously executed upon starting the
  Decision Engine
- A periodical job that gets triggered every configurable interval
  of time

Change-Id: I3f5442f81933a19565217b894bd86c186e339762
Partially-Implements: bluprint cluster-model-objects-wrapper

watcher/tests/decision_engine/strategy/strategies/faker_cluster_state.py

@@ -29,7 +29,7 @@ class FakerModelCollector(base.BaseClusterDataModelCollector):
 
     def __init__(self, config=None, osc=None):
         if config is None:
-            config = mock.Mock()
+            config = mock.Mock(period=777)
         super(FakerModelCollector, self).__init__(config)
 
     def execute(self):

watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py

@@ -168,6 +168,14 @@ class TestBasicConsolidation(base.BaseTestCase):
         self.assertEqual(expected_num_migrations, num_migrations)
         self.assertEqual(expected_power_state, num_hypervisor_state_change)
 
+    def test_exception_stale_cdm(self):
+        self.fake_cluster.set_cluster_data_model_as_stale()
+        self.m_model.return_value = self.fake_cluster.cluster_data_model
+
+        self.assertRaises(
+            exception.ClusterStateNotDefined,
+            self.strategy.execute)
+
     # calculate_weight
     def test_execute_no_workload(self):
         model = (

watcher/tests/decision_engine/strategy/strategies/test_outlet_temp_control.py

@@ -109,6 +109,14 @@ class TestOutletTempControl(base.BaseTestCase):
         self.m_model.return_value = model
         self.assertRaises(exception.ClusterEmpty, self.strategy.execute)
 
+    def test_exception_stale_cdm(self):
+        self.fake_cluster.set_cluster_data_model_as_stale()
+        self.m_model.return_value = self.fake_cluster.cluster_data_model
+
+        self.assertRaises(
+            exception.ClusterStateNotDefined,
+            self.strategy.execute)
+
     def test_execute_cluster_empty(self):
         model = model_root.ModelRoot()
         self.m_model.return_value = model

watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py

@@ -136,6 +136,14 @@ class TestUniformAirflow(base.BaseTestCase):
         self.m_model.return_value = model
         self.assertRaises(exception.ClusterEmpty, self.strategy.execute)
 
+    def test_exception_stale_cdm(self):
+        self.fake_cluster.set_cluster_data_model_as_stale()
+        self.m_model.return_value = self.fake_cluster.cluster_data_model
+
+        self.assertRaises(
+            exception.ClusterStateNotDefined,
+            self.strategy.execute)
+
     def test_execute_cluster_empty(self):
         model = model_root.ModelRoot()
         self.m_model.return_value = model

watcher/tests/decision_engine/strategy/strategies/test_vm_workload_consolidation.py

@@ -20,6 +20,7 @@
 
 import mock
 
+from watcher.common import exception
 from watcher.decision_engine.model import model_root
 from watcher.decision_engine.strategy import strategies
 from watcher.tests import base
@@ -56,6 +57,14 @@ class TestVMWorkloadConsolidation(base.BaseTestCase):
             statistic_aggregation=self.fake_metrics.mock_get_statistics)
         self.strategy = strategies.VMWorkloadConsolidation(config=mock.Mock())
 
+    def test_exception_stale_cdm(self):
+        self.fake_cluster.set_cluster_data_model_as_stale()
+        self.m_model.return_value = self.fake_cluster.cluster_data_model
+
+        self.assertRaises(
+            exception.ClusterStateNotDefined,
+            self.strategy.execute)
+
     def test_get_vm_utilization(self):
         model = self.fake_cluster.generate_scenario_1()
         self.m_model.return_value = model

watcher/tests/decision_engine/strategy/strategies/test_workload_balance.py

@@ -126,6 +126,14 @@ class TestWorkloadBalance(base.BaseTestCase):
         self.m_model.return_value = model
         self.assertRaises(exception.ClusterEmpty, self.strategy.execute)
 
+    def test_exception_stale_cdm(self):
+        self.fake_cluster.set_cluster_data_model_as_stale()
+        self.m_model.return_value = self.fake_cluster.cluster_data_model
+
+        self.assertRaises(
+            exception.ClusterStateNotDefined,
+            self.strategy.execute)
+
     def test_execute_cluster_empty(self):
         model = model_root.ModelRoot()
         self.m_model.return_value = model

watcher/tests/decision_engine/test_scheduling.py

@@ -0,0 +1,87 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from apscheduler.schedulers import background
+from apscheduler.triggers import interval as interval_trigger
+import eventlet
+import mock
+
+from watcher.decision_engine import scheduling
+from watcher.metrics_engine.loading import default as default_loading
+from watcher.tests import base
+from watcher.tests.decision_engine.strategy.strategies import \
+    faker_cluster_state
+
+
+class TestDecisionEngineSchedulingService(base.TestCase):
+
+    @mock.patch.object(
+        default_loading.ClusterDataModelCollectorLoader, 'load')
+    @mock.patch.object(
+        default_loading.ClusterDataModelCollectorLoader, 'list_available')
+    @mock.patch.object(background.BackgroundScheduler, 'start')
+    def test_start_de_scheduling_service(self, m_start, m_list_available,
+                                         m_load):
+        m_list_available.return_value = {
+            'fake': faker_cluster_state.FakerModelCollector}
+        fake_collector = faker_cluster_state.FakerModelCollector(
+            config=mock.Mock(period=777))
+        m_load.return_value = fake_collector
+
+        scheduler = scheduling.DecisionEngineSchedulingService()
+
+        scheduler.start()
+
+        m_start.assert_called_once_with(scheduler)
+        jobs = scheduler.get_jobs()
+        self.assertEqual(1, len(jobs))
+
+        job = jobs[0]
+        self.assertTrue(bool(fake_collector.cluster_data_model))
+
+        self.assertIsInstance(job.trigger, interval_trigger.IntervalTrigger)
+
+    @mock.patch.object(
+        default_loading.ClusterDataModelCollectorLoader, 'load')
+    @mock.patch.object(
+        default_loading.ClusterDataModelCollectorLoader, 'list_available')
+    @mock.patch.object(background.BackgroundScheduler, 'start')
+    def test_execute_sync_job_fails(self, m_start, m_list_available,
+                                    m_load):
+        fake_config = mock.Mock(period=.01)
+        fake_collector = faker_cluster_state.FakerModelCollector(
+            config=fake_config)
+        fake_collector.synchronize = mock.Mock(
+            side_effect=lambda: eventlet.sleep(.5))
+        m_list_available.return_value = {
+            'fake': faker_cluster_state.FakerModelCollector}
+        m_load.return_value = fake_collector
+
+        scheduler = scheduling.DecisionEngineSchedulingService()
+
+        scheduler.start()
+
+        m_start.assert_called_once_with(scheduler)
+        jobs = scheduler.get_jobs()
+        self.assertEqual(1, len(jobs))
+
+        job = jobs[0]
+        job.func()
+        self.assertFalse(bool(fake_collector.cluster_data_model))
+
+        self.assertIsInstance(job.trigger, interval_trigger.IntervalTrigger)