Allow for global datasources preference from config

Allows to define a global preference for metric datasources with the
ability for strategy specific overrides. In addition, strategies which
do not require datasources have the config options removed this is
done to prevent confusion.

Some documentation that details the inner workings of selecting
datasources is updated.

Imports for some files in watcher/common have been changed to resolve
circular dependencies and now match the overall method to import
configuration.

Addtional datasources will be retrieved by the manager if the
datasource throws an error.

Implements: blueprint global-datasource-preference
Change-Id: I6fc455b288e338c20d2c4cfec5a0c95350bebc36

watcher/tests/conf/test_list_opts.py

@@ -29,8 +29,8 @@ class TestListOpts(base.TestCase):
         super(TestListOpts, self).setUp()
         self.base_sections = [
             'DEFAULT', 'api', 'database', 'watcher_decision_engine',
-            'watcher_applier', 'watcher_planner', 'nova_client',
-            'glance_client', 'gnocchi_client', 'cinder_client',
+            'watcher_applier', 'watcher_datasources', 'watcher_planner',
+            'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client',
             'ceilometer_client', 'monasca_client', 'ironic_client',
             'neutron_client', 'watcher_clients_auth', 'collector']
         self.opt_sections = list(dict(opts.list_opts()).keys())

watcher/tests/datasources/test_ceilometer_helper.py

@@ -20,7 +20,7 @@ from __future__ import unicode_literals
 import mock
 
 from watcher.common import clients
-from watcher.datasource import ceilometer as ceilometer_helper
+from watcher.datasources import ceilometer as ceilometer_helper
 from watcher.tests import base
 
 

watcher/tests/datasources/test_gnocchi_helper.py

@@ -18,7 +18,7 @@ import mock
 from oslo_config import cfg
 
 from watcher.common import clients
-from watcher.datasource import gnocchi as gnocchi_helper
+from watcher.datasources import gnocchi as gnocchi_helper
 from watcher.tests import base
 
 CONF = cfg.CONF

watcher/tests/datasources/test_manager.py

@@ -17,7 +17,8 @@
 import mock
 
 from watcher.common import exception
-from watcher.datasource import manager as ds_manager
+from watcher.datasources import gnocchi
+from watcher.datasources import manager as ds_manager
 from watcher.tests import base
 
 
@@ -46,3 +47,13 @@ class TestDataSourceManager(base.BaseTestCase):
             osc=mock.MagicMock())
         self.assertRaises(exception.NoSuchMetric, manager.get_backend,
                           ['host_cpu', 'instance_cpu_usage'])
+
+    @mock.patch.object(gnocchi, 'GnocchiHelper')
+    def test_get_backend_error_datasource(self, m_gnocchi):
+        m_gnocchi.side_effect = exception.DataSourceNotAvailable
+        manager = ds_manager.DataSourceManager(
+            config=mock.MagicMock(
+                datasources=['gnocchi', 'ceilometer', 'monasca']),
+            osc=mock.MagicMock())
+        backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
+        self.assertEqual(backend, manager.ceilometer)

watcher/tests/datasources/test_monasca_helper.py

@@ -18,7 +18,7 @@ import mock
 from oslo_config import cfg
 
 from watcher.common import clients
-from watcher.datasource import monasca as monasca_helper
+from watcher.datasources import monasca as monasca_helper
 from watcher.tests import base
 
 CONF = cfg.CONF

watcher/tests/decision_engine/strategy/strategies/test_base.py

@@ -17,6 +17,7 @@
 import mock
 
 from watcher.common import exception
+from watcher.datasources import manager
 from watcher.decision_engine.model import model_root
 from watcher.decision_engine.strategy import strategies
 from watcher.tests import base
@@ -49,6 +50,67 @@ class TestBaseStrategy(base.TestCase):
         self.strategy = strategies.DummyStrategy(config=mock.Mock())
 
 
+class TestBaseStrategyDatasource(TestBaseStrategy):
+
+    def setUp(self):
+        super(TestBaseStrategyDatasource, self).setUp()
+        self.strategy = strategies.DummyStrategy(
+            config=mock.Mock(datasources=None))
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_global_preference(self, m_conf, m_manager):
+        """Test if the global preference is used"""
+
+        m_conf.watcher_datasources.datasources = \
+            ['gnocchi', 'monasca', 'ceilometer']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=m_conf.watcher_datasources, osc=None)
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_global_preference_reverse(self, m_conf, m_manager):
+        """Test if the global preference is used with another order"""
+
+        m_conf.watcher_datasources.datasources = \
+            ['ceilometer', 'monasca', 'gnocchi']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=m_conf.watcher_datasources, osc=None)
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_strategy_preference_override(self, m_conf, m_manager):
+        """Test if the global preference can be overridden"""
+
+        datasources = mock.Mock(datasources=['ceilometer'])
+
+        self.strategy = strategies.DummyStrategy(
+            config=datasources)
+
+        m_conf.watcher_datasources.datasources = \
+            ['ceilometer', 'monasca', 'gnocchi']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=datasources, osc=None)
+
+
 class TestBaseStrategyException(TestBaseStrategy):
 
     def setUp(self):