Move datasource query_retry into baseclass.

Moves the query_retry method into the baseclass and makes the query
retry and timeout options part of the watcher_datasources config group.
This makes the query_retry behavior uniform across all datasources.

A new baseclass method named query_retry_reset is added so datasources
can define operations to perform when recovering from a query error.
Test cases are added to verify the behavior of query_retry.

The query_max_retries and query_timeout config parameters are
deprecated in the gnocchi_client group and will be removed in a future
release.

Change-Id: I33e9dc2d1f5ba8f83fcf1488ff583ca5be5529cc

watcher/tests/datasources/test_base.py

@@ -0,0 +1,69 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 European Organization for Nuclear Research (CERN)
+#
+# Authors: Corne Lukken <info@dantalion.nl>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+from oslo_config import cfg
+
+from watcher.common import exception
+from watcher.datasources import base as datasource
+from watcher.tests import base
+
+CONF = cfg.CONF
+
+
+class TestBaseDatasourceHelper(base.BaseTestCase):
+
+    def test_query_retry(self):
+        exc = Exception()
+        method = mock.Mock()
+        # first call will fail but second will succeed
+        method.side_effect = [exc, True]
+        # Max 2 attempts
+        CONF.set_override("query_max_retries", 2,
+                          group='watcher_datasources')
+        # Reduce sleep time to 0
+        CONF.set_override("query_timeout", 0,
+                          group='watcher_datasources')
+
+        helper = datasource.DataSourceBase()
+        helper.query_retry_reset = mock.Mock()
+
+        self.assertTrue(helper.query_retry(f=method))
+        helper.query_retry_reset.assert_called_once_with(exc)
+
+    def test_query_retry_exception(self):
+        exc = Exception()
+        method = mock.Mock()
+        # only third call will succeed
+        method.side_effect = [exc, exc, True]
+        # Max 2 attempts
+        CONF.set_override("query_max_retries", 2,
+                          group='watcher_datasources')
+        # Reduce sleep time to 0
+        CONF.set_override("query_timeout", 0,
+                          group='watcher_datasources')
+
+        helper = datasource.DataSourceBase()
+        helper.query_retry_reset = mock.Mock()
+
+        # Maximum number of retries exceeded query_retry should raise error
+        self.assertRaises(exception.DataSourceNotAvailable,
+                          helper.query_retry, f=method)
+        # query_retry_reset should be called twice
+        helper.query_retry_reset.assert_has_calls(
+            [mock.call(exc), mock.call(exc)])

watcher/tests/datasources/test_gnocchi_helper.py

@@ -127,7 +127,7 @@ class TestGnocchiHelper(base.BaseTestCase):
 
     def test_gnocchi_check_availability_with_failure(self, mock_gnocchi):
         cfg.CONF.set_override("query_max_retries", 1,
-                              group='gnocchi_client')
+                              group='watcher_datasources')
         gnocchi = mock.MagicMock()
         gnocchi.status.get.side_effect = Exception()
         mock_gnocchi.return_value = gnocchi
@@ -147,7 +147,7 @@ class TestGnocchiHelper(base.BaseTestCase):
 
     def test_gnocchi_list_metrics_with_failure(self, mock_gnocchi):
         cfg.CONF.set_override("query_max_retries", 1,
-                              group='gnocchi_client')
+                              group='watcher_datasources')
         gnocchi = mock.MagicMock()
         gnocchi.metric.list.side_effect = Exception()
         mock_gnocchi.return_value = gnocchi