Avoid performing retries in case of missing resources

There may be no available metrics for instances that are stopped
or were recently spawned. This makes retries unnecessary and time
consuming.

For this reason, we'll ignore gnocchi MetricNotFound errors.

Change-Id: I79cd03bf04db634b931d6dfd32d5150f58e82044

watcher/decision_engine/datasources/base.py

@@ -63,7 +63,7 @@ class DataSourceBase(object):
             raise exception.MetricNotAvailable(metric=meter_name)
         return meter
 
-    def query_retry(self, f, *args, **kwargs):
+    def query_retry(self, f, *args, ignored_exc=None, **kwargs):
         """Attempts to retrieve metrics from the external service
 
         Attempts to access data from the external service and handles
@@ -71,15 +71,23 @@ class DataSourceBase(object):
         to the value of query_max_retries
         :param f: The method that performs the actual querying for metrics
         :param args: Array of arguments supplied to the method
+        :param ignored_exc: An exception or tuple of exceptions that shouldn't
+                            be retried, for example "NotFound" exceptions.
         :param kwargs: The amount of arguments supplied to the method
         :return: The value as retrieved from the external service
         """
 
         num_retries = CONF.watcher_datasources.query_max_retries
         timeout = CONF.watcher_datasources.query_timeout
+        ignored_exc = ignored_exc or tuple()
+
         for i in range(num_retries):
             try:
                 return f(*args, **kwargs)
+            except ignored_exc as e:
+                LOG.debug("Got an ignored exception (%s) while calling: %s ",
+                          e, f)
+                return
             except Exception as e:
                 LOG.exception(e)
                 self.query_retry_reset(e)

watcher/decision_engine/datasources/gnocchi.py

@@ -19,6 +19,7 @@
 from datetime import datetime
 from datetime import timedelta
 
+from gnocchiclient import exceptions as gnc_exc
 from oslo_config import cfg
 from oslo_log import log
 
@@ -84,7 +85,9 @@ class GnocchiHelper(base.DataSourceBase):
             kwargs = dict(query={"=": {"original_resource_id": resource_id}},
                           limit=1)
             resources = self.query_retry(
-                f=self.gnocchi.resource.search, **kwargs)
+                f=self.gnocchi.resource.search,
+                ignored_exc=gnc_exc.NotFound,
+                **kwargs)
 
             if not resources:
                 LOG.warning("The {0} resource {1} could not be "
@@ -105,7 +108,9 @@ class GnocchiHelper(base.DataSourceBase):
         kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
 
         statistics = self.query_retry(
-            f=self.gnocchi.metric.get_measures, **kwargs)
+            f=self.gnocchi.metric.get_measures,
+            ignored_exc=gnc_exc.NotFound,
+            **kwargs)
 
         return_value = None
         if statistics:
@@ -132,7 +137,9 @@ class GnocchiHelper(base.DataSourceBase):
             kwargs = dict(query={"=": {"original_resource_id": resource_id}},
                           limit=1)
             resources = self.query_retry(
-                f=self.gnocchi.resource.search, **kwargs)
+                f=self.gnocchi.resource.search,
+                ignored_exc=gnc_exc.NotFound,
+                **kwargs)
 
             if not resources:
                 LOG.warning("The {0} resource {1} could not be "
@@ -152,7 +159,9 @@ class GnocchiHelper(base.DataSourceBase):
         kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
 
         statistics = self.query_retry(
-            f=self.gnocchi.metric.get_measures, **kwargs)
+            f=self.gnocchi.metric.get_measures,
+            ignored_exc=gnc_exc.NotFound,
+            **kwargs)
 
         return_value = None
         if statistics: