Move queue declaration to project level

This moves the watcher queue declaration from the pipeline level
(where it is no longer valid) to the project level.

https: //lists.openstack.org/pipermail/openstack-discuss/2022-May/028603.html
Change-Id: I06923abb00f7eecd59587f44cd1f6a069e88a9fc
(cherry picked from commit 6003322711)

.zuul.yaml

@@ -1,4 +1,5 @@
 - project:
+    queue: watcher
     templates:
       - check-requirements
       - openstack-cover-jobs
@@ -17,7 +18,6 @@
         - watcher-tls-test
         - watcher-tempest-functional-ipv6-only
     gate:
-      queue: watcher
       jobs:
         - watcher-tempest-functional
         - watcher-tempest-functional-ipv6-only

watcher/api/controllers/v1/audit_template.py

@@ -138,6 +138,9 @@ class AuditTemplatePostType(wtypes.Base):
             raise exception.InvalidGoal(goal=audit_template.goal)
 
         if audit_template.scope:
+            keys = [list(s)[0] for s in audit_template.scope]
+            if keys[0] not in ('compute', 'storage'):
+                audit_template.scope = [dict(compute=audit_template.scope)]
             common_utils.Draft4Validator(
                 AuditTemplatePostType._build_schema()
                 ).validate(audit_template.scope)
@@ -158,18 +161,23 @@ class AuditTemplatePostType(wtypes.Base):
                         "included and excluded together"))
 
         if audit_template.strategy:
-            available_strategies = objects.Strategy.list(
+            try:
-                AuditTemplatePostType._ctx)
+                if (common_utils.is_uuid_like(audit_template.strategy) or
-            available_strategies_map = {
+                        common_utils.is_int_like(audit_template.strategy)):
-                s.uuid: s for s in available_strategies}
+                    strategy = objects.Strategy.get(
-            if audit_template.strategy not in available_strategies_map:
+                        AuditTemplatePostType._ctx, audit_template.strategy)
+                else:
+                    strategy = objects.Strategy.get_by_name(
+                        AuditTemplatePostType._ctx, audit_template.strategy)
+            except Exception:
                 raise exception.InvalidStrategy(
                     strategy=audit_template.strategy)
 
-            strategy = available_strategies_map[audit_template.strategy]
             # Check that the strategy we indicate is actually related to the
             # specified goal
             if strategy.goal_id != goal.id:
+                available_strategies = objects.Strategy.list(
+                    AuditTemplatePostType._ctx)
                 choices = ["'%s' (%s)" % (s.uuid, s.name)
                            for s in available_strategies]
                 raise exception.InvalidStrategy(

watcher/decision_engine/datasources/base.py

@@ -19,8 +19,6 @@ import time
 from oslo_config import cfg
 from oslo_log import log
 
-from watcher.common import exception
-
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
@@ -79,7 +77,6 @@ class DataSourceBase(object):
                 LOG.warning("Retry {0} of {1} while retrieving metrics retry "
                             "in {2} seconds".format(i+1, num_retries, timeout))
                 time.sleep(timeout)
-        raise exception.DataSourceNotAvailable(datasource=self.NAME)
 
     @abc.abstractmethod
     def query_retry_reset(self, exception_instance):

watcher/decision_engine/datasources/ceilometer.py

@@ -136,19 +136,18 @@ class CeilometerHelper(base.DataSourceBase):
 
     def list_metrics(self):
         """List the user's meters."""
-        try:
+        meters = self.query_retry(f=self.ceilometer.meters.list)
-            meters = self.query_retry(f=self.ceilometer.meters.list)
+        if not meters:
-        except Exception:
             return set()
         else:
             return meters
 
     def check_availability(self):
-        try:
+        status = self.query_retry(self.ceilometer.resources.list)
-            self.query_retry(self.ceilometer.resources.list)
+        if status:
-        except Exception:
+            return 'available'
+        else:
             return 'not available'
-        return 'available'
 
     def query_sample(self, meter_name, query, limit=1):
         return self.query_retry(f=self.ceilometer.samples.list,

watcher/decision_engine/datasources/gnocchi.py

@@ -52,17 +52,16 @@ class GnocchiHelper(base.DataSourceBase):
         self.gnocchi = self.osc.gnocchi()
 
     def check_availability(self):
-        try:
+        status = self.query_retry(self.gnocchi.status.get)
-            self.query_retry(self.gnocchi.status.get)
+        if status:
-        except Exception:
+            return 'available'
+        else:
             return 'not available'
-        return 'available'
 
     def list_metrics(self):
         """List the user's meters."""
-        try:
+        response = self.query_retry(f=self.gnocchi.metric.list)
-            response = self.query_retry(f=self.gnocchi.metric.list)
+        if not response:
-        except Exception:
             return set()
         else:
             return set([metric['name'] for metric in response])
@@ -91,8 +90,9 @@ class GnocchiHelper(base.DataSourceBase):
                 f=self.gnocchi.resource.search, **kwargs)
 
             if not resources:
-                raise exception.ResourceNotFound(name='gnocchi',
+                LOG.warning("The {0} resource {1} could not be "
-                                                 id=resource_id)
+                            "found".format(self.NAME, resource_id))
+                return
 
             resource_id = resources[0]['id']
 
@@ -110,6 +110,7 @@ class GnocchiHelper(base.DataSourceBase):
         statistics = self.query_retry(
             f=self.gnocchi.metric.get_measures, **kwargs)
 
+        return_value = None
         if statistics:
             # return value of latest measure
             # measure has structure [time, granularity, value]
@@ -120,7 +121,7 @@ class GnocchiHelper(base.DataSourceBase):
                 # 1/10 th of actual CFM
                 return_value *= 10
 
-            return return_value
+        return return_value
 
     def get_host_cpu_usage(self, resource, period, aggregate,
                            granularity=300):

watcher/decision_engine/datasources/grafana.py

@@ -179,6 +179,9 @@ class GrafanaHelper(base.DataSourceBase):
         kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
 
         resp = self.query_retry(self._request, **kwargs)
+        if not resp:
+            LOG.warning("Datasource {0} is not available.".format(self.NAME))
+            return
 
         result = translator.extract_result(resp.content)
 

watcher/decision_engine/datasources/monasca.py

@@ -73,11 +73,11 @@ class MonascaHelper(base.DataSourceBase):
             self.monasca = self.osc.monasca()
 
     def check_availability(self):
-        try:
+        result = self.query_retry(self.monasca.metrics.list)
-            self.query_retry(self.monasca.metrics.list)
+        if result:
-        except Exception:
+            return 'available'
+        else:
             return 'not available'
-        return 'available'
 
     def list_metrics(self):
         # TODO(alexchadin): this method should be implemented in accordance to

watcher/decision_engine/model/collector/cinder.py

@@ -152,6 +152,9 @@ class CinderClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build storage data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build storage data model")
+            return
 
         builder = CinderModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/decision_engine/model/collector/ironic.py

@@ -63,6 +63,9 @@ class BaremetalClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build Baremetal data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build Baremetal data model")
+            return
 
         builder = BareMetalModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/decision_engine/model/collector/nova.py

@@ -181,6 +181,9 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build compute data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build compute data model")
+            return
 
         builder = NovaModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/tests/api/base.py

@@ -105,7 +105,6 @@ class FunctionalTest(base.DbTestCase):
         :param path_prefix: prefix of the url path
         """
         full_path = path_prefix + path
-        print('%s: %s %s' % (method.upper(), full_path, params))
 
         response = getattr(self.app, "%s_json" % method)(
             str(full_path),
@@ -115,7 +114,6 @@ class FunctionalTest(base.DbTestCase):
             extra_environ=extra_environ,
             expect_errors=expect_errors
         )
-        print('GOT:%s' % response)
         return response
 
     def put_json(self, path, params, expect_errors=False, headers=None,
@@ -192,13 +190,11 @@ class FunctionalTest(base.DbTestCase):
         :param path_prefix: prefix of the url path
         """
         full_path = path_prefix + path
-        print('DELETE: %s' % (full_path))
         response = self.app.delete(str(full_path),
                                    headers=headers,
                                    status=status,
                                    extra_environ=extra_environ,
                                    expect_errors=expect_errors)
-        print('GOT:%s' % response)
         return response
 
     def get_json(self, path, expect_errors=False, headers=None,
@@ -229,7 +225,6 @@ class FunctionalTest(base.DbTestCase):
         all_params.update(params)
         if q:
             all_params.update(query_params)
-        print('GET: %s %r' % (full_path, all_params))
 
         response = self.app.get(full_path,
                                 params=all_params,
@@ -238,7 +233,6 @@ class FunctionalTest(base.DbTestCase):
                                 expect_errors=expect_errors)
         if return_json and not expect_errors:
             response = response.json
-        print('GOT:%s' % response)
         return response
 
     def validate_link(self, link, bookmark=False):

watcher/tests/api/v1/test_audit_templates.py

@@ -555,6 +555,35 @@ class TestPost(FunctionalTestWithSetup):
             response.json['created_at']).replace(tzinfo=None)
         self.assertEqual(test_time, return_created_at)
 
+    @mock.patch.object(timeutils, 'utcnow')
+    def test_create_audit_template_with_strategy_name(self, mock_utcnow):
+        audit_template_dict = post_get_test_audit_template(
+            goal=self.fake_goal1.uuid,
+            strategy=self.fake_strategy1.name)
+        test_time = datetime.datetime(2000, 1, 1, 0, 0)
+        mock_utcnow.return_value = test_time
+
+        response = self.post_json('/audit_templates', audit_template_dict)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(201, response.status_int)
+        # Check location header
+        self.assertIsNotNone(response.location)
+        expected_location = \
+            '/v1/audit_templates/%s' % response.json['uuid']
+        self.assertEqual(urlparse.urlparse(response.location).path,
+                         expected_location)
+        self.assertTrue(utils.is_uuid_like(response.json['uuid']))
+        self.assertNotIn('updated_at', response.json.keys)
+        self.assertNotIn('deleted_at', response.json.keys)
+        self.assertEqual(self.fake_goal1.uuid, response.json['goal_uuid'])
+        self.assertEqual(self.fake_strategy1.uuid,
+                         response.json['strategy_uuid'])
+        self.assertEqual(self.fake_strategy1.name,
+                         response.json['strategy_name'])
+        return_created_at = timeutils.parse_isotime(
+            response.json['created_at']).replace(tzinfo=None)
+        self.assertEqual(test_time, return_created_at)
+
     def test_create_audit_template_validation_with_aggregates(self):
         scope = [{'compute': [{'host_aggregates': [{'id': '*'}]},
                               {'availability_zones': [{'name': 'AZ1'},
@@ -663,6 +692,26 @@ class TestPost(FunctionalTestWithSetup):
         self.assertEqual(400, response.status_int)
         assert not cn_mock.called
 
+    def test_create_audit_template_with_old_scope(self):
+        scope = [{'host_aggregates': [{'id': '*'}]},
+                 {'availability_zones': [{'name': 'AZ1'},
+                                         {'name': 'AZ2'}]},
+                 {'exclude': [
+                     {'instances': [
+                         {'uuid': 'INSTANCE_1'},
+                         {'uuid': 'INSTANCE_2'}]},
+                     {'compute_nodes': [
+                         {'name': 'Node_1'},
+                         {'name': 'Node_2'}]},
+                 ]}
+                 ]
+        audit_template_dict = post_get_test_audit_template(
+            goal=self.fake_goal1.uuid,
+            strategy=self.fake_strategy1.uuid, scope=scope)
+        response = self.post_json('/audit_templates',
+                                  audit_template_dict)
+        self.assertEqual(201, response.status_int)
+
 
 class TestDelete(api_base.FunctionalTest):
 

watcher/tests/decision_engine/cluster/test_cluster_data_model_collector.py

@@ -17,6 +17,9 @@
 import mock
 
 from watcher.decision_engine.model.collector import base
+from watcher.decision_engine.model.collector import cinder
+from watcher.decision_engine.model.collector import ironic
+from watcher.decision_engine.model.collector import nova
 from watcher.decision_engine.model import model_root
 from watcher.tests import base as test_base
 
@@ -55,3 +58,36 @@ class TestClusterDataModelCollector(test_base.TestCase):
         self.assertIsNot(
             collector.cluster_data_model,
             collector.get_latest_cluster_data_model())
+
+
+class TestComputeDataModelCollector(test_base.TestCase):
+
+    def test_model_scope_is_none(self):
+        m_config = mock.Mock()
+        collector = nova.NovaClusterDataModelCollector(config=m_config)
+
+        collector._audit_scope_handler = mock.Mock()
+        collector._data_model_scope = None
+        self.assertIsNone(collector.execute())
+
+
+class TestStorageDataModelCollector(test_base.TestCase):
+
+    def test_model_scope_is_none(self):
+        m_config = mock.Mock()
+        collector = cinder.CinderClusterDataModelCollector(config=m_config)
+
+        collector._audit_scope_handler = mock.Mock()
+        collector._data_model_scope = None
+        self.assertIsNone(collector.execute())
+
+
+class TestBareMetalDataModelCollector(test_base.TestCase):
+
+    def test_model_scope_is_none(self):
+        m_config = mock.Mock()
+        collector = ironic.BaremetalClusterDataModelCollector(config=m_config)
+
+        collector._audit_scope_handler = mock.Mock()
+        collector._data_model_scope = None
+        self.assertIsNone(collector.execute())

watcher/tests/decision_engine/datasources/test_base.py

@@ -19,7 +19,6 @@
 import mock
 from oslo_config import cfg
 
-from watcher.common import exception
 from watcher.decision_engine.datasources import base as datasource
 from watcher.tests import base
 
@@ -61,9 +60,8 @@ class TestBaseDatasourceHelper(base.BaseTestCase):
         helper = datasource.DataSourceBase()
         helper.query_retry_reset = mock.Mock()
 
-        # Maximum number of retries exceeded query_retry should raise error
+        # Maximum number of retries exceeded query_retry should return None
-        self.assertRaises(exception.DataSourceNotAvailable,
+        self.assertIsNone(helper.query_retry(f=method))
-                          helper.query_retry, f=method)
         # query_retry_reset should be called twice
         helper.query_retry_reset.assert_has_calls(
             [mock.call(exc), mock.call(exc)])

watcher/tests/decision_engine/datasources/test_grafana_helper.py

@@ -132,11 +132,7 @@ class TestGrafana(base.BaseTestCase):
 
         t_grafana = grafana.GrafanaHelper(osc=mock.Mock())
 
-        self.assertRaises(
+        self.assertIsNone(t_grafana.get_host_cpu_usage(self.m_compute_node))
-            exception.DataSourceNotAvailable,
-            t_grafana.get_host_cpu_usage,
-            self.m_compute_node
-        )
 
     def test_no_metric_raise_error(self):
         """Test raising error when specified meter does not exist"""