Move queue declaration to project level

This moves the watcher queue declaration from the pipeline level
(where it is no longer valid) to the project level.

https: //lists.openstack.org/pipermail/openstack-discuss/2022-May/028603.html
Change-Id: I06923abb00f7eecd59587f44cd1f6a069e88a9fc
(cherry picked from commit 6003322711)

.zuul.yaml

@@ -1,4 +1,5 @@
 - project:
+    queue: watcher
     templates:
       - check-requirements
       - openstack-cover-jobs
@@ -16,7 +17,6 @@
         - watcher-tls-test
         - watcher-tempest-functional-ipv6-only
     gate:
-      queue: watcher
       jobs:
         - watcher-tempest-functional
         - watcher-tempest-functional-ipv6-only
@@ -160,7 +160,6 @@
     timeout: 7200
     required-projects: &base_required_projects
       - openstack/ceilometer
-      - openstack/devstack-gate
       - openstack/python-openstackclient
       - openstack/python-watcherclient
       - openstack/watcher
@@ -179,14 +178,10 @@
         s-container: false
         s-object: false
         s-proxy: false
-      devstack_localrc:
+      tempest_plugins:
-        TEMPEST_PLUGINS: /opt/stack/watcher-tempest-plugin
+        - watcher-tempest-plugin
-        USE_PYTHON3: true
       tempest_test_regex: watcher_tempest_plugin.tests.api
       tox_envlist: all
-      tox_environment:
-        # Do we really need to set this? It's cargo culted
-        PYTHONUNBUFFERED: 'true'
       zuul_copy_output:
         /etc/hosts: logs
 
@@ -200,10 +195,12 @@
 
 - job:
     name: watcher-grenade
-    parent: legacy-dsvm-base
+    parent: grenade
-    timeout: 10800
+    required-projects:
-    run: playbooks/legacy/grenade-devstack-watcher/run.yaml
+      - openstack/watcher
-    post-run: playbooks/legacy/grenade-devstack-watcher/post.yaml
+      - openstack/python-watcherclient
+      - openstack/watcher-tempest-plugin
+    vars: *base_vars
     irrelevant-files:
       - ^(test-|)requirements.txt$
       - ^.*\.rst$
@@ -215,12 +212,6 @@
       - ^setup.cfg$
       - ^tools/.*$
       - ^tox.ini$
-    required-projects:
-      - openstack/grenade
-      - openstack/devstack-gate
-      - openstack/watcher
-      - openstack/python-watcherclient
-      - openstack/watcher-tempest-plugin
 
 - job:
     # This job is used in python-watcherclient repo

playbooks/legacy/grenade-devstack-watcher/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/grenade-devstack-watcher/run.yaml

@@ -1,60 +0,0 @@
-- hosts: all
-  name: legacy-grenade-dsvm-watcher
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-
-          export PROJECTS="openstack/grenade $PROJECTS"
-          export PROJECTS="openstack/watcher $PROJECTS"
-          export PROJECTS="openstack/watcher-tempest-plugin $PROJECTS"
-          export PROJECTS="openstack/python-watcherclient $PROJECTS"
-          export DEVSTACK_PROJECT_FROM_GIT="python-watcherclient $DEVSTACK_PROJECT_FROM_GIT"
-
-          export GRENADE_PLUGINRC="enable_grenade_plugin watcher https://opendev.org/openstack/watcher"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"export TEMPEST_PLUGINS='/opt/stack/new/watcher-tempest-plugin'"
-
-          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
-          export DEVSTACK_GATE_GRENADE=pullup
-          export DEVSTACK_GATE_USE_PYTHON3=True
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-          # Add configuration values for enabling security features in local.conf
-          function pre_test_hook {
-              if [ -f /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh ] ; then
-                  . /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh
-              fi
-          }
-          export -f pre_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

watcher/api/controllers/v1/audit_template.py

@@ -138,6 +138,9 @@ class AuditTemplatePostType(wtypes.Base):
             raise exception.InvalidGoal(goal=audit_template.goal)
 
         if audit_template.scope:
+            keys = [list(s)[0] for s in audit_template.scope]
+            if keys[0] not in ('compute', 'storage'):
+                audit_template.scope = [dict(compute=audit_template.scope)]
             common_utils.Draft4Validator(
                 AuditTemplatePostType._build_schema()
                 ).validate(audit_template.scope)
@@ -158,18 +161,23 @@ class AuditTemplatePostType(wtypes.Base):
                         "included and excluded together"))
 
         if audit_template.strategy:
-            available_strategies = objects.Strategy.list(
+            try:
-                AuditTemplatePostType._ctx)
+                if (common_utils.is_uuid_like(audit_template.strategy) or
-            available_strategies_map = {
+                        common_utils.is_int_like(audit_template.strategy)):
-                s.uuid: s for s in available_strategies}
+                    strategy = objects.Strategy.get(
-            if audit_template.strategy not in available_strategies_map:
+                        AuditTemplatePostType._ctx, audit_template.strategy)
+                else:
+                    strategy = objects.Strategy.get_by_name(
+                        AuditTemplatePostType._ctx, audit_template.strategy)
+            except Exception:
                 raise exception.InvalidStrategy(
                     strategy=audit_template.strategy)
 
-            strategy = available_strategies_map[audit_template.strategy]
             # Check that the strategy we indicate is actually related to the
             # specified goal
             if strategy.goal_id != goal.id:
+                available_strategies = objects.Strategy.list(
+                    AuditTemplatePostType._ctx)
                 choices = ["'%s' (%s)" % (s.uuid, s.name)
                            for s in available_strategies]
                 raise exception.InvalidStrategy(

watcher/cmd/__init__.py

@@ -18,3 +18,10 @@
 import eventlet
 
 eventlet.monkey_patch()
+
+# Monkey patch the original current_thread to use the up-to-date _active
+# global variable. See https://bugs.launchpad.net/bugs/1863021 and
+# https://github.com/eventlet/eventlet/issues/592
+import __original_module_threading as orig_threading  # noqa
+import threading  # noqa
+orig_threading.current_thread.__globals__['_active'] = threading._active

watcher/decision_engine/model/collector/cinder.py

@@ -152,6 +152,9 @@ class CinderClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build storage data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build storage data model")
+            return
 
         builder = CinderModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/decision_engine/model/collector/ironic.py

@@ -63,6 +63,9 @@ class BaremetalClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build Baremetal data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build Baremetal data model")
+            return
 
         builder = BareMetalModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/decision_engine/model/collector/nova.py

@@ -184,6 +184,9 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build compute data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build compute data model")
+            return
 
         builder = NovaModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/tests/api/v1/test_audit_templates.py

@@ -555,6 +555,35 @@ class TestPost(FunctionalTestWithSetup):
             response.json['created_at']).replace(tzinfo=None)
         self.assertEqual(test_time, return_created_at)
 
+    @mock.patch.object(timeutils, 'utcnow')
+    def test_create_audit_template_with_strategy_name(self, mock_utcnow):
+        audit_template_dict = post_get_test_audit_template(
+            goal=self.fake_goal1.uuid,
+            strategy=self.fake_strategy1.name)
+        test_time = datetime.datetime(2000, 1, 1, 0, 0)
+        mock_utcnow.return_value = test_time
+
+        response = self.post_json('/audit_templates', audit_template_dict)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(201, response.status_int)
+        # Check location header
+        self.assertIsNotNone(response.location)
+        expected_location = \
+            '/v1/audit_templates/%s' % response.json['uuid']
+        self.assertEqual(urlparse.urlparse(response.location).path,
+                         expected_location)
+        self.assertTrue(utils.is_uuid_like(response.json['uuid']))
+        self.assertNotIn('updated_at', response.json.keys)
+        self.assertNotIn('deleted_at', response.json.keys)
+        self.assertEqual(self.fake_goal1.uuid, response.json['goal_uuid'])
+        self.assertEqual(self.fake_strategy1.uuid,
+                         response.json['strategy_uuid'])
+        self.assertEqual(self.fake_strategy1.name,
+                         response.json['strategy_name'])
+        return_created_at = timeutils.parse_isotime(
+            response.json['created_at']).replace(tzinfo=None)
+        self.assertEqual(test_time, return_created_at)
+
     def test_create_audit_template_validation_with_aggregates(self):
         scope = [{'compute': [{'host_aggregates': [{'id': '*'}]},
                               {'availability_zones': [{'name': 'AZ1'},
@@ -663,6 +692,26 @@ class TestPost(FunctionalTestWithSetup):
         self.assertEqual(400, response.status_int)
         assert not cn_mock.called
 
+    def test_create_audit_template_with_old_scope(self):
+        scope = [{'host_aggregates': [{'id': '*'}]},
+                 {'availability_zones': [{'name': 'AZ1'},
+                                         {'name': 'AZ2'}]},
+                 {'exclude': [
+                     {'instances': [
+                         {'uuid': 'INSTANCE_1'},
+                         {'uuid': 'INSTANCE_2'}]},
+                     {'compute_nodes': [
+                         {'name': 'Node_1'},
+                         {'name': 'Node_2'}]},
+                 ]}
+                 ]
+        audit_template_dict = post_get_test_audit_template(
+            goal=self.fake_goal1.uuid,
+            strategy=self.fake_strategy1.uuid, scope=scope)
+        response = self.post_json('/audit_templates',
+                                  audit_template_dict)
+        self.assertEqual(201, response.status_int)
+
 
 class TestDelete(api_base.FunctionalTest):
 

watcher/tests/decision_engine/cluster/test_cluster_data_model_collector.py

@@ -17,6 +17,9 @@
 import mock
 
 from watcher.decision_engine.model.collector import base
+from watcher.decision_engine.model.collector import cinder
+from watcher.decision_engine.model.collector import ironic
+from watcher.decision_engine.model.collector import nova
 from watcher.decision_engine.model import model_root
 from watcher.tests import base as test_base
 
@@ -55,3 +58,36 @@ class TestClusterDataModelCollector(test_base.TestCase):
         self.assertIsNot(
             collector.cluster_data_model,
             collector.get_latest_cluster_data_model())
+
+
+class TestComputeDataModelCollector(test_base.TestCase):
+
+    def test_model_scope_is_none(self):
+        m_config = mock.Mock()
+        collector = nova.NovaClusterDataModelCollector(config=m_config)
+
+        collector._audit_scope_handler = mock.Mock()
+        collector._data_model_scope = None
+        self.assertIsNone(collector.execute())
+
+
+class TestStorageDataModelCollector(test_base.TestCase):
+
+    def test_model_scope_is_none(self):
+        m_config = mock.Mock()
+        collector = cinder.CinderClusterDataModelCollector(config=m_config)
+
+        collector._audit_scope_handler = mock.Mock()
+        collector._data_model_scope = None
+        self.assertIsNone(collector.execute())
+
+
+class TestBareMetalDataModelCollector(test_base.TestCase):
+
+    def test_model_scope_is_none(self):
+        m_config = mock.Mock()
+        collector = ironic.BaremetalClusterDataModelCollector(config=m_config)
+
+        collector._audit_scope_handler = mock.Mock()
+        collector._data_model_scope = None
+        self.assertIsNone(collector.execute())