Move queue declaration to project level

This moves the watcher queue declaration from the pipeline level
(where it is no longer valid) to the project level.

https: //lists.openstack.org/pipermail/openstack-discuss/2022-May/028603.html
Change-Id: I06923abb00f7eecd59587f44cd1f6a069e88a9fc
(cherry picked from commit 6003322711)

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
+defaultbranch=stable/xena

.zuul.yaml

@@ -3,7 +3,7 @@
     templates:
       - check-requirements
       - openstack-cover-jobs
-      - openstack-python3-zed-jobs
+      - openstack-python3-xena-jobs
       - publish-openstack-docs-pti
       - release-notes-jobs-python3
     check:

doc/source/conf.py

@@ -56,8 +56,8 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = 'Watcher'
-copyright = 'OpenStack Foundation'
+project = u'Watcher'
+copyright = u'OpenStack Foundation'
 
 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['watcher.']
@@ -91,14 +91,14 @@ pygments_style = 'native'
 # List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'
 
 man_pages = [
-    ('man/watcher-api', 'watcher-api', 'Watcher API Server',
-     ['OpenStack'], 1),
-    ('man/watcher-applier', 'watcher-applier', 'Watcher Applier',
-     ['OpenStack'], 1),
+    ('man/watcher-api', 'watcher-api', u'Watcher API Server',
+     [u'OpenStack'], 1),
+    ('man/watcher-applier', 'watcher-applier', u'Watcher Applier',
+     [u'OpenStack'], 1),
     ('man/watcher-db-manage', 'watcher-db-manage',
-     'Watcher Db Management Utility', ['OpenStack'], 1),
+     u'Watcher Db Management Utility', [u'OpenStack'], 1),
     ('man/watcher-decision-engine', 'watcher-decision-engine',
-     'Watcher Decision Engine', ['OpenStack'], 1),
+     u'Watcher Decision Engine', [u'OpenStack'], 1),
 ]
 
 # -- Options for HTML output --------------------------------------------------
@@ -128,8 +128,8 @@ openstackdocs_bug_tag = ''
 latex_documents = [
     ('index',
      'doc-watcher.tex',
-     'Watcher Documentation',
-     'OpenStack Foundation', 'manual'),
+     u'Watcher Documentation',
+     u'OpenStack Foundation', 'manual'),
 ]
 
 # If false, no module index is generated.

releasenotes/source/conf.py

@@ -53,7 +53,7 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-copyright = '2016, Watcher developers'
+copyright = u'2016, Watcher developers'
 
 # Release notes are version independent
 # The short X.Y version.
@@ -196,8 +196,8 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual])
 latex_documents = [
-  ('index', 'watcher.tex', 'Watcher Documentation',
-   'Watcher developers', 'manual'),
+  ('index', 'watcher.tex', u'Watcher Documentation',
+   u'Watcher developers', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -226,8 +226,8 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'watcher', 'Watcher Documentation',
-     ['Watcher developers'], 1)
+    ('index', 'watcher', u'Watcher Documentation',
+     [u'Watcher developers'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -240,8 +240,8 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'watcher', 'Watcher Documentation',
-   'Watcher developers', 'watcher', 'One line description of project.',
+  ('index', 'watcher', u'Watcher Documentation',
+   u'Watcher developers', 'watcher', 'One line description of project.',
    'Miscellaneous'),
 ]
 

releasenotes/source/index.rst

@@ -21,8 +21,6 @@ Contents:
    :maxdepth: 1
 
    unreleased
-   yoga
-   xena
    wallaby
    victoria
    ussuri

releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po

@@ -1,16 +1,15 @@
 # Andi Chandler <andi@gowling.com>, 2017. #zanata
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
 # Andi Chandler <andi@gowling.com>, 2020. #zanata
-# Andi Chandler <andi@gowling.com>, 2022. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: python-watcher\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-08-29 03:02+0000\n"
+"POT-Creation-Date: 2020-10-27 04:13+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2022-05-31 08:39+0000\n"
+"PO-Revision-Date: 2020-10-28 11:13+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -65,9 +64,6 @@ msgstr "3.0.0"
 msgid "4.0.0"
 msgstr "4.0.0"
 
-msgid "6.0.0"
-msgstr "6.0.0"
-
 msgid "A ``watcher-status upgrade check`` has been added for this."
 msgstr "A ``watcher-status upgrade check`` has been added for this."
 
@@ -748,23 +744,6 @@ msgstr ""
 "The configuration options for query retries in `[gnocchi_client]` are "
 "deprecated and the option in `[watcher_datasources]` should now be used."
 
-msgid ""
-"The default value of ``[oslo_policy] policy_file`` config option has been "
-"changed from ``policy.json`` to ``policy.yaml``. Operators who are utilizing "
-"customized or previously generated static policy JSON files (which are not "
-"needed by default), should generate new policy files or convert them in YAML "
-"format. Use the `oslopolicy-convert-json-to-yaml <https://docs.openstack.org/"
-"oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ tool to "
-"convert a JSON to YAML formatted policy file in backward compatible way."
-msgstr ""
-"The default value of ``[oslo_policy] policy_file`` config option has been "
-"changed from ``policy.json`` to ``policy.yaml``. Operators who are utilizing "
-"customized or previously generated static policy JSON files (which are not "
-"needed by default), should generate new policy files or convert them in YAML "
-"format. Use the `oslopolicy-convert-json-to-yaml <https://docs.openstack.org/"
-"oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ tool to "
-"convert a JSON to YAML formatted policy file in backward compatible way."
-
 msgid ""
 "The graph model describes how VMs are associated to compute hosts. This "
 "allows for seeing relationships upfront between the entities and hence can "
@@ -820,21 +799,6 @@ msgstr "Train Series Release Notes"
 msgid "Upgrade Notes"
 msgstr "Upgrade Notes"
 
-msgid ""
-"Use of JSON policy files was deprecated by the ``oslo.policy`` library "
-"during the Victoria development cycle. As a result, this deprecation is "
-"being noted in the Wallaby cycle with an anticipated future removal of "
-"support by ``oslo.policy``. As such operators will need to convert to YAML "
-"policy files. Please see the upgrade notes for details on migration of any "
-"custom policy files."
-msgstr ""
-"Use of JSON policy files was deprecated by the ``oslo.policy`` library "
-"during the Victoria development cycle. As a result, this deprecation is "
-"being noted in the Wallaby cycle with an anticipated future removal of "
-"support by ``oslo.policy``. As such operators will need to convert to YAML "
-"policy files. Please see the upgrade notes for details on migration of any "
-"custom policy files."
-
 msgid ""
 "Using ``watcher/api/app.wsgi`` script is deprecated and it will be removed "
 "in U release. Please switch to automatically generated ``watcher-api-wsgi`` "
@@ -850,9 +814,6 @@ msgstr "Ussuri Series Release Notes"
 msgid "Victoria Series Release Notes"
 msgstr "Victoria Series Release Notes"
 
-msgid "Wallaby Series Release Notes"
-msgstr "Wallaby Series Release Notes"
-
 msgid ""
 "Watcher can continuously optimize the OpenStack cloud for a specific "
 "strategy or goal by triggering an audit periodically which generates an "
@@ -963,12 +924,6 @@ msgstr "We also add some new properties:"
 msgid "Welcome to watcher's Release Notes documentation!"
 msgstr "Welcome to watcher's Release Notes documentation!"
 
-msgid "Xena Series Release Notes"
-msgstr "Xena Series Release Notes"
-
-msgid "Yoga Series Release Notes"
-msgstr "Yoga Series Release Notes"
-
 msgid "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
 msgstr "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
 

releasenotes/source/xena.rst

@@ -1,6 +0,0 @@
-=========================
-Xena Series Release Notes
-=========================
-
-.. release-notes::
-   :branch: stable/xena

releasenotes/source/yoga.rst

@@ -1,6 +0,0 @@
-=========================
-Yoga Series Release Notes
-=========================
-
-.. release-notes::
-   :branch: stable/yoga

requirements.txt

@@ -30,6 +30,7 @@ pbr>=3.1.1 # Apache-2.0
 pecan>=1.3.2 # BSD
 PrettyTable>=0.7.2 # BSD
 gnocchiclient>=7.0.1 # Apache-2.0
+python-ceilometerclient>=2.9.0 # Apache-2.0
 python-cinderclient>=3.5.0 # Apache-2.0
 python-glanceclient>=2.9.1 # Apache-2.0
 python-keystoneclient>=3.15.0 # Apache-2.0

test-requirements.txt

@@ -12,4 +12,3 @@ testtools>=2.3.0 # MIT
 stestr>=2.0.0 # Apache-2.0
 os-api-ref>=1.4.0 # Apache-2.0
 bandit>=1.6.0 # Apache-2.0
-WebTest>=2.0.27 # MIT

tox.ini

@@ -9,7 +9,7 @@ basepython = python3
 usedevelop = True
 allowlist_externals = find
                       rm
-install_command = pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages}
+install_command = pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/xena} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps =

watcher/db/sqlalchemy/alembic/versions/0f6042416884_add_apscheduler_jobs.py

@@ -6,7 +6,6 @@ Create Date: 2017-03-24 11:21:29.036532
 
 """
 from alembic import op
-from sqlalchemy import inspect
 import sqlalchemy as sa
 
 from watcher.db.sqlalchemy import models
@@ -15,17 +14,8 @@ from watcher.db.sqlalchemy import models
 revision = '0f6042416884'
 down_revision = '001'
 
-def _table_exists(table_name):
-    bind = op.get_context().bind
-    insp = inspect(bind)
-    names = insp.get_table_names()
-    return any(t == table_name for t in names)
-
 
 def upgrade():
-    if _table_exists('apscheduler_jobs'):
-        return
-
     op.create_table(
         'apscheduler_jobs',
         sa.Column('id', sa.Unicode(191, _warn_on_bytestring=False),

watcher/db/sqlalchemy/api.py

@@ -44,11 +44,7 @@ _FACADE = None
 def _create_facade_lazily():
     global _FACADE
     if _FACADE is None:
-        # FIXME(amoralej): Remove autocommit=True (and ideally use of
-        # LegacyEngineFacade) asap since it's not compatible with SQLAlchemy
-        # 2.0.
-        _FACADE = db_session.EngineFacade.from_config(CONF,
-                                                      autocommit=True)
+        _FACADE = db_session.EngineFacade.from_config(CONF)
     return _FACADE
 
 

watcher/locale/en_GB/LC_MESSAGES/watcher.po

@@ -1,16 +1,15 @@
 # Andi Chandler <andi@gowling.com>, 2017. #zanata
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
 # Andi Chandler <andi@gowling.com>, 2020. #zanata
-# Andi Chandler <andi@gowling.com>, 2022. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: watcher VERSION\n"
 "Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
-"POT-Creation-Date: 2022-08-29 03:03+0000\n"
+"POT-Creation-Date: 2020-10-27 04:14+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2022-05-31 08:38+0000\n"
+"PO-Revision-Date: 2020-10-28 11:02+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -508,9 +507,6 @@ msgstr ""
 msgid "Plugins"
 msgstr "Plugins"
 
-msgid "Policy File JSON to YAML Migration"
-msgstr "Policy File JSON to YAML Migration"
-
 #, python-format
 msgid "Policy doesn't allow %(action)s to be performed."
 msgstr "Policy doesn't allow %(action)s to be performed."

watcher/tests/common/test_clients.py

@@ -10,6 +10,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from ceilometerclient import client as ceclient
+import ceilometerclient.v2.client as ceclient_v2
 from unittest import mock
 
 from cinderclient import client as ciclient
@@ -268,6 +270,56 @@ class TestClients(base.TestCase):
         cinder_cached = osc.cinder()
         self.assertEqual(cinder, cinder_cached)
 
+    @mock.patch.object(ceclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_ceilometer(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        osc.ceilometer()
+        mock_call.assert_called_once_with(
+            CONF.ceilometer_client.api_version,
+            None,
+            endpoint_type=CONF.ceilometer_client.endpoint_type,
+            region_name=CONF.ceilometer_client.region_name,
+            session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    @mock.patch.object(ceclient_v2.Client, '_get_redirect_client')
+    def test_clients_ceilometer_diff_vers(self, mock_get_redirect_client,
+                                          mock_session):
+        '''ceilometerclient currently only has one version (v2)'''
+        mock_get_redirect_client.return_value = [mock.Mock(), mock.Mock()]
+        CONF.set_override('api_version', '2',
+                          group='ceilometer_client')
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        osc.ceilometer()
+        self.assertEqual(ceclient_v2.Client,
+                         type(osc.ceilometer()))
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    @mock.patch.object(ceclient_v2.Client, '_get_redirect_client')
+    def test_clients_ceilometer_diff_endpoint(self, mock_get_redirect_client,
+                                              mock_session):
+        mock_get_redirect_client.return_value = [mock.Mock(), mock.Mock()]
+        CONF.set_override('endpoint_type', 'publicURL',
+                          group='ceilometer_client')
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        osc.ceilometer()
+        self.assertEqual('publicURL', osc.ceilometer().http_client.interface)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    @mock.patch.object(ceclient_v2.Client, '_get_redirect_client')
+    def test_clients_ceilometer_cached(self, mock_get_redirect_client,
+                                       mock_session):
+        mock_get_redirect_client.return_value = [mock.Mock(), mock.Mock()]
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        ceilometer = osc.ceilometer()
+        ceilometer_cached = osc.ceilometer()
+        self.assertEqual(ceilometer, ceilometer_cached)
+
     @mock.patch.object(netclient, 'Client')
     @mock.patch.object(clients.OpenStackClients, 'session')
     def test_clients_neutron(self, mock_session, mock_call):

watcher/tests/decision_engine/datasources/test_ceilometer_helper.py

@@ -0,0 +1,168 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from unittest import mock
+
+from watcher.common import clients
+from watcher.common import exception
+from watcher.decision_engine.datasources import ceilometer as ceilometer_helper
+from watcher.tests import base
+
+
+@mock.patch.object(clients.OpenStackClients, 'ceilometer')
+class TestCeilometerHelper(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestCeilometerHelper, self).setUp()
+        self.osc_mock = mock.Mock()
+        self.helper = ceilometer_helper.CeilometerHelper(osc=self.osc_mock)
+        stat_agg_patcher = mock.patch.object(
+            self.helper, 'statistic_aggregation',
+            spec=ceilometer_helper.CeilometerHelper.statistic_aggregation)
+        self.mock_aggregation = stat_agg_patcher.start()
+        self.addCleanup(stat_agg_patcher.stop)
+
+    def test_build_query(self, mock_ceilometer):
+        mock_ceilometer.return_value = mock.MagicMock()
+        cm = ceilometer_helper.CeilometerHelper()
+        expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'},
+                    {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'},
+                    {'field': 'resource_id', 'op': 'eq',
+                     'value': u'resource_id'}]
+
+        query = cm.build_query(user_id="user_id",
+                               tenant_id="tenant_id",
+                               resource_id="resource_id",
+                               user_ids=["user_ids"],
+                               tenant_ids=["tenant_ids"],
+                               resource_ids=["resource_ids"])
+        self.assertEqual(expected, query)
+
+    def test_statistic_aggregation(self, mock_ceilometer):
+        ceilometer = mock.MagicMock()
+        statistic = mock.MagicMock()
+        expected_result = 100
+        statistic[-1]._info = {'aggregate': {'avg': expected_result}}
+        ceilometer.statistics.list.return_value = statistic
+        mock_ceilometer.return_value = ceilometer
+        cm = ceilometer_helper.CeilometerHelper()
+        val = cm.statistic_aggregation(
+            resource=mock.Mock(id="INSTANCE_ID"),
+            resource_type='instance',
+            meter_name="instance_cpu_usage",
+            period="7300",
+            granularity=None
+        )
+        self.assertEqual(expected_result, val)
+
+    def test_statistic_aggregation_metric_unavailable(self, mock_ceilometer):
+        helper = ceilometer_helper.CeilometerHelper()
+
+        # invalidate instance_cpu_usage in metric map
+        original_metric_value = helper.METRIC_MAP.get('instance_cpu_usage')
+        helper.METRIC_MAP.update(
+            instance_cpu_usage=None
+        )
+
+        self.assertRaises(
+            exception.MetricNotAvailable,
+            helper.statistic_aggregation, resource=mock.Mock(id="INSTANCE_ID"),
+            resource_type='instance', meter_name="instance_cpu_usage",
+            period="7300",
+            granularity=None
+        )
+
+        # restore the metric map as it is a static attribute that does not get
+        # restored between unit tests!
+        helper.METRIC_MAP.update(
+            instance_cpu_usage=original_metric_value
+        )
+
+    def test_get_host_cpu_usage(self, mock_ceilometer):
+        self.helper.get_host_cpu_usage('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'compute_node', 'host_cpu_usage', 600, 'mean', None)
+
+    def test_get_host_ram_usage(self, mock_ceilometer):
+        self.helper.get_host_ram_usage('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'compute_node', 'host_ram_usage', 600, 'mean', None)
+
+    def test_get_host_outlet_temp(self, mock_ceilometer):
+        self.helper.get_host_outlet_temp('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'compute_node', 'host_outlet_temp', 600, 'mean', None)
+
+    def test_get_host_inlet_temp(self, mock_ceilometer):
+        self.helper.get_host_inlet_temp('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'compute_node', 'host_inlet_temp', 600, 'mean', None)
+
+    def test_get_host_airflow(self, mock_ceilometer):
+        self.helper.get_host_airflow('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'compute_node', 'host_airflow', 600, 'mean', None)
+
+    def test_get_host_power(self, mock_ceilometer):
+        self.helper.get_host_power('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'compute_node', 'host_power', 600, 'mean', None)
+
+    def test_get_instance_cpu_usage(self, mock_ceilometer):
+        self.helper.get_instance_cpu_usage('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'instance', 'instance_cpu_usage', 600, 'mean',
+            None)
+
+    def test_get_instance_ram_usage(self, mock_ceilometer):
+        self.helper.get_instance_ram_usage('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'instance', 'instance_ram_usage', 600, 'mean',
+            None)
+
+    def test_get_instance_ram_allocated(self, mock_ceilometer):
+        self.helper.get_instance_ram_allocated('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'instance', 'instance_ram_allocated', 600, 'mean',
+            None)
+
+    def test_get_instance_l3_cache_usage(self, mock_ceilometer):
+        self.helper.get_instance_l3_cache_usage('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'instance', 'instance_l3_cache_usage', 600, 'mean',
+            None)
+
+    def test_get_instance_root_disk_size(self, mock_ceilometer):
+        self.helper.get_instance_root_disk_size('compute1', 600, 'mean')
+        self.mock_aggregation.assert_called_once_with(
+            'compute1', 'instance', 'instance_root_disk_size', 600, 'mean',
+            None)
+
+    def test_check_availability(self, mock_ceilometer):
+        ceilometer = mock.MagicMock()
+        ceilometer.resources.list.return_value = True
+        mock_ceilometer.return_value = ceilometer
+        helper = ceilometer_helper.CeilometerHelper()
+        result = helper.check_availability()
+        self.assertEqual('available', result)
+
+    def test_check_availability_with_failure(self, mock_ceilometer):
+        ceilometer = mock.MagicMock()
+        ceilometer.resources.list.side_effect = Exception()
+        mock_ceilometer.return_value = ceilometer
+        helper = ceilometer_helper.CeilometerHelper()
+
+        self.assertEqual('not available', helper.check_availability())