Merge "use cinder migrate for swap volume" into stable/2024.2

Adds a tempest configuration for min and max microversions supported
by watcher. This help us to define the correct range of microversion
to be tested on each stable branch.
New microversion proposals should also increase the default
max_microversion, in order to work with watcher-tempest-plugin
microversion testing.

Change-Id: I0b695ba4530eb89ed17b3935b87e938cadec84cc
(cherry picked from commit adfe3858aa)
(cherry picked from commit defd3953d8)
Signed-off-by: Douglas Viroel <viroel@gmail.com>

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
+defaultbranch=stable/2024.2

.zuul.yaml

@@ -1,8 +1,9 @@
 - project:
+    queue: watcher
     templates:
       - check-requirements
       - openstack-cover-jobs
-      - openstack-python3-wallaby-jobs
+      - openstack-python3-jobs
       - publish-openstack-docs-pti
       - release-notes-jobs-python3
     check:
@@ -12,10 +13,8 @@
         - watcher-tempest-strategies
         - watcher-tempest-actuator
         - watcherclient-tempest-functional
-        - watcher-tls-test
         - watcher-tempest-functional-ipv6-only
     gate:
-      queue: watcher
       jobs:
         - watcher-tempest-functional
         - watcher-tempest-functional-ipv6-only
@@ -86,22 +85,12 @@
     vars:
       tempest_concurrency: 1
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_strategies
-
-- job:
-    name: watcher-tls-test
-    parent: watcher-tempest-multinode
-    group-vars:
-      subnode:
-          devstack_services:
-            tls-proxy: true
-    vars:
-      devstack_services:
-        tls-proxy: true
+      tempest_exclude_regex: .*\[.*\breal_load\b.*\].*
 
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
-    nodeset: openstack-two-node-focal
+    nodeset: openstack-two-node-jammy
     roles:
       - zuul: openstack/tempest
     group-vars:
@@ -119,8 +108,7 @@
           watcher-api: false
           watcher-decision-engine: true
           watcher-applier: false
-          # We need to add TLS support for watcher plugin
-          tls-proxy: false
+          c-bak: false
           ceilometer: false
           ceilometer-acompute: false
           ceilometer-acentral: false
@@ -168,7 +156,6 @@
       devstack_plugins:
         watcher: https://opendev.org/openstack/watcher
       devstack_services:
-        tls-proxy: false
         watcher-api: true
         watcher-decision-engine: true
         watcher-applier: true

devstack/lib/watcher

@@ -275,6 +275,9 @@ function install_watcherclient {
         git_clone_by_name "python-watcherclient"
         setup_dev_lib "python-watcherclient"
     fi
+    if [[ "$GLOBAL_VENV" == "True" ]]; then
+        sudo ln -sf /opt/stack/data/venv/bin/watcher /usr/local/bin
+    fi
 }
 
 # install_watcher() - Collect source and prepare
@@ -338,6 +341,19 @@ function stop_watcher {
     done
 }
 
+# configure_tempest_for_watcher() - Configure Tempest for watcher
+function configure_tempest_for_watcher {
+    # Set default microversion for watcher-tempest-plugin
+    # Please make sure to update this when the microversion is updated, otherwise
+    # new tests may be skipped.
+    TEMPEST_WATCHER_MIN_MICROVERSION=${TEMPEST_WATCHER_MIN_MICROVERSION:-"1.0"}
+    TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.4"}
+
+    # Set microversion options in tempest.conf
+    iniset $TEMPEST_CONFIG optimize min_microversion $TEMPEST_WATCHER_MIN_MICROVERSION
+    iniset $TEMPEST_CONFIG optimize max_microversion $TEMPEST_WATCHER_MAX_MICROVERSION
+}
+
 # Restore xtrace
 $_XTRACE_WATCHER
 

devstack/plugin.sh

@@ -38,6 +38,9 @@ if is_service_enabled watcher-api watcher-decision-engine watcher-applier; then
         # Start the watcher components
         echo_summary "Starting watcher"
         start_watcher
+    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
+        echo_summary "Configuring tempest for watcher"
+        configure_tempest_for_watcher
     fi
 
     if [[ "$1" == "unstack" ]]; then

devstack/upgrade/upgrade.sh

@@ -70,7 +70,7 @@ then write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
 fi
 
 # Migrate the database
-watcher-db-manage upgrade || die $LINO "DB migration error"
+$WATCHER_BIN_DIR/watcher-db-manage upgrade || die $LINO "DB migration error"
 
 start_watcher
 

doc/source/conf.py

@@ -56,8 +56,8 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Watcher'
-copyright = u'OpenStack Foundation'
+project = 'Watcher'
+copyright = 'OpenStack Foundation'
 
 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['watcher.']
@@ -91,14 +91,14 @@ pygments_style = 'native'
 # List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'
 
 man_pages = [
-    ('man/watcher-api', 'watcher-api', u'Watcher API Server',
-     [u'OpenStack'], 1),
-    ('man/watcher-applier', 'watcher-applier', u'Watcher Applier',
-     [u'OpenStack'], 1),
+    ('man/watcher-api', 'watcher-api', 'Watcher API Server',
+     ['OpenStack'], 1),
+    ('man/watcher-applier', 'watcher-applier', 'Watcher Applier',
+     ['OpenStack'], 1),
     ('man/watcher-db-manage', 'watcher-db-manage',
-     u'Watcher Db Management Utility', [u'OpenStack'], 1),
+     'Watcher Db Management Utility', ['OpenStack'], 1),
     ('man/watcher-decision-engine', 'watcher-decision-engine',
-     u'Watcher Decision Engine', [u'OpenStack'], 1),
+     'Watcher Decision Engine', ['OpenStack'], 1),
 ]
 
 # -- Options for HTML output --------------------------------------------------
@@ -128,8 +128,8 @@ openstackdocs_bug_tag = ''
 latex_documents = [
     ('index',
      'doc-watcher.tex',
-     u'Watcher Documentation',
-     u'OpenStack Foundation', 'manual'),
+     'Watcher Documentation',
+     'OpenStack Foundation', 'manual'),
 ]
 
 # If false, no module index is generated.

doc/source/configuration/configuring.rst

@@ -372,7 +372,7 @@ You can configure and install Ceilometer by following the documentation below :
 #. https://docs.openstack.org/ceilometer/latest
 
 The built-in strategy 'basic_consolidation' provided by watcher requires
-"**compute.node.cpu.percent**" and "**cpu_util**" measurements to be collected
+"**compute.node.cpu.percent**" and "**cpu**" measurements to be collected
 by Ceilometer.
 The measurements available depend on the hypervisors that OpenStack manages on
 the specific implementation.

doc/source/contributor/devstack.rst

@@ -47,6 +47,8 @@ unavailable as well as `instance_l3_cpu_cache`::
         [[local|localrc]]
         enable_plugin watcher https://opendev.org/openstack/watcher
 
+        enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard
+
         enable_plugin ceilometer https://opendev.org/openstack/ceilometer.git
         CEILOMETER_BACKEND=gnocchi
 

doc/source/contributor/plugin/strategy-plugin.rst

@@ -300,6 +300,6 @@ Using that you can now query the values for that specific metric:
 .. code-block:: py
 
     avg_meter = self.datasource_backend.statistic_aggregation(
-        instance.uuid, 'cpu_util', self.periods['instance'],
+        instance.uuid, 'instance_cpu_usage', self.periods['instance'],
         self.granularity,
         aggregation=self.aggregation_method['instance'])

doc/source/strategies/basic-server-consolidation.rst

@@ -26,8 +26,7 @@ metric                       service name plugins comment
                                                   ``compute_monitors`` option
                                                   to ``cpu.virt_driver`` in
                                                   the nova.conf.
-``cpu_util``                 ceilometer_  none    cpu_util has been removed
-                                                  since Stein.
+``cpu``                      ceilometer_  none
 ============================ ============ ======= ===========================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute

doc/source/strategies/saving_energy.rst

@@ -89,9 +89,9 @@ step 2: Create audit to do optimization
 .. code-block:: shell
 
     $ openstack optimize audittemplate create \
-      at1 saving_energy --strategy saving_energy
+      saving_energy_template1 saving_energy --strategy saving_energy
 
-    $ openstack optimize audit create -a at1 \
+    $ openstack optimize audit create -a saving_energy_audit1 \
       -p free_used_percent=20.0
 
 External Links

doc/source/strategies/vm_workload_consolidation.rst

@@ -22,14 +22,19 @@ The *vm_workload_consolidation* strategy requires the following metrics:
 ============================ ============ ======= =========================
 metric                       service name plugins comment
 ============================ ============ ======= =========================
-``cpu_util``                 ceilometer_  none    cpu_util has been removed
-                                                  since Stein.
+``cpu``                      ceilometer_  none
 ``memory.resident``          ceilometer_  none
 ``memory``                   ceilometer_  none
 ``disk.root.size``           ceilometer_  none
+``compute.node.cpu.percent`` ceilometer_  none    (optional) need to set the
+                                                  ``compute_monitors`` option
+                                                  to ``cpu.virt_driver`` in the
+                                                  nova.conf.
+``hardware.memory.used``     ceilometer_  SNMP_   (optional)
 ============================ ============ ======= =========================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
+.. _SNMP: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#snmp-based-meters
 
 Cluster data model
 ******************

doc/source/strategies/workload-stabilization.rst

@@ -27,9 +27,8 @@ metric                       service name plugins comment
                                                   to ``cpu.virt_driver`` in the
                                                   nova.conf.
 ``hardware.memory.used``     ceilometer_  SNMP_
-``cpu_util``                 ceilometer_  none    cpu_util has been removed
-                                                  since Stein.
-``memory.resident``          ceilometer_  none
+``cpu``                      ceilometer_  none
+``instance_ram_usage``       ceilometer_  none
 ============================ ============ ======= =============================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
@@ -107,10 +106,10 @@ parameter            type   default Value         description
                                                   period of all received ones.
 ==================== ====== ===================== =============================
 
-.. |metrics| replace:: ["cpu_util", "memory.resident"]
-.. |thresholds| replace:: {"cpu_util": 0.2, "memory.resident": 0.2}
-.. |weights| replace:: {"cpu_util_weight": 1.0, "memory.resident_weight": 1.0}
-.. |instance_metrics| replace:: {"cpu_util": "compute.node.cpu.percent", "memory.resident": "hardware.memory.used"}
+.. |metrics| replace:: ["instance_cpu_usage", "instance_ram_usage"]
+.. |thresholds| replace:: {"instance_cpu_usage": 0.2, "instance_ram_usage": 0.2}
+.. |weights| replace:: {"instance_cpu_usage_weight": 1.0, "instance_ram_usage_weight": 1.0}
+.. |instance_metrics| replace:: {"instance_cpu_usage": "compute.node.cpu.percent", "instance_ram_usage": "hardware.memory.used"}
 .. |periods| replace:: {"instance": 720, "node": 600}
 
 Efficacy Indicator
@@ -136,8 +135,8 @@ How to use it ?
       at1 workload_balancing --strategy workload_stabilization
 
     $ openstack optimize audit create -a at1 \
-      -p thresholds='{"memory.resident": 0.05}' \
-      -p metrics='["memory.resident"]'
+      -p thresholds='{"instance_ram_usage": 0.05}' \
+      -p metrics='["instance_ram_usage"]'
 
 External Links
 --------------

doc/source/strategies/workload_balance.rst

@@ -24,8 +24,7 @@ The *workload_balance* strategy requires the following metrics:
 ======================= ============ ======= =========================
 metric                  service name plugins comment
 ======================= ============ ======= =========================
-``cpu_util``            ceilometer_  none    cpu_util has been removed
-                                             since Stein.
+``cpu``                 ceilometer_  none
 ``memory.resident``     ceilometer_  none
 ======================= ============ ======= =========================
 
@@ -65,15 +64,16 @@ Configuration
 
 Strategy parameters are:
 
-============== ====== ============= ====================================
-parameter      type   default Value description
-============== ====== ============= ====================================
-``metrics``    String 'cpu_util'    Workload balance base on cpu or ram
-                                    utilization. choice: ['cpu_util',
-                                    'memory.resident']
-``threshold``  Number 25.0          Workload threshold for migration
-``period``     Number 300           Aggregate time period of ceilometer
-============== ====== ============= ====================================
+============== ====== ==================== ====================================
+parameter      type   default Value        description
+============== ====== ==================== ====================================
+``metrics``    String 'instance_cpu_usage' Workload balance base on cpu or ram
+                                           utilization. Choices:
+                                           ['instance_cpu_usage',
+                                           'instance_ram_usage']
+``threshold``  Number 25.0                 Workload threshold for migration
+``period``     Number 300                  Aggregate time period of ceilometer
+============== ====== ==================== ====================================
 
 Efficacy Indicator
 ------------------
@@ -95,7 +95,7 @@ How to use it ?
       at1 workload_balancing --strategy workload_balance
 
     $ openstack optimize audit create -a at1 -p threshold=26.0 \
-            -p period=310 -p metrics=cpu_util
+            -p period=310 -p metrics=instance_cpu_usage
 
 External Links
 --------------

releasenotes/notes/bug-2112187-763bae283e0b736d.yaml

@@ -0,0 +1,47 @@
+---
+security:
+  - |
+    Watchers no longer forges requests on behalf of a tenant when
+    swapping volumes. Prior to this release watcher had 2 implementations
+    of moving a volume, it could use cinders volume migrate api or its own
+    internal implementation that directly calls nova volume attachment update
+    api. The former is safe and the recommend way to move volumes between
+    cinder storage backend the internal implementation was insecure, fragile
+    due to a lack of error handling and capable of deleting user data.
+
+    Insecure: the internal volume migration operation created a new keystone
+    user with a weak name and password and added it to the tenants project
+    with the admin role. It then used that user to forge request on behalf
+    of the tenant with admin right to swap the volume. if the applier was
+    restarted during the execution of this operation it would never be cleaned
+    up.
+
+    Fragile: the error handling was minimal, the swap volume api is async
+    so watcher has to poll for completion, there was no support to resume
+    that if interrupted of the time out was exceeded.
+
+    Data-loss: while the internal polling logic returned success or failure
+    watcher did not check the result, once the function returned it
+    unconditionally deleted the source volume. For larger volumes this
+    could result in irretrievable data loss.
+
+    Finally if a volume was swapped using the internal workflow it put
+    the nova instance in an out of sync state. If the VM was live migrated
+    after the swap volume completed successfully prior to a hard reboot
+    then the migration would fail or succeed and break tenant isolation.
+
+    see: https://bugs.launchpad.net/nova/+bug/2112187 for details.
+fixes:
+  - |
+    All code related to creating keystone user and granting roles has been
+    removed. The internal swap volume implementation has been removed and
+    replaced by cinders volume migrate api. Note as part of this change
+    Watcher will no longer attempt volume migrations or retypes if the
+    instance is in the `Verify Resize` task state. This resolves several
+    issues related to volume migration in the zone migration and
+    Storage capacity balance strategies. While efforts have been made
+    to maintain backward compatibility these changes are required to
+    address a security weakness in watcher's prior approach.
+
+    see: https://bugs.launchpad.net/nova/+bug/2112187 for more context.
+

releasenotes/source/2023.1.rst

@@ -0,0 +1,6 @@
+===========================
+2023.1 Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/2023.1

releasenotes/source/2023.2.rst

@@ -0,0 +1,6 @@
+===========================
+2023.2 Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/2023.2

releasenotes/source/conf.py

@@ -53,7 +53,7 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-copyright = u'2016, Watcher developers'
+copyright = '2016, Watcher developers'
 
 # Release notes are version independent
 # The short X.Y version.
@@ -196,8 +196,8 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual])
 latex_documents = [
-  ('index', 'watcher.tex', u'Watcher Documentation',
-   u'Watcher developers', 'manual'),
+  ('index', 'watcher.tex', 'Watcher Documentation',
+   'Watcher developers', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -226,8 +226,8 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'watcher', u'Watcher Documentation',
-     [u'Watcher developers'], 1)
+    ('index', 'watcher', 'Watcher Documentation',
+     ['Watcher developers'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -240,8 +240,8 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'watcher', u'Watcher Documentation',
-   u'Watcher developers', 'watcher', 'One line description of project.',
+  ('index', 'watcher', 'Watcher Documentation',
+   'Watcher developers', 'watcher', 'One line description of project.',
    'Miscellaneous'),
 ]
 

releasenotes/source/index.rst

@@ -21,6 +21,12 @@ Contents:
    :maxdepth: 1
 
    unreleased
+   2023.2
+   2023.1
+   zed
+   yoga
+   xena
+   wallaby
    victoria
    ussuri
    train

releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po

@@ -1,15 +1,18 @@
 # Andi Chandler <andi@gowling.com>, 2017. #zanata
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
 # Andi Chandler <andi@gowling.com>, 2020. #zanata
+# Andi Chandler <andi@gowling.com>, 2022. #zanata
+# Andi Chandler <andi@gowling.com>, 2023. #zanata
+# Andi Chandler <andi@gowling.com>, 2024. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: python-watcher\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-10-27 04:13+0000\n"
+"POT-Creation-Date: 2024-05-31 14:40+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2020-10-28 11:13+0000\n"
+"PO-Revision-Date: 2024-04-18 12:21+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -58,12 +61,21 @@ msgstr "1.9.0"
 msgid "2.0.0"
 msgstr "2.0.0"
 
+msgid "2023.1 Series Release Notes"
+msgstr "2023.1 Series Release Notes"
+
+msgid "2023.2 Series Release Notes"
+msgstr "2023.2 Series Release Notes"
+
 msgid "3.0.0"
 msgstr "3.0.0"
 
 msgid "4.0.0"
 msgstr "4.0.0"
 
+msgid "6.0.0"
+msgstr "6.0.0"
+
 msgid "A ``watcher-status upgrade check`` has been added for this."
 msgstr "A ``watcher-status upgrade check`` has been added for this."
 
@@ -744,6 +756,23 @@ msgstr ""
 "The configuration options for query retries in `[gnocchi_client]` are "
 "deprecated and the option in `[watcher_datasources]` should now be used."
 
+msgid ""
+"The default value of ``[oslo_policy] policy_file`` config option has been "
+"changed from ``policy.json`` to ``policy.yaml``. Operators who are utilizing "
+"customized or previously generated static policy JSON files (which are not "
+"needed by default), should generate new policy files or convert them in YAML "
+"format. Use the `oslopolicy-convert-json-to-yaml <https://docs.openstack.org/"
+"oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ tool to "
+"convert a JSON to YAML formatted policy file in backward compatible way."
+msgstr ""
+"The default value of ``[oslo_policy] policy_file`` config option has been "
+"changed from ``policy.json`` to ``policy.yaml``. Operators who are utilizing "
+"customized or previously generated static policy JSON files (which are not "
+"needed by default), should generate new policy files or convert them in YAML "
+"format. Use the `oslopolicy-convert-json-to-yaml <https://docs.openstack.org/"
+"oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ tool to "
+"convert a JSON to YAML formatted policy file in backward compatible way."
+
 msgid ""
 "The graph model describes how VMs are associated to compute hosts. This "
 "allows for seeing relationships upfront between the entities and hence can "
@@ -799,6 +828,21 @@ msgstr "Train Series Release Notes"
 msgid "Upgrade Notes"
 msgstr "Upgrade Notes"
 
+msgid ""
+"Use of JSON policy files was deprecated by the ``oslo.policy`` library "
+"during the Victoria development cycle. As a result, this deprecation is "
+"being noted in the Wallaby cycle with an anticipated future removal of "
+"support by ``oslo.policy``. As such operators will need to convert to YAML "
+"policy files. Please see the upgrade notes for details on migration of any "
+"custom policy files."
+msgstr ""
+"Use of JSON policy files was deprecated by the ``oslo.policy`` library "
+"during the Victoria development cycle. As a result, this deprecation is "
+"being noted in the Wallaby cycle with an anticipated future removal of "
+"support by ``oslo.policy``. As such operators will need to convert to YAML "
+"policy files. Please see the upgrade notes for details on migration of any "
+"custom policy files."
+
 msgid ""
 "Using ``watcher/api/app.wsgi`` script is deprecated and it will be removed "
 "in U release. Please switch to automatically generated ``watcher-api-wsgi`` "
@@ -814,6 +858,9 @@ msgstr "Ussuri Series Release Notes"
 msgid "Victoria Series Release Notes"
 msgstr "Victoria Series Release Notes"
 
+msgid "Wallaby Series Release Notes"
+msgstr "Wallaby Series Release Notes"
+
 msgid ""
 "Watcher can continuously optimize the OpenStack cloud for a specific "
 "strategy or goal by triggering an audit periodically which generates an "
@@ -924,6 +971,15 @@ msgstr "We also add some new properties:"
 msgid "Welcome to watcher's Release Notes documentation!"
 msgstr "Welcome to watcher's Release Notes documentation!"
 
+msgid "Xena Series Release Notes"
+msgstr "Xena Series Release Notes"
+
+msgid "Yoga Series Release Notes"
+msgstr "Yoga Series Release Notes"
+
+msgid "Zed Series Release Notes"
+msgstr "Zed Series Release Notes"
+
 msgid "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
 msgstr "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
 

releasenotes/source/wallaby.rst

@@ -0,0 +1,6 @@
+============================
+Wallaby Series Release Notes
+============================
+
+.. release-notes::
+   :branch: stable/wallaby

releasenotes/source/xena.rst

@@ -0,0 +1,6 @@
+=========================
+Xena Series Release Notes
+=========================
+
+.. release-notes::
+   :branch: stable/xena

releasenotes/source/yoga.rst

@@ -0,0 +1,6 @@
+=========================
+Yoga Series Release Notes
+=========================
+
+.. release-notes::
+   :branch: stable/yoga

releasenotes/source/zed.rst

@@ -0,0 +1,6 @@
+========================
+Zed Series Release Notes
+========================
+
+.. release-notes::
+   :branch: stable/zed

requirements.txt

@@ -17,7 +17,7 @@ oslo.context>=2.21.0 # Apache-2.0
 oslo.db>=4.44.0 # Apache-2.0
 oslo.i18n>=3.20.0 # Apache-2.0
 oslo.log>=3.37.0 # Apache-2.0
-oslo.messaging>=8.1.2 # Apache-2.0
+oslo.messaging>=14.1.0 # Apache-2.0
 oslo.policy>=3.6.0 # Apache-2.0
 oslo.reports>=1.27.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
@@ -30,7 +30,6 @@ pbr>=3.1.1 # Apache-2.0
 pecan>=1.3.2 # BSD
 PrettyTable>=0.7.2 # BSD
 gnocchiclient>=7.0.1 # Apache-2.0
-python-ceilometerclient>=2.9.0 # Apache-2.0
 python-cinderclient>=3.5.0 # Apache-2.0
 python-glanceclient>=2.9.1 # Apache-2.0
 python-keystoneclient>=3.15.0 # Apache-2.0

setup.cfg

@@ -1,12 +1,12 @@
 [metadata]
 name = python-watcher
 summary = OpenStack Watcher provides a flexible and scalable resource optimization service for multi-tenant OpenStack-based clouds.
-description-file =
+description_file =
     README.rst
 author = OpenStack
-author-email = openstack-discuss@lists.openstack.org
-home-page = https://docs.openstack.org/watcher/latest/
-python-requires = >=3.6
+author_email = openstack-discuss@lists.openstack.org
+home_page = https://docs.openstack.org/watcher/latest/
+python_requires = >=3.8
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -17,9 +17,10 @@ classifier =
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.6
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [files]
 packages =

test-requirements.txt

@@ -7,9 +7,9 @@ doc8>=0.8.0 # Apache-2.0
 freezegun>=0.3.10 # Apache-2.0
 hacking>=3.0.1,<3.1.0 # Apache-2.0
 oslotest>=3.3.0 # Apache-2.0
-os-testr>=1.0.0 # Apache-2.0
 testscenarios>=0.5.0 # Apache-2.0/BSD
 testtools>=2.3.0 # MIT
 stestr>=2.0.0 # Apache-2.0
 os-api-ref>=1.4.0 # Apache-2.0
 bandit>=1.6.0 # Apache-2.0
+WebTest>=2.0.27 # MIT

tox.ini

@@ -1,37 +1,41 @@
 [tox]
-minversion = 2.0
-envlist = py38,pep8
-skipsdist = True
+minversion = 3.18.0
+envlist = py3,pep8
 ignore_basepython_conflict = True
 
 [testenv]
 basepython = python3
 usedevelop = True
-whitelist_externals = find
+allowlist_externals = find
                       rm
-install_command = pip install {opts} {packages}
+install_command = pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2024.2} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
+    python-libmaas>=0.6.8
 commands =
   rm -f .testrepository/times.dbm
   find . -type f -name "*.py[c|o]" -delete
   stestr run {posargs}
-passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+passenv =
+  http_proxy
+  HTTP_PROXY
+  https_proxy
+  HTTPS_PROXY
+  no_proxy
+  NO_PROXY
 
 [testenv:pep8]
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
+    #bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:venv]
 setenv = PYTHONHASHSEED=0
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
     -r{toxinidir}/doc/requirements.txt
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
@@ -49,14 +53,15 @@ commands =
 
 [testenv:docs]
 setenv = PYTHONHASHSEED=0
-deps = -r{toxinidir}/doc/requirements.txt
+deps =
+  -r{toxinidir}/doc/requirements.txt
 commands =
   rm -fr doc/build doc/source/api/ .autogenerated
   sphinx-build -W --keep-going -b html doc/source doc/build/html
 
 [testenv:api-ref]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals = bash
+allowlist_externals = bash
 commands =
   bash -c 'rm -rf api-ref/build'
   sphinx-build -W --keep-going -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
@@ -73,6 +78,28 @@ commands =
 commands =
     oslopolicy-sample-generator --config-file etc/watcher/oslo-policy-generator/watcher-policy-generator.conf
 
+[testenv:wheel]
+commands = python setup.py bdist_wheel
+
+[testenv:pdf-docs]
+envdir = {toxworkdir}/docs
+deps = {[testenv:docs]deps}
+allowlist_externals =
+  rm
+  make
+commands =
+  rm -rf doc/build/pdf
+  sphinx-build -W --keep-going -b latex doc/source doc/build/pdf
+  make -C doc/build/pdf
+
+[testenv:releasenotes]
+deps = -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
+
+[testenv:bandit]
+deps = -r{toxinidir}/test-requirements.txt
+commands = bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
+
 [flake8]
 filename = *.py,app.wsgi
 show-source=True
@@ -82,9 +109,6 @@ builtins= _
 enable-extensions = H106,H203,H904
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*,demo/,releasenotes
 
-[testenv:wheel]
-commands = python setup.py bdist_wheel
-
 [hacking]
 import_exceptions = watcher._i18n
 
@@ -108,27 +132,7 @@ extension =
    N366 = checks:import_stock_mock
 paths = ./watcher/hacking
 
-
 [doc8]
 extension=.rst
 # todo: stop ignoring doc/source/man when https://bugs.launchpad.net/doc8/+bug/1502391 is fixed
 ignore-path=doc/source/image_src,doc/source/man,doc/source/api
-
-[testenv:pdf-docs]
-envdir = {toxworkdir}/docs
-deps = {[testenv:docs]deps}
-whitelist_externals =
-  rm
-  make
-commands =
-  rm -rf doc/build/pdf
-  sphinx-build -W --keep-going -b latex doc/source doc/build/pdf
-  make -C doc/build/pdf
-
-[testenv:releasenotes]
-deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
-
-[testenv:bandit]
-deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r watcher -x watcher/tests/* -n5 -ll -s B320

watcher/api/controllers/v1/action.py

@@ -57,6 +57,7 @@ are dynamically loaded by Watcher at launch time.
 
 import datetime
 
+from http import HTTPStatus
 import pecan
 from pecan import rest
 import wsme
@@ -362,7 +363,7 @@ class ActionsController(rest.RestController):
 
         return Action.convert_with_links(action)
 
-    @wsme_pecan.wsexpose(Action, body=Action, status_code=201)
+    @wsme_pecan.wsexpose(Action, body=Action, status_code=HTTPStatus.CREATED)
     def post(self, action):
         """Create a new action(forbidden).
 
@@ -422,7 +423,7 @@ class ActionsController(rest.RestController):
         action_to_update.save()
         return Action.convert_with_links(action_to_update)
 
-    @wsme_pecan.wsexpose(None, types.uuid, status_code=204)
+    @wsme_pecan.wsexpose(None, types.uuid, status_code=HTTPStatus.NO_CONTENT)
     def delete(self, action_uuid):
         """Delete a action(forbidden).
 

watcher/api/controllers/v1/action_plan.py

@@ -56,6 +56,7 @@ state machine <action_plan_state_machine>`.
 
 import datetime
 
+from http import HTTPStatus
 from oslo_log import log
 import pecan
 from pecan import rest
@@ -460,7 +461,7 @@ class ActionPlansController(rest.RestController):
 
         return ActionPlan.convert_with_links(action_plan)
 
-    @wsme_pecan.wsexpose(None, types.uuid, status_code=204)
+    @wsme_pecan.wsexpose(None, types.uuid, status_code=HTTPStatus.NO_CONTENT)
     def delete(self, action_plan_uuid):
         """Delete an action plan.
 

watcher/api/controllers/v1/audit.py

@@ -32,6 +32,7 @@ states, visit :ref:`the Audit State machine <audit_state_machine>`.
 import datetime
 from dateutil import tz
 
+from http import HTTPStatus
 import pecan
 from pecan import rest
 import wsme
@@ -595,7 +596,8 @@ class AuditsController(rest.RestController):
 
         return Audit.convert_with_links(rpc_audit)
 
-    @wsme_pecan.wsexpose(Audit, body=AuditPostType, status_code=201)
+    @wsme_pecan.wsexpose(Audit, body=AuditPostType,
+                         status_code=HTTPStatus.CREATED)
     def post(self, audit_p):
         """Create a new audit.
 
@@ -717,7 +719,7 @@ class AuditsController(rest.RestController):
         audit_to_update.save()
         return Audit.convert_with_links(audit_to_update)
 
-    @wsme_pecan.wsexpose(None, wtypes.text, status_code=204)
+    @wsme_pecan.wsexpose(None, wtypes.text, status_code=HTTPStatus.NO_CONTENT)
     def delete(self, audit):
         """Delete an audit.
 

watcher/api/controllers/v1/audit_template.py

@@ -45,6 +45,7 @@ will be launched automatically or will need a manual confirmation from the
 
 import datetime
 
+from http import HTTPStatus
 import pecan
 from pecan import rest
 import wsme
@@ -618,7 +619,7 @@ class AuditTemplatesController(rest.RestController):
 
     @wsme.validate(types.uuid, AuditTemplatePostType)
     @wsme_pecan.wsexpose(AuditTemplate, body=AuditTemplatePostType,
-                         status_code=201)
+                         status_code=HTTPStatus.CREATED)
     def post(self, audit_template_postdata):
         """Create a new audit template.
 
@@ -694,7 +695,7 @@ class AuditTemplatesController(rest.RestController):
         audit_template_to_update.save()
         return AuditTemplate.convert_with_links(audit_template_to_update)
 
-    @wsme_pecan.wsexpose(None, wtypes.text, status_code=204)
+    @wsme_pecan.wsexpose(None, wtypes.text, status_code=HTTPStatus.NO_CONTENT)
     def delete(self, audit_template):
         """Delete a audit template.
 

watcher/api/controllers/v1/webhooks.py

@@ -14,6 +14,7 @@
 Webhook endpoint for Watcher v1 REST API.
 """
 
+from http import HTTPStatus
 from oslo_log import log
 import pecan
 from pecan import rest
@@ -36,7 +37,7 @@ class WebhookController(rest.RestController):
         self.dc_client = rpcapi.DecisionEngineAPI()
 
     @wsme_pecan.wsexpose(None, wtypes.text, body=types.jsontype,
-                         status_code=202)
+                         status_code=HTTPStatus.ACCEPTED)
     def post(self, audit_ident, body):
         """Trigger the given audit.
 

watcher/api/hooks.py

@@ -15,7 +15,7 @@
 # under the License.
 
 
-from http import client as http_client
+from http import HTTPStatus
 from oslo_config import cfg
 from pecan import hooks
 
@@ -91,8 +91,8 @@ class NoExceptionTracebackHook(hooks.PecanHook):
         # Do nothing if there is no error.
         # Status codes in the range 200 (OK) to 399 (400 = BAD_REQUEST) are not
         # an error.
-        if (http_client.OK <= state.response.status_int <
-                http_client.BAD_REQUEST):
+        if (HTTPStatus.OK <= state.response.status_int <
+                HTTPStatus.BAD_REQUEST):
             return
 
         json_body = state.response.json

watcher/applier/actions/change_node_power_state.py

@@ -17,17 +17,17 @@
 # limitations under the License.
 #
 
-import enum
 import time
 
+from oslo_log import log
+
 from watcher._i18n import _
 from watcher.applier.actions import base
 from watcher.common import exception
+from watcher.common.metal_helper import constants as metal_constants
+from watcher.common.metal_helper import factory as metal_helper_factory
 
-
-class NodeState(enum.Enum):
-    POWERON = 'on'
-    POWEROFF = 'off'
+LOG = log.getLogger(__name__)
 
 
 class ChangeNodePowerState(base.BaseAction):
@@ -43,8 +43,8 @@ class ChangeNodePowerState(base.BaseAction):
          'state': str,
         })
 
-    The `resource_id` references a ironic node id (list of available
-    ironic node is returned by this command: ``ironic node-list``).
+    The `resource_id` references a baremetal node id (list of available
+    ironic nodes is returned by this command: ``ironic node-list``).
     The `state` value should either be `on` or `off`.
     """
 
@@ -59,10 +59,14 @@ class ChangeNodePowerState(base.BaseAction):
                     'type': 'string',
                     "minlength": 1
                 },
+                'resource_name': {
+                    'type': 'string',
+                    "minlength": 1
+                },
                 'state': {
                     'type': 'string',
-                    'enum': [NodeState.POWERON.value,
-                             NodeState.POWEROFF.value]
+                    'enum': [metal_constants.PowerState.ON.value,
+                             metal_constants.PowerState.OFF.value]
                 }
             },
             'required': ['resource_id', 'state'],
@@ -82,10 +86,10 @@ class ChangeNodePowerState(base.BaseAction):
         return self._node_manage_power(target_state)
 
     def revert(self):
-        if self.state == NodeState.POWERON.value:
-            target_state = NodeState.POWEROFF.value
-        elif self.state == NodeState.POWEROFF.value:
-            target_state = NodeState.POWERON.value
+        if self.state == metal_constants.PowerState.ON.value:
+            target_state = metal_constants.PowerState.OFF.value
+        elif self.state == metal_constants.PowerState.OFF.value:
+            target_state = metal_constants.PowerState.ON.value
         return self._node_manage_power(target_state)
 
     def _node_manage_power(self, state, retry=60):
@@ -93,30 +97,32 @@ class ChangeNodePowerState(base.BaseAction):
             raise exception.IllegalArgumentException(
                 message=_("The target state is not defined"))
 
-        ironic_client = self.osc.ironic()
-        nova_client = self.osc.nova()
-        current_state = ironic_client.node.get(self.node_uuid).power_state
-        # power state: 'power on' or 'power off', if current node state
-        # is the same as state, just return True
-        if state in current_state:
+        metal_helper = metal_helper_factory.get_helper(self.osc)
+        node = metal_helper.get_node(self.node_uuid)
+        current_state = node.get_power_state()
+
+        if state == current_state.value:
             return True
 
-        if state == NodeState.POWEROFF.value:
-            node_info = ironic_client.node.get(self.node_uuid).to_dict()
-            compute_node_id = node_info['extra']['compute_node_id']
-            compute_node = nova_client.hypervisors.get(compute_node_id)
-            compute_node = compute_node.to_dict()
+        if state == metal_constants.PowerState.OFF.value:
+            compute_node = node.get_hypervisor_node().to_dict()
             if (compute_node['running_vms'] == 0):
-                ironic_client.node.set_power_state(
-                    self.node_uuid, state)
+                node.set_power_state(state)
+            else:
+                LOG.warning(
+                    "Compute node %s has %s running vms and will "
+                    "NOT be shut off.",
+                    compute_node["hypervisor_hostname"],
+                    compute_node['running_vms'])
+                return False
         else:
-            ironic_client.node.set_power_state(self.node_uuid, state)
+            node.set_power_state(state)
 
-        ironic_node = ironic_client.node.get(self.node_uuid)
-        while ironic_node.power_state == current_state and retry:
+        node = metal_helper.get_node(self.node_uuid)
+        while node.get_power_state() == current_state and retry:
             time.sleep(10)
             retry -= 1
-            ironic_node = ironic_client.node.get(self.node_uuid)
+            node = metal_helper.get_node(self.node_uuid)
         if retry > 0:
             return True
         else:
@@ -130,4 +136,4 @@ class ChangeNodePowerState(base.BaseAction):
 
     def get_description(self):
         """Description of the action"""
-        return ("Compute node power on/off through ironic.")
+        return ("Compute node power on/off through Ironic or MaaS.")

watcher/applier/actions/volume_migration.py

@@ -17,14 +17,11 @@ import jsonschema
 
 from oslo_log import log
 
-from cinderclient import client as cinder_client
 from watcher._i18n import _
 from watcher.applier.actions import base
 from watcher.common import cinder_helper
 from watcher.common import exception
-from watcher.common import keystone_helper
 from watcher.common import nova_helper
-from watcher.common import utils
 from watcher import conf
 
 CONF = conf.CONF
@@ -70,8 +67,6 @@ class VolumeMigrate(base.BaseAction):
 
     def __init__(self, config, osc=None):
         super(VolumeMigrate, self).__init__(config)
-        self.temp_username = utils.random_string(10)
-        self.temp_password = utils.random_string(10)
         self.cinder_util = cinder_helper.CinderHelper(osc=self.osc)
         self.nova_util = nova_helper.NovaHelper(osc=self.osc)
 
@@ -134,83 +129,42 @@ class VolumeMigrate(base.BaseAction):
 
     def _can_swap(self, volume):
         """Judge volume can be swapped"""
+        # TODO(sean-k-mooney): rename this to _can_migrate and update
+        # tests to reflect that.
 
+        # cinder volume migration can migrate volumes that are not
+        # attached to instances or nova can migrate the data for cinder
+        # if the volume is in-use. If the volume has no attachments
+        # allow cinder to decided if it can be migrated.
         if not volume.attachments:
-            return False
-        instance_id = volume.attachments[0]['server_id']
-        instance_status = self.nova_util.find_instance(instance_id).status
-
-        if (volume.status == 'in-use' and
-                instance_status in ('ACTIVE', 'PAUSED', 'RESIZED')):
+            LOG.debug(f"volume: {volume.id} has no attachments")
             return True
 
-        return False
-
-    def _create_user(self, volume, user):
-        """Create user with volume attribute and user information"""
-        keystone_util = keystone_helper.KeystoneHelper(osc=self.osc)
-        project_id = getattr(volume, 'os-vol-tenant-attr:tenant_id')
-        user['project'] = project_id
-        user['domain'] = keystone_util.get_project(project_id).domain_id
-        user['roles'] = ['admin']
-        return keystone_util.create_user(user)
-
-    def _get_cinder_client(self, session):
-        """Get cinder client by session"""
-        return cinder_client.Client(
-            CONF.cinder_client.api_version,
-            session=session,
-            endpoint_type=CONF.cinder_client.endpoint_type)
-
-    def _swap_volume(self, volume, dest_type):
-        """Swap volume to dest_type
-
-           Limitation note: only for compute libvirt driver
-        """
-        if not dest_type:
-            raise exception.Invalid(
-                message=(_("destination type is required when "
-                           "migration type is swap")))
-
-        if not self._can_swap(volume):
-            raise exception.Invalid(
-                message=(_("Invalid state for swapping volume")))
-
-        user_info = {
-            'name': self.temp_username,
-            'password': self.temp_password}
-        user = self._create_user(volume, user_info)
-        keystone_util = keystone_helper.KeystoneHelper(osc=self.osc)
-        try:
-            session = keystone_util.create_session(
-                user.id, self.temp_password)
-            temp_cinder = self._get_cinder_client(session)
-
-            # swap volume
-            new_volume = self.cinder_util.create_volume(
-                temp_cinder, volume, dest_type)
-            self.nova_util.swap_volume(volume, new_volume)
-
-            # delete old volume
-            self.cinder_util.delete_volume(volume)
-
-        finally:
-            keystone_util.delete_user(user)
-
-        return True
+        # since it has attachments we need to validate nova's constraints
+        instance_id = volume.attachments[0]['server_id']
+        instance_status = self.nova_util.find_instance(instance_id).status
+        LOG.debug(
+            f"volume: {volume.id} is attached to instance: {instance_id} "
+            f"in instance status: {instance_status}")
+        # NOTE(sean-k-mooney): This used to allow RESIZED which
+        # is the resize_verify task state, that is not an acceptable time
+        # to migrate volumes, if nova does not block this in the API
+        # today that is probably a bug. PAUSED is also questionable but
+        # it should generally be safe.
+        return (volume.status == 'in-use' and
+                instance_status in ('ACTIVE', 'PAUSED'))
 
     def _migrate(self, volume_id, dest_node, dest_type):
-
         try:
             volume = self.cinder_util.get_volume(volume_id)
-            if self.migration_type == self.SWAP:
-                if dest_node:
-                    LOG.warning("dest_node is ignored")
-                return self._swap_volume(volume, dest_type)
+            # for backward compatibility map swap to migrate.
+            if self.migration_type in (self.SWAP, self.MIGRATE):
+                if not self._can_swap(volume):
+                    raise exception.Invalid(
+                        message=(_("Invalid state for swapping volume")))
+                return self.cinder_util.migrate(volume, dest_node)
             elif self.migration_type == self.RETYPE:
                 return self.cinder_util.retype(volume, dest_type)
-            elif self.migration_type == self.MIGRATE:
-                return self.cinder_util.migrate(volume, dest_node)
             else:
                 raise exception.Invalid(
                     message=(_("Migration of type '%(migration_type)s' is not "

watcher/common/cinder_helper.py

@@ -17,7 +17,7 @@ import time
 from oslo_log import log
 
 from cinderclient import exceptions as cinder_exception
-from cinderclient.v2.volumes import Volume
+from cinderclient.v3.volumes import Volume
 from watcher._i18n import _
 from watcher.common import clients
 from watcher.common import exception

watcher/common/clients.py

@@ -25,6 +25,7 @@ from novaclient import api_versions as nova_api_versions
 from novaclient import client as nvclient
 
 from watcher.common import exception
+from watcher.common import utils
 
 try:
     from ceilometerclient import client as ceclient
@@ -32,6 +33,12 @@ try:
 except ImportError:
     HAS_CEILCLIENT = False
 
+try:
+    from maas import client as maas_client
+except ImportError:
+    maas_client = None
+
+
 CONF = cfg.CONF
 
 _CLIENTS_AUTH_GROUP = 'watcher_clients_auth'
@@ -74,6 +81,7 @@ class OpenStackClients(object):
         self._monasca = None
         self._neutron = None
         self._ironic = None
+        self._maas = None
         self._placement = None
 
     def _get_keystone_session(self):
@@ -265,6 +273,23 @@ class OpenStackClients(object):
                                            session=self.session)
         return self._ironic
 
+    def maas(self):
+        if self._maas:
+            return self._maas
+
+        if not maas_client:
+            raise exception.UnsupportedError(
+                "MAAS client unavailable. Please install python-libmaas.")
+
+        url = self._get_client_option('maas', 'url')
+        api_key = self._get_client_option('maas', 'api_key')
+        timeout = self._get_client_option('maas', 'timeout')
+        self._maas = utils.async_compat_call(
+            maas_client.connect,
+            url, apikey=api_key,
+            timeout=timeout)
+        return self._maas
+
     @exception.wrap_keystone_exception
     def placement(self):
         if self._placement:

watcher/common/context.py

@@ -11,12 +11,15 @@
 # under the License.
 
 from oslo_context import context
+from oslo_db.sqlalchemy import enginefacade
 from oslo_log import log
 from oslo_utils import timeutils
 
+
 LOG = log.getLogger(__name__)
 
 
+@enginefacade.transaction_context_provider
 class RequestContext(context.RequestContext):
     """Extends security contexts from the OpenStack common library."""
 

watcher/common/exception.py

@@ -25,6 +25,7 @@ SHOULD include dedicated exception logging.
 import functools
 import sys
 
+from http import HTTPStatus
 from keystoneclient import exceptions as keystone_exceptions
 from oslo_config import cfg
 from oslo_log import log
@@ -62,7 +63,7 @@ class WatcherException(Exception):
 
     """
     msg_fmt = _("An unknown exception occurred")
-    code = 500
+    code = HTTPStatus.INTERNAL_SERVER_ERROR
     headers = {}
     safe = False
 
@@ -114,12 +115,12 @@ class UnsupportedError(WatcherException):
 
 class NotAuthorized(WatcherException):
     msg_fmt = _("Not authorized")
-    code = 403
+    code = HTTPStatus.FORBIDDEN
 
 
 class NotAcceptable(WatcherException):
     msg_fmt = _("Request not acceptable.")
-    code = 406
+    code = HTTPStatus.NOT_ACCEPTABLE
 
 
 class PolicyNotAuthorized(NotAuthorized):
@@ -132,7 +133,7 @@ class OperationNotPermitted(NotAuthorized):
 
 class Invalid(WatcherException, ValueError):
     msg_fmt = _("Unacceptable parameters")
-    code = 400
+    code = HTTPStatus.BAD_REQUEST
 
 
 class ObjectNotFound(WatcherException):
@@ -141,12 +142,12 @@ class ObjectNotFound(WatcherException):
 
 class Conflict(WatcherException):
     msg_fmt = _('Conflict')
-    code = 409
+    code = HTTPStatus.CONFLICT
 
 
 class ResourceNotFound(ObjectNotFound):
     msg_fmt = _("The %(name)s resource %(id)s could not be found")
-    code = 404
+    code = HTTPStatus.NOT_FOUND
 
 
 class InvalidParameter(Invalid):

watcher/common/keystone_helper.py

@@ -15,8 +15,6 @@
 from oslo_log import log
 
 from keystoneauth1.exceptions import http as ks_exceptions
-from keystoneauth1 import loading
-from keystoneauth1 import session
 from watcher._i18n import _
 from watcher.common import clients
 from watcher.common import exception
@@ -90,35 +88,3 @@ class KeystoneHelper(object):
                     message=(_("Domain name seems ambiguous: %s") %
                              name_or_id))
             return domains[0]
-
-    def create_session(self, user_id, password):
-        user = self.get_user(user_id)
-        loader = loading.get_plugin_loader('password')
-        auth = loader.load_from_options(
-            auth_url=CONF.watcher_clients_auth.auth_url,
-            password=password,
-            user_id=user_id,
-            project_id=user.default_project_id)
-        return session.Session(auth=auth)
-
-    def create_user(self, user):
-        project = self.get_project(user['project'])
-        domain = self.get_domain(user['domain'])
-        _user = self.keystone.users.create(
-            user['name'],
-            password=user['password'],
-            domain=domain,
-            project=project,
-        )
-        for role in user['roles']:
-            role = self.get_role(role)
-            self.keystone.roles.grant(
-                role.id, user=_user.id, project=project.id)
-        return _user
-
-    def delete_user(self, user):
-        try:
-            user = self.get_user(user)
-            self.keystone.users.delete(user)
-        except exception.Invalid:
-            pass

watcher/common/metal_helper/base.py

@@ -0,0 +1,81 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+
+from watcher.common import exception
+from watcher.common.metal_helper import constants as metal_constants
+
+
+class BaseMetalNode(abc.ABC):
+    hv_up_when_powered_off = False
+
+    def __init__(self, nova_node=None):
+        self._nova_node = nova_node
+
+    def get_hypervisor_node(self):
+        if not self._nova_node:
+            raise exception.Invalid(message="No associated hypervisor.")
+        return self._nova_node
+
+    def get_hypervisor_hostname(self):
+        return self.get_hypervisor_node().hypervisor_hostname
+
+    @abc.abstractmethod
+    def get_power_state(self):
+        # TODO(lpetrut): document the following methods
+        pass
+
+    @abc.abstractmethod
+    def get_id(self):
+        """Return the node id provided by the bare metal service."""
+        pass
+
+    @abc.abstractmethod
+    def power_on(self):
+        pass
+
+    @abc.abstractmethod
+    def power_off(self):
+        pass
+
+    def set_power_state(self, state):
+        state = metal_constants.PowerState(state)
+        if state == metal_constants.PowerState.ON:
+            self.power_on()
+        elif state == metal_constants.PowerState.OFF:
+            self.power_off()
+        else:
+            raise exception.UnsupportedActionType(
+                "Cannot set power state: %s" % state)
+
+
+class BaseMetalHelper(abc.ABC):
+    def __init__(self, osc):
+        self._osc = osc
+
+    @property
+    def nova_client(self):
+        if not getattr(self, "_nova_client", None):
+            self._nova_client = self._osc.nova()
+        return self._nova_client
+
+    @abc.abstractmethod
+    def list_compute_nodes(self):
+        pass
+
+    @abc.abstractmethod
+    def get_node(self, node_id):
+        pass

watcher/common/metal_helper/constants.py

@@ -0,0 +1,23 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import enum
+
+
+class PowerState(str, enum.Enum):
+    ON = "on"
+    OFF = "off"
+    UNKNOWN = "unknown"
+    ERROR = "error"

watcher/common/metal_helper/factory.py

@@ -0,0 +1,33 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from watcher.common import clients
+from watcher.common.metal_helper import ironic
+from watcher.common.metal_helper import maas
+
+CONF = cfg.CONF
+
+
+def get_helper(osc=None):
+    # TODO(lpetrut): consider caching this client.
+    if not osc:
+        osc = clients.OpenStackClients()
+
+    if CONF.maas_client.url:
+        return maas.MaasHelper(osc)
+    else:
+        return ironic.IronicHelper(osc)

watcher/common/metal_helper/ironic.py

@@ -0,0 +1,94 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log
+
+from watcher.common.metal_helper import base
+from watcher.common.metal_helper import constants as metal_constants
+
+LOG = log.getLogger(__name__)
+
+POWER_STATES_MAP = {
+    'power on': metal_constants.PowerState.ON,
+    'power off': metal_constants.PowerState.OFF,
+    # For now, we only use ON/OFF states
+    'rebooting': metal_constants.PowerState.ON,
+    'soft power off': metal_constants.PowerState.OFF,
+    'soft reboot': metal_constants.PowerState.ON,
+}
+
+
+class IronicNode(base.BaseMetalNode):
+    hv_up_when_powered_off = True
+
+    def __init__(self, ironic_node, nova_node, ironic_client):
+        super().__init__(nova_node)
+
+        self._ironic_client = ironic_client
+        self._ironic_node = ironic_node
+
+    def get_power_state(self):
+        return POWER_STATES_MAP.get(self._ironic_node.power_state,
+                                    metal_constants.PowerState.UNKNOWN)
+
+    def get_id(self):
+        return self._ironic_node.uuid
+
+    def power_on(self):
+        self._ironic_client.node.set_power_state(self.get_id(), "on")
+
+    def power_off(self):
+        self._ironic_client.node.set_power_state(self.get_id(), "off")
+
+
+class IronicHelper(base.BaseMetalHelper):
+    @property
+    def _client(self):
+        if not getattr(self, "_cached_client", None):
+            self._cached_client = self._osc.ironic()
+        return self._cached_client
+
+    def list_compute_nodes(self):
+        out_list = []
+        # TODO(lpetrut): consider using "detailed=True" instead of making
+        # an additional GET request per node
+        node_list = self._client.node.list()
+
+        for node in node_list:
+            node_info = self._client.node.get(node.uuid)
+            hypervisor_id = node_info.extra.get('compute_node_id', None)
+            if hypervisor_id is None:
+                LOG.warning('Cannot find compute_node_id in extra '
+                            'of ironic node %s', node.uuid)
+                continue
+
+            hypervisor_node = self.nova_client.hypervisors.get(hypervisor_id)
+            if hypervisor_node is None:
+                LOG.warning('Cannot find hypervisor %s', hypervisor_id)
+                continue
+
+            out_node = IronicNode(node, hypervisor_node, self._client)
+            out_list.append(out_node)
+
+        return out_list
+
+    def get_node(self, node_id):
+        ironic_node = self._client.node.get(node_id)
+        compute_node_id = ironic_node.extra.get('compute_node_id')
+        if compute_node_id:
+            compute_node = self.nova_client.hypervisors.get(compute_node_id)
+        else:
+            compute_node = None
+        return IronicNode(ironic_node, compute_node, self._client)

watcher/common/metal_helper/maas.py

@@ -0,0 +1,125 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+
+from watcher.common import exception
+from watcher.common.metal_helper import base
+from watcher.common.metal_helper import constants as metal_constants
+from watcher.common import utils
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+try:
+    from maas.client import enum as maas_enum
+except ImportError:
+    maas_enum = None
+
+
+class MaasNode(base.BaseMetalNode):
+    hv_up_when_powered_off = False
+
+    def __init__(self, maas_node, nova_node, maas_client):
+        super().__init__(nova_node)
+
+        self._maas_client = maas_client
+        self._maas_node = maas_node
+
+    def get_power_state(self):
+        maas_state = utils.async_compat_call(
+            self._maas_node.query_power_state,
+            timeout=CONF.maas_client.timeout)
+
+        # python-libmaas may not be available, so we'll avoid a global
+        # variable.
+        power_states_map = {
+            maas_enum.PowerState.ON: metal_constants.PowerState.ON,
+            maas_enum.PowerState.OFF: metal_constants.PowerState.OFF,
+            maas_enum.PowerState.ERROR: metal_constants.PowerState.ERROR,
+            maas_enum.PowerState.UNKNOWN: metal_constants.PowerState.UNKNOWN,
+        }
+        return power_states_map.get(maas_state,
+                                    metal_constants.PowerState.UNKNOWN)
+
+    def get_id(self):
+        return self._maas_node.system_id
+
+    def power_on(self):
+        LOG.info("Powering on MAAS node: %s %s",
+                 self._maas_node.fqdn,
+                 self._maas_node.system_id)
+        utils.async_compat_call(
+            self._maas_node.power_on,
+            timeout=CONF.maas_client.timeout)
+
+    def power_off(self):
+        LOG.info("Powering off MAAS node: %s %s",
+                 self._maas_node.fqdn,
+                 self._maas_node.system_id)
+        utils.async_compat_call(
+            self._maas_node.power_off,
+            timeout=CONF.maas_client.timeout)
+
+
+class MaasHelper(base.BaseMetalHelper):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not maas_enum:
+            raise exception.UnsupportedError(
+                "MAAS client unavailable. Please install python-libmaas.")
+
+    @property
+    def _client(self):
+        if not getattr(self, "_cached_client", None):
+            self._cached_client = self._osc.maas()
+        return self._cached_client
+
+    def list_compute_nodes(self):
+        out_list = []
+        node_list = utils.async_compat_call(
+            self._client.machines.list,
+            timeout=CONF.maas_client.timeout)
+
+        compute_nodes = self.nova_client.hypervisors.list()
+        compute_node_map = dict()
+        for compute_node in compute_nodes:
+            compute_node_map[compute_node.hypervisor_hostname] = compute_node
+
+        for node in node_list:
+            hypervisor_node = compute_node_map.get(node.fqdn)
+            if not hypervisor_node:
+                LOG.info('Cannot find hypervisor %s', node.fqdn)
+                continue
+
+            out_node = MaasNode(node, hypervisor_node, self._client)
+            out_list.append(out_node)
+
+        return out_list
+
+    def _get_compute_node_by_hostname(self, hostname):
+        compute_nodes = self.nova_client.hypervisors.search(
+            hostname, detailed=True)
+        for compute_node in compute_nodes:
+            if compute_node.hypervisor_hostname == hostname:
+                return compute_node
+
+    def get_node(self, node_id):
+        maas_node = utils.async_compat_call(
+            self._client.machines.get, node_id,
+            timeout=CONF.maas_client.timeout)
+        compute_node = self._get_compute_node_by_hostname(maas_node.fqdn)
+        return MaasNode(maas_node, compute_node, self._client)

watcher/common/placement_helper.py

@@ -11,6 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_log import log as logging
 
@@ -53,7 +54,7 @@ class PlacementHelper(object):
         if rp_name:
             url += '?name=%s' % rp_name
         resp = self.get(url)
-        if resp.status_code == 200:
+        if resp.status_code == HTTPStatus.OK:
             json_resp = resp.json()
             return json_resp['resource_providers']
 
@@ -77,7 +78,7 @@ class PlacementHelper(object):
         """
         url = '/resource_providers/%s/inventories' % rp_uuid
         resp = self.get(url)
-        if resp.status_code == 200:
+        if resp.status_code == HTTPStatus.OK:
             json = resp.json()
             return json['inventories']
         msg = ("Failed to get resource provider %(rp_uuid)s inventories. "
@@ -97,7 +98,7 @@ class PlacementHelper(object):
         """
         resp = self.get("/resource_providers/%s/traits" % rp_uuid)
 
-        if resp.status_code == 200:
+        if resp.status_code == HTTPStatus.OK:
             json = resp.json()
             return json['traits']
         msg = ("Failed to get resource provider %(rp_uuid)s traits. "
@@ -118,7 +119,7 @@ class PlacementHelper(object):
         """
         url = '/allocations/%s' % consumer_uuid
         resp = self.get(url)
-        if resp.status_code == 200:
+        if resp.status_code == HTTPStatus.OK:
             json = resp.json()
             return json['allocations']
         msg = ("Failed to get allocations for consumer %(c_uuid). "
@@ -139,7 +140,7 @@ class PlacementHelper(object):
         """
         url = '/resource_providers/%s/usages' % rp_uuid
         resp = self.get(url)
-        if resp.status_code == 200:
+        if resp.status_code == HTTPStatus.OK:
             json = resp.json()
             return json['usages']
         msg = ("Failed to get resource provider %(rp_uuid)s usages. "
@@ -164,7 +165,7 @@ class PlacementHelper(object):
         """
         url = "/allocation_candidates?%s" % resources
         resp = self.get(url)
-        if resp.status_code == 200:
+        if resp.status_code == HTTPStatus.OK:
             data = resp.json()
             return data['provider_summaries']
 

watcher/common/rpc.py

@@ -121,7 +121,7 @@ class RequestContextSerializer(messaging.Serializer):
 def get_client(target, version_cap=None, serializer=None):
     assert TRANSPORT is not None
     serializer = RequestContextSerializer(serializer)
-    return messaging.RPCClient(
+    return messaging.get_rpc_client(
         TRANSPORT,
         target,
         version_cap=version_cap,

watcher/common/utils.py

@@ -16,12 +16,14 @@
 
 """Utilities and helper functions."""
 
+import asyncio
 import datetime
-import random
+import inspect
 import re
-import string
 
 from croniter import croniter
+import eventlet
+from eventlet import tpool
 
 from jsonschema import validators
 from oslo_config import cfg
@@ -156,9 +158,39 @@ def extend_with_strict_schema(validator_class):
 StrictDefaultValidatingDraft4Validator = extend_with_default(
     extend_with_strict_schema(validators.Draft4Validator))
 
+
 Draft4Validator = validators.Draft4Validator
 
 
-def random_string(n):
-    return ''.join([random.choice(
-        string.ascii_letters + string.digits) for i in range(n)])
+# Some clients (e.g. MAAS) use asyncio, which isn't compatible with Eventlet.
+# As a workaround, we're delegating such calls to a native thread.
+def async_compat_call(f, *args, **kwargs):
+    timeout = kwargs.pop('timeout', None)
+
+    async def async_wrapper():
+        ret = f(*args, **kwargs)
+        if inspect.isawaitable(ret):
+            return await asyncio.wait_for(ret, timeout)
+        return ret
+
+    def tpool_wrapper():
+        # This will run in a separate native thread. Ideally, there should be
+        # a single thread permanently running an asyncio loop, but for
+        # convenience we'll use eventlet.tpool, which leverages a thread pool.
+        #
+        # That being considered, we're setting up a temporary asyncio loop to
+        # handle this call.
+        loop = asyncio.new_event_loop()
+        try:
+            asyncio.set_event_loop(loop)
+            return loop.run_until_complete(async_wrapper())
+        finally:
+            loop.close()
+
+    # We'll use eventlet timeouts as an extra precaution and asyncio timeouts
+    # to avoid lingering threads. For consistency, we'll convert eventlet
+    # timeout exceptions to asyncio timeout errors.
+    with eventlet.timeout.Timeout(
+            seconds=timeout,
+            exception=asyncio.TimeoutError("Timeout: %ss" % timeout)):
+        return tpool.execute(tpool_wrapper)

watcher/conf/__init__.py

@@ -35,6 +35,7 @@ from watcher.conf import grafana_client
 from watcher.conf import grafana_translators
 from watcher.conf import ironic_client
 from watcher.conf import keystone_client
+from watcher.conf import maas_client
 from watcher.conf import monasca_client
 from watcher.conf import neutron_client
 from watcher.conf import nova_client
@@ -54,6 +55,7 @@ db.register_opts(CONF)
 planner.register_opts(CONF)
 applier.register_opts(CONF)
 decision_engine.register_opts(CONF)
+maas_client.register_opts(CONF)
 monasca_client.register_opts(CONF)
 nova_client.register_opts(CONF)
 glance_client.register_opts(CONF)

watcher/conf/grafana_client.py

@@ -134,7 +134,13 @@ GRAFANA_CLIENT_OPTS = [
                      "InfluxDB this will be the retention period. "
                      "These queries will need to be constructed using tools "
                      "such as Postman. Example: SELECT cpu FROM {4}."
-                     "cpu_percent WHERE host == '{1}' AND time > now()-{2}s")]
+                     "cpu_percent WHERE host == '{1}' AND time > now()-{2}s"),
+    cfg.IntOpt('http_timeout',
+               min=0,
+               default=60,
+               mutable=True,
+               help='Timeout for Grafana request')
+]
 
 
 def register_opts(conf):

watcher/conf/maas_client.py

@@ -0,0 +1,38 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslo_config import cfg
+
+maas_client = cfg.OptGroup(name='maas_client',
+                           title='Configuration Options for MaaS')
+
+MAAS_CLIENT_OPTS = [
+    cfg.StrOpt('url',
+               help='MaaS URL, example: http://1.2.3.4:5240/MAAS'),
+    cfg.StrOpt('api_key',
+               help='MaaS API authentication key.'),
+    cfg.IntOpt('timeout',
+               default=60,
+               help='MaaS client operation timeout in seconds.')]
+
+
+def register_opts(conf):
+    conf.register_group(maas_client)
+    conf.register_opts(MAAS_CLIENT_OPTS, group=maas_client)
+
+
+def list_opts():
+    return [(maas_client, MAAS_CLIENT_OPTS)]

watcher/db/sqlalchemy/alembic/env.py

@@ -13,8 +13,8 @@
 from logging import config as log_config
 
 from alembic import context
+from oslo_db.sqlalchemy import enginefacade
 
-from watcher.db.sqlalchemy import api as sqla_api
 from watcher.db.sqlalchemy import models
 
 # this is the Alembic Config object, which provides
@@ -43,7 +43,7 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    engine = sqla_api.get_engine()
+    engine = enginefacade.writer.get_engine()
     with engine.connect() as connection:
         context.configure(connection=connection,
                           target_metadata=target_metadata)

watcher/db/sqlalchemy/alembic/versions/0f6042416884_add_apscheduler_jobs.py

@@ -6,6 +6,7 @@ Create Date: 2017-03-24 11:21:29.036532
 
 """
 from alembic import op
+from sqlalchemy import inspect
 import sqlalchemy as sa
 
 from watcher.db.sqlalchemy import models
@@ -14,11 +15,20 @@ from watcher.db.sqlalchemy import models
 revision = '0f6042416884'
 down_revision = '001'
 
+def _table_exists(table_name):
+    bind = op.get_context().bind
+    insp = inspect(bind)
+    names = insp.get_table_names()
+    return any(t == table_name for t in names)
+
 
 def upgrade():
+    if _table_exists('apscheduler_jobs'):
+        return
+
     op.create_table(
         'apscheduler_jobs',
-        sa.Column('id', sa.Unicode(191, _warn_on_bytestring=False),
+        sa.Column('id', sa.Unicode(191),
                   nullable=False),
         sa.Column('next_run_time', sa.Float(25), index=True),
         sa.Column('job_state', sa.LargeBinary, nullable=False),

watcher/db/sqlalchemy/api.py

@@ -19,10 +19,12 @@
 import collections
 import datetime
 import operator
+import threading
 
 from oslo_config import cfg
+from oslo_db import api as oslo_db_api
 from oslo_db import exception as db_exc
-from oslo_db.sqlalchemy import session as db_session
+from oslo_db.sqlalchemy import enginefacade
 from oslo_db.sqlalchemy import utils as db_utils
 from oslo_utils import timeutils
 from sqlalchemy.inspection import inspect
@@ -38,24 +40,7 @@ from watcher import objects
 
 CONF = cfg.CONF
 
-_FACADE = None
-
-
-def _create_facade_lazily():
-    global _FACADE
-    if _FACADE is None:
-        _FACADE = db_session.EngineFacade.from_config(CONF)
-    return _FACADE
-
-
-def get_engine():
-    facade = _create_facade_lazily()
-    return facade.get_engine()
-
-
-def get_session(**kwargs):
-    facade = _create_facade_lazily()
-    return facade.get_session(**kwargs)
+_CONTEXT = threading.local()
 
 
 def get_backend():
@@ -63,14 +48,15 @@ def get_backend():
     return Connection()
 
 
-def model_query(model, *args, **kwargs):
-    """Query helper for simpler session usage.
+def _session_for_read():
+    return enginefacade.reader.using(_CONTEXT)
 
-    :param session: if present, the session to use
-    """
-    session = kwargs.get('session') or get_session()
-    query = session.query(model, *args)
-    return query
+
+# NOTE(tylerchristie) Please add @oslo_db_api.retry_on_deadlock decorator to
+# any new methods using _session_for_write (as deadlocks happen on write), so
+# that oslo_db is able to retry in case of deadlocks.
+def _session_for_write():
+    return enginefacade.writer.using(_CONTEXT)
 
 
 def add_identity_filter(query, value):
@@ -93,8 +79,6 @@ def add_identity_filter(query, value):
 
 def _paginate_query(model, limit=None, marker=None, sort_key=None,
                     sort_dir=None, query=None):
-    if not query:
-        query = model_query(model)
     sort_keys = ['id']
     if sort_key and sort_key not in sort_keys:
         sort_keys.insert(0, sort_key)
@@ -244,52 +228,58 @@ class Connection(api.BaseConnection):
         for relationship in relationships:
             if not relationship.uselist:
                 # We have a One-to-X relationship
-                query = query.options(joinedload(relationship.key))
+                query = query.options(joinedload(
+                    getattr(model, relationship.key)))
         return query
 
+    @oslo_db_api.retry_on_deadlock
     def _create(self, model, values):
-        obj = model()
-        cleaned_values = {k: v for k, v in values.items()
-                          if k not in self._get_relationships(model)}
-        obj.update(cleaned_values)
-        obj.save()
-        return obj
+        with _session_for_write() as session:
+            obj = model()
+            cleaned_values = {k: v for k, v in values.items()
+                              if k not in self._get_relationships(model)}
+            obj.update(cleaned_values)
+            session.add(obj)
+            session.flush()
+            return obj
 
     def _get(self, context, model, fieldname, value, eager):
-        query = model_query(model)
-        if eager:
-            query = self._set_eager_options(model, query)
+        with _session_for_read() as session:
+            query = session.query(model)
+            if eager:
+                query = self._set_eager_options(model, query)
 
-        query = query.filter(getattr(model, fieldname) == value)
-        if not context.show_deleted:
-            query = query.filter(model.deleted_at.is_(None))
+            query = query.filter(getattr(model, fieldname) == value)
+            if not context.show_deleted:
+                query = query.filter(model.deleted_at.is_(None))
 
-        try:
-            obj = query.one()
-        except exc.NoResultFound:
-            raise exception.ResourceNotFound(name=model.__name__, id=value)
+            try:
+                obj = query.one()
+            except exc.NoResultFound:
+                raise exception.ResourceNotFound(name=model.__name__, id=value)
 
-        return obj
+            return obj
 
     @staticmethod
+    @oslo_db_api.retry_on_deadlock
     def _update(model, id_, values):
-        session = get_session()
-        with session.begin():
-            query = model_query(model, session=session)
+        with _session_for_write() as session:
+            query = session.query(model)
             query = add_identity_filter(query, id_)
             try:
-                ref = query.with_lockmode('update').one()
+                ref = query.with_for_update().one()
             except exc.NoResultFound:
                 raise exception.ResourceNotFound(name=model.__name__, id=id_)
 
             ref.update(values)
-        return ref
+
+            return ref
 
     @staticmethod
+    @oslo_db_api.retry_on_deadlock
     def _soft_delete(model, id_):
-        session = get_session()
-        with session.begin():
-            query = model_query(model, session=session)
+        with _session_for_write() as session:
+            query = session.query(model)
             query = add_identity_filter(query, id_)
             try:
                 row = query.one()
@@ -301,10 +291,10 @@ class Connection(api.BaseConnection):
             return row
 
     @staticmethod
+    @oslo_db_api.retry_on_deadlock
     def _destroy(model, id_):
-        session = get_session()
-        with session.begin():
-            query = model_query(model, session=session)
+        with _session_for_write() as session:
+            query = session.query(model)
             query = add_identity_filter(query, id_)
 
             try:
@@ -317,14 +307,15 @@ class Connection(api.BaseConnection):
     def _get_model_list(self, model, add_filters_func, context, filters=None,
                         limit=None, marker=None, sort_key=None, sort_dir=None,
                         eager=False):
-        query = model_query(model)
-        if eager:
-            query = self._set_eager_options(model, query)
-        query = add_filters_func(query, filters)
-        if not context.show_deleted:
-            query = query.filter(model.deleted_at.is_(None))
-        return _paginate_query(model, limit, marker,
-                               sort_key, sort_dir, query)
+        with _session_for_read() as session:
+            query = session.query(model)
+            if eager:
+                query = self._set_eager_options(model, query)
+            query = add_filters_func(query, filters)
+            if not context.show_deleted:
+                query = query.filter(model.deleted_at.is_(None))
+            return _paginate_query(model, limit, marker,
+                                   sort_key, sort_dir, query)
 
     # NOTE(erakli): _add_..._filters methods should be refactored to have same
     # content. join_fieldmap should be filled with JoinMap instead of dict
@@ -419,11 +410,12 @@ class Connection(api.BaseConnection):
             plain_fields=plain_fields, join_fieldmap=join_fieldmap)
 
         if 'audit_uuid' in filters:
-            stmt = model_query(models.ActionPlan).join(
-                models.Audit,
-                models.Audit.id == models.ActionPlan.audit_id)\
-                .filter_by(uuid=filters['audit_uuid']).subquery()
-            query = query.filter_by(action_plan_id=stmt.c.id)
+            with _session_for_read() as session:
+                stmt = session.query(models.ActionPlan).join(
+                    models.Audit,
+                    models.Audit.id == models.ActionPlan.audit_id)\
+                    .filter_by(uuid=filters['audit_uuid']).subquery()
+                query = query.filter_by(action_plan_id=stmt.c.id)
 
         return query
 
@@ -601,20 +593,21 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        query = model_query(models.AuditTemplate)
-        query = query.filter_by(name=values.get('name'),
-                                deleted_at=None)
+        with _session_for_write() as session:
+            query = session.query(models.AuditTemplate)
+            query = query.filter_by(name=values.get('name'),
+                                    deleted_at=None)
 
-        if len(query.all()) > 0:
-            raise exception.AuditTemplateAlreadyExists(
-                audit_template=values['name'])
+            if len(query.all()) > 0:
+                raise exception.AuditTemplateAlreadyExists(
+                    audit_template=values['name'])
 
-        try:
-            audit_template = self._create(models.AuditTemplate, values)
-        except db_exc.DBDuplicateEntry:
-            raise exception.AuditTemplateAlreadyExists(
-                audit_template=values['name'])
-        return audit_template
+            try:
+                audit_template = self._create(models.AuditTemplate, values)
+            except db_exc.DBDuplicateEntry:
+                raise exception.AuditTemplateAlreadyExists(
+                    audit_template=values['name'])
+            return audit_template
 
     def _get_audit_template(self, context, fieldname, value, eager):
         try:
@@ -676,25 +669,26 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        query = model_query(models.Audit)
-        query = query.filter_by(name=values.get('name'),
-                                deleted_at=None)
+        with _session_for_write() as session:
+            query = session.query(models.Audit)
+            query = query.filter_by(name=values.get('name'),
+                                    deleted_at=None)
 
-        if len(query.all()) > 0:
-            raise exception.AuditAlreadyExists(
-                audit=values['name'])
+            if len(query.all()) > 0:
+                raise exception.AuditAlreadyExists(
+                    audit=values['name'])
 
-        if values.get('state') is None:
-            values['state'] = objects.audit.State.PENDING
+            if values.get('state') is None:
+                values['state'] = objects.audit.State.PENDING
 
-        if not values.get('auto_trigger'):
-            values['auto_trigger'] = False
+            if not values.get('auto_trigger'):
+                values['auto_trigger'] = False
 
-        try:
-            audit = self._create(models.Audit, values)
-        except db_exc.DBDuplicateEntry:
-            raise exception.AuditAlreadyExists(audit=values['uuid'])
-        return audit
+            try:
+                audit = self._create(models.Audit, values)
+            except db_exc.DBDuplicateEntry:
+                raise exception.AuditAlreadyExists(audit=values['uuid'])
+            return audit
 
     def _get_audit(self, context, fieldname, value, eager):
         try:
@@ -718,14 +712,13 @@ class Connection(api.BaseConnection):
     def destroy_audit(self, audit_id):
         def is_audit_referenced(session, audit_id):
             """Checks whether the audit is referenced by action_plan(s)."""
-            query = model_query(models.ActionPlan, session=session)
+            query = session.query(models.ActionPlan)
             query = self._add_action_plans_filters(
                 query, {'audit_id': audit_id})
             return query.count() != 0
 
-        session = get_session()
-        with session.begin():
-            query = model_query(models.Audit, session=session)
+        with _session_for_write() as session:
+            query = session.query(models.Audit)
             query = add_identity_filter(query, audit_id)
 
             try:
@@ -792,9 +785,8 @@ class Connection(api.BaseConnection):
             context, fieldname="uuid", value=action_uuid, eager=eager)
 
     def destroy_action(self, action_id):
-        session = get_session()
-        with session.begin():
-            query = model_query(models.Action, session=session)
+        with _session_for_write() as session:
+            query = session.query(models.Action)
             query = add_identity_filter(query, action_id)
             count = query.delete()
             if count != 1:
@@ -810,17 +802,16 @@ class Connection(api.BaseConnection):
 
     @staticmethod
     def _do_update_action(action_id, values):
-        session = get_session()
-        with session.begin():
-            query = model_query(models.Action, session=session)
+        with _session_for_write() as session:
+            query = session.query(models.Action)
             query = add_identity_filter(query, action_id)
             try:
-                ref = query.with_lockmode('update').one()
+                ref = query.with_for_update().one()
             except exc.NoResultFound:
                 raise exception.ActionNotFound(action=action_id)
 
             ref.update(values)
-        return ref
+            return ref
 
     def soft_delete_action(self, action_id):
         try:
@@ -864,14 +855,13 @@ class Connection(api.BaseConnection):
     def destroy_action_plan(self, action_plan_id):
         def is_action_plan_referenced(session, action_plan_id):
             """Checks whether the action_plan is referenced by action(s)."""
-            query = model_query(models.Action, session=session)
+            query = session.query(models.Action)
             query = self._add_actions_filters(
                 query, {'action_plan_id': action_plan_id})
             return query.count() != 0
 
-        session = get_session()
-        with session.begin():
-            query = model_query(models.ActionPlan, session=session)
+        with _session_for_write() as session:
+            query = session.query(models.ActionPlan)
             query = add_identity_filter(query, action_plan_id)
 
             try:
@@ -895,17 +885,16 @@ class Connection(api.BaseConnection):
 
     @staticmethod
     def _do_update_action_plan(action_plan_id, values):
-        session = get_session()
-        with session.begin():
-            query = model_query(models.ActionPlan, session=session)
+        with _session_for_write() as session:
+            query = session.query(models.ActionPlan)
             query = add_identity_filter(query, action_plan_id)
             try:
-                ref = query.with_lockmode('update').one()
+                ref = query.with_for_update().one()
             except exc.NoResultFound:
                 raise exception.ActionPlanNotFound(action_plan=action_plan_id)
 
             ref.update(values)
-        return ref
+            return ref
 
     def soft_delete_action_plan(self, action_plan_id):
         try:

watcher/db/sqlalchemy/job_store.py

@@ -22,6 +22,7 @@ from apscheduler.jobstores.base import ConflictingIdError
 from apscheduler.jobstores import sqlalchemy
 from apscheduler.util import datetime_to_utc_timestamp
 from apscheduler.util import maybe_ref
+from apscheduler.util import utc_timestamp_to_datetime
 
 from watcher.common import context
 from watcher.common import service
@@ -32,7 +33,7 @@ try:
 except ImportError:  # pragma: nocover
     import pickle
 
-from sqlalchemy import Table, MetaData, select, and_
+from sqlalchemy import Table, MetaData, select, and_, null
 from sqlalchemy.exc import IntegrityError
 
 
@@ -58,8 +59,7 @@ class WatcherJobStore(sqlalchemy.SQLAlchemyJobStore):
         super(WatcherJobStore, self).__init__(url, engine, tablename,
                                               metadata, pickle_protocol)
         metadata = maybe_ref(metadata) or MetaData()
-        self.jobs_t = Table(tablename, metadata, autoload=True,
-                            autoload_with=engine)
+        self.jobs_t = Table(tablename, metadata, autoload_with=engine)
         service_ident = service.ServiceHeartbeat.get_service_name()
         self.tag = tag or {'host': service_ident[0], 'name': service_ident[1]}
         self.service_id = objects.Service.list(context=context.make_context(),
@@ -79,7 +79,8 @@ class WatcherJobStore(sqlalchemy.SQLAlchemyJobStore):
             'tag': jsonutils.dumps(self.tag)
         })
         try:
-            self.engine.execute(insert)
+            with self.engine.begin() as conn:
+                conn.execute(insert)
         except IntegrityError:
             raise ConflictingIdError(job.id)
 
@@ -88,20 +89,36 @@ class WatcherJobStore(sqlalchemy.SQLAlchemyJobStore):
         self._fix_paused_jobs_sorting(jobs)
         return jobs
 
+    def get_next_run_time(self):
+        selectable = select(self.jobs_t.c.next_run_time).\
+            where(self.jobs_t.c.next_run_time != null()).\
+            order_by(self.jobs_t.c.next_run_time).limit(1)
+        with self.engine.begin() as connection:
+            # NOTE(danms): The apscheduler implementation of this gets a
+            # decimal.Decimal back from scalar() which causes
+            # utc_timestamp_to_datetime() to choke since it is expecting a
+            # python float. Assume this is SQLAlchemy 2.0 stuff, so just
+            # coerce to a float here.
+            next_run_time = connection.execute(selectable).scalar()
+            return utc_timestamp_to_datetime(float(next_run_time)
+                                             if next_run_time is not None
+                                             else None)
+
     def _get_jobs(self, *conditions):
         jobs = []
         conditions += (self.jobs_t.c.service_id == self.service_id,)
         selectable = select(
-            [self.jobs_t.c.id, self.jobs_t.c.job_state, self.jobs_t.c.tag]
+            self.jobs_t.c.id, self.jobs_t.c.job_state, self.jobs_t.c.tag
         ).order_by(self.jobs_t.c.next_run_time).where(and_(*conditions))
         failed_job_ids = set()
-        for row in self.engine.execute(selectable):
-            try:
-                jobs.append(self._reconstitute_job(row.job_state))
-            except Exception:
-                self._logger.exception(
-                    'Unable to restore job "%s" -- removing it', row.id)
-                failed_job_ids.add(row.id)
+        with self.engine.begin() as conn:
+            for row in conn.execute(selectable):
+                try:
+                    jobs.append(self._reconstitute_job(row.job_state))
+                except Exception:
+                    self._logger.exception(
+                        'Unable to restore job "%s" -- removing it', row.id)
+                    failed_job_ids.add(row.id)
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:

watcher/db/sqlalchemy/migration.py

@@ -20,9 +20,9 @@ import alembic
 from alembic import config as alembic_config
 import alembic.migration as alembic_migration
 from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import enginefacade
 
 from watcher._i18n import _
-from watcher.db.sqlalchemy import api as sqla_api
 from watcher.db.sqlalchemy import models
 
 
@@ -39,7 +39,7 @@ def version(engine=None):
     :rtype: string
     """
     if engine is None:
-        engine = sqla_api.get_engine()
+        engine = enginefacade.reader.get_engine()
     with engine.connect() as conn:
         context = alembic_migration.MigrationContext.configure(conn)
         return context.get_current_revision()
@@ -63,7 +63,7 @@ def create_schema(config=None, engine=None):
     Can be used for initial installation instead of upgrade('head').
     """
     if engine is None:
-        engine = sqla_api.get_engine()
+        engine = enginefacade.writer.get_engine()
 
     # NOTE(viktors): If we will use metadata.create_all() for non empty db
     #                schema, it will only add the new tables, but leave

watcher/db/sqlalchemy/models.py

@@ -93,14 +93,6 @@ class WatcherBase(models.SoftDeleteMixin,
             d[c.name] = self[c.name]
         return d
 
-    def save(self, session=None):
-        import watcher.db.sqlalchemy.api as db_api
-
-        if session is None:
-            session = db_api.get_session()
-
-        super(WatcherBase, self).save(session)
-
 
 Base = declarative_base(cls=WatcherBase)
 

watcher/decision_engine/audit/continuous.py

@@ -52,7 +52,7 @@ class ContinuousAuditHandler(base.AuditHandler):
             self._audit_scheduler = scheduling.BackgroundSchedulerService(
                 jobstores={
                     'default': job_store.WatcherJobStore(
-                        engine=sq_api.get_engine()),
+                        engine=sq_api.enginefacade.writer.get_engine()),
                 }
             )
         return self._audit_scheduler

watcher/decision_engine/datasources/base.py

@@ -63,7 +63,7 @@ class DataSourceBase(object):
             raise exception.MetricNotAvailable(metric=meter_name)
         return meter
 
-    def query_retry(self, f, *args, **kwargs):
+    def query_retry(self, f, *args, ignored_exc=None, **kwargs):
         """Attempts to retrieve metrics from the external service
 
         Attempts to access data from the external service and handles
@@ -71,15 +71,23 @@ class DataSourceBase(object):
         to the value of query_max_retries
         :param f: The method that performs the actual querying for metrics
         :param args: Array of arguments supplied to the method
+        :param ignored_exc: An exception or tuple of exceptions that shouldn't
+                            be retried, for example "NotFound" exceptions.
         :param kwargs: The amount of arguments supplied to the method
         :return: The value as retrieved from the external service
         """
 
         num_retries = CONF.watcher_datasources.query_max_retries
         timeout = CONF.watcher_datasources.query_timeout
+        ignored_exc = ignored_exc or tuple()
+
         for i in range(num_retries):
             try:
                 return f(*args, **kwargs)
+            except ignored_exc as e:
+                LOG.debug("Got an ignored exception (%s) while calling: %s ",
+                          e, f)
+                return
             except Exception as e:
                 LOG.exception(e)
                 self.query_retry_reset(e)

watcher/decision_engine/datasources/gnocchi.py

@@ -19,6 +19,7 @@
 from datetime import datetime
 from datetime import timedelta
 
+from gnocchiclient import exceptions as gnc_exc
 from oslo_config import cfg
 from oslo_log import log
 
@@ -38,7 +39,7 @@ class GnocchiHelper(base.DataSourceBase):
                       host_inlet_temp='hardware.ipmi.node.temperature',
                       host_airflow='hardware.ipmi.node.airflow',
                       host_power='hardware.ipmi.node.power',
-                      instance_cpu_usage='cpu_util',
+                      instance_cpu_usage='cpu',
                       instance_ram_usage='memory.resident',
                       instance_ram_allocated='memory',
                       instance_l3_cache_usage='cpu_l3_cache',
@@ -84,7 +85,9 @@ class GnocchiHelper(base.DataSourceBase):
             kwargs = dict(query={"=": {"original_resource_id": resource_id}},
                           limit=1)
             resources = self.query_retry(
-                f=self.gnocchi.resource.search, **kwargs)
+                f=self.gnocchi.resource.search,
+                ignored_exc=gnc_exc.NotFound,
+                **kwargs)
 
             if not resources:
                 LOG.warning("The {0} resource {1} could not be "
@@ -93,6 +96,25 @@ class GnocchiHelper(base.DataSourceBase):
 
             resource_id = resources[0]['id']
 
+        if meter_name == "instance_cpu_usage":
+            if resource_type != "instance":
+                LOG.warning("Unsupported resource type for metric "
+                            "'instance_cpu_usage': ", resource_type)
+                return
+
+            # The "cpu_util" gauge (percentage) metric has been removed.
+            # We're going to obtain the same result by using the rate of change
+            # aggregate operation.
+            if aggregate not in ("mean", "rate:mean"):
+                LOG.warning("Unsupported aggregate for instance_cpu_usage "
+                            "metric: %s. "
+                            "Supported aggregates: mean, rate:mean ",
+                            aggregate)
+                return
+
+            # TODO(lpetrut): consider supporting other aggregates.
+            aggregate = "rate:mean"
+
         raw_kwargs = dict(
             metric=meter,
             start=start_time,
@@ -105,7 +127,9 @@ class GnocchiHelper(base.DataSourceBase):
         kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
 
         statistics = self.query_retry(
-            f=self.gnocchi.metric.get_measures, **kwargs)
+            f=self.gnocchi.metric.get_measures,
+            ignored_exc=gnc_exc.NotFound,
+            **kwargs)
 
         return_value = None
         if statistics:
@@ -117,6 +141,17 @@ class GnocchiHelper(base.DataSourceBase):
                 # Airflow from hardware.ipmi.node.airflow is reported as
                 # 1/10 th of actual CFM
                 return_value *= 10
+            if meter_name == "instance_cpu_usage":
+                # "rate:mean" can return negative values for migrated vms.
+                return_value = max(0, return_value)
+
+                # We're converting the cumulative cpu time (ns) to cpu usage
+                # percentage.
+                vcpus = resource.vcpus
+                if not vcpus:
+                    LOG.warning("instance vcpu count not set, assuming 1")
+                    vcpus = 1
+                return_value *= 100 / (granularity * 10e+8) / vcpus
 
         return return_value
 
@@ -132,7 +167,9 @@ class GnocchiHelper(base.DataSourceBase):
             kwargs = dict(query={"=": {"original_resource_id": resource_id}},
                           limit=1)
             resources = self.query_retry(
-                f=self.gnocchi.resource.search, **kwargs)
+                f=self.gnocchi.resource.search,
+                ignored_exc=gnc_exc.NotFound,
+                **kwargs)
 
             if not resources:
                 LOG.warning("The {0} resource {1} could not be "
@@ -152,7 +189,9 @@ class GnocchiHelper(base.DataSourceBase):
         kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
 
         statistics = self.query_retry(
-            f=self.gnocchi.metric.get_measures, **kwargs)
+            f=self.gnocchi.metric.get_measures,
+            ignored_exc=gnc_exc.NotFound,
+            **kwargs)
 
         return_value = None
         if statistics:

watcher/decision_engine/datasources/grafana.py

@@ -18,6 +18,7 @@
 
 from urllib import parse as urlparse
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_log import log
 
@@ -137,12 +138,13 @@ class GrafanaHelper(base.DataSourceBase):
             raise exception.DataSourceNotAvailable(self.NAME)
 
         resp = requests.get(self._base_url + str(project_id) + '/query',
-                            params=params, headers=self._headers)
-        if resp.status_code == 200:
+                            params=params, headers=self._headers,
+                            timeout=CONF.grafana_client.http_timeout)
+        if resp.status_code == HTTPStatus.OK:
             return resp
-        elif resp.status_code == 400:
+        elif resp.status_code == HTTPStatus.BAD_REQUEST:
             LOG.error("Query for metric is invalid")
-        elif resp.status_code == 401:
+        elif resp.status_code == HTTPStatus.UNAUTHORIZED:
             LOG.error("Authorization token is invalid")
         raise exception.DataSourceNotAvailable(self.NAME)
 

watcher/decision_engine/model/collector/cinder.py

@@ -205,7 +205,7 @@ class CinderModelBuilder(base.BaseModelBuilder):
         """Build a storage node from a Cinder storage node
 
         :param node: A storage node
-        :type node: :py:class:`~cinderclient.v2.services.Service`
+        :type node: :py:class:`~cinderclient.v3.services.Service`
         """
         # node.host is formatted as host@backendname since ocata,
         # or may be only host as of ocata
@@ -233,7 +233,7 @@ class CinderModelBuilder(base.BaseModelBuilder):
         """Build a storage pool from a Cinder storage pool
 
         :param pool: A storage pool
-        :type pool: :py:class:`~cinderclient.v2.pools.Pool`
+        :type pool: :py:class:`~cinderclient.v3.pools.Pool`
         :raises: exception.InvalidPoolAttributeValue
         """
         # build up the storage pool.

watcher/decision_engine/model/collector/ironic.py

@@ -81,6 +81,7 @@ class BareMetalModelBuilder(base.BaseModelBuilder):
     def __init__(self, osc):
         self.osc = osc
         self.model = model_root.BaremetalModelRoot()
+        # TODO(lpetrut): add MAAS support
         self.ironic_helper = ironic_helper.IronicHelper(osc=self.osc)
 
     def add_ironic_node(self, node):

watcher/decision_engine/model/collector/nova.py

@@ -48,7 +48,7 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
                     "type": "array",
                     "items": {
                         "anyOf": [
-                            {"$ref": HOST_AGGREGATES + "id"},
+                            {"$ref": HOST_AGGREGATES + "host_aggr_id"},
                             {"$ref": HOST_AGGREGATES + "name"},
                         ]
                     }
@@ -98,7 +98,8 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
                                 "type": "array",
                                 "items": {
                                     "anyOf": [
-                                        {"$ref": HOST_AGGREGATES + "id"},
+                                        {"$ref":
+                                            HOST_AGGREGATES + "host_aggr_id"},
                                         {"$ref": HOST_AGGREGATES + "name"},
                                     ]
                                 }
@@ -129,7 +130,7 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
             "additionalProperties": False
         },
         "host_aggregates": {
-            "id": {
+            "host_aggr_id": {
                 "properties": {
                     "id": {
                         "oneOf": [

watcher/decision_engine/model/model_root.py

@@ -157,7 +157,7 @@ class ModelRoot(nx.DiGraph, base.Model):
             if node_list:
                 return node_list[0]
             else:
-                raise exception.ComputeResourceNotFound
+                raise exception.ComputeNodeNotFound(name=name)
         except exception.ComputeResourceNotFound:
             raise exception.ComputeNodeNotFound(name=name)
 

watcher/decision_engine/strategy/strategies/base.py

@@ -252,9 +252,6 @@ class BaseStrategy(loadable.Loadable, metaclass=abc.ABCMeta):
         if not self.compute_model:
             raise exception.ClusterStateNotDefined()
 
-        if self.compute_model.stale:
-            raise exception.ClusterStateStale()
-
         LOG.debug(self.compute_model.to_string())
 
     def execute(self, audit=None):

watcher/decision_engine/strategy/strategies/saving_energy.py

@@ -23,6 +23,8 @@ from oslo_log import log
 
 from watcher._i18n import _
 from watcher.common import exception
+from watcher.common.metal_helper import constants as metal_constants
+from watcher.common.metal_helper import factory as metal_helper_factory
 from watcher.decision_engine.strategy.strategies import base
 
 LOG = log.getLogger(__name__)
@@ -81,7 +83,7 @@ class SavingEnergy(base.SavingEnergyBaseStrategy):
     def __init__(self, config, osc=None):
 
         super(SavingEnergy, self).__init__(config, osc)
-        self._ironic_client = None
+        self._metal_helper = None
         self._nova_client = None
 
         self.with_vms_node_pool = []
@@ -91,10 +93,10 @@ class SavingEnergy(base.SavingEnergyBaseStrategy):
         self.min_free_hosts_num = 1
 
     @property
-    def ironic_client(self):
-        if not self._ironic_client:
-            self._ironic_client = self.osc.ironic()
-        return self._ironic_client
+    def metal_helper(self):
+        if not self._metal_helper:
+            self._metal_helper = metal_helper_factory.get_helper(self.osc)
+        return self._metal_helper
 
     @property
     def nova_client(self):
@@ -149,10 +151,10 @@ class SavingEnergy(base.SavingEnergyBaseStrategy):
         :return: None
         """
         params = {'state': state,
-                  'resource_name': node.hostname}
+                  'resource_name': node.get_hypervisor_hostname()}
         self.solution.add_action(
             action_type='change_node_power_state',
-            resource_id=node.uuid,
+            resource_id=node.get_id(),
             input_parameters=params)
 
     def get_hosts_pool(self):
@@ -162,57 +164,61 @@ class SavingEnergy(base.SavingEnergyBaseStrategy):
 
         """
 
-        node_list = self.ironic_client.node.list()
+        node_list = self.metal_helper.list_compute_nodes()
         for node in node_list:
-            node_info = self.ironic_client.node.get(node.uuid)
-            hypervisor_id = node_info.extra.get('compute_node_id', None)
-            if hypervisor_id is None:
-                LOG.warning(('Cannot find compute_node_id in extra '
-                             'of ironic node %s'), node.uuid)
-                continue
-            hypervisor_node = self.nova_client.hypervisors.get(hypervisor_id)
-            if hypervisor_node is None:
-                LOG.warning(('Cannot find hypervisor %s'), hypervisor_id)
-                continue
-            node.hostname = hypervisor_node.hypervisor_hostname
-            hypervisor_node = hypervisor_node.to_dict()
+            hypervisor_node = node.get_hypervisor_node().to_dict()
+
             compute_service = hypervisor_node.get('service', None)
             host_name = compute_service.get('host')
+            LOG.debug("Found hypervisor: %s", hypervisor_node)
             try:
                 self.compute_model.get_node_by_name(host_name)
             except exception.ComputeNodeNotFound:
+                LOG.info("The compute model does not contain the host: %s",
+                         host_name)
                 continue
 
-            if not (hypervisor_node.get('state') == 'up'):
-                """filter nodes that are not in 'up' state"""
+            if (node.hv_up_when_powered_off and
+                    hypervisor_node.get('state') != 'up'):
+                # filter nodes that are not in 'up' state
+                LOG.info("Ignoring node that isn't in 'up' state: %s",
+                         host_name)
                 continue
             else:
                 if (hypervisor_node['running_vms'] == 0):
-                    if (node_info.power_state == 'power on'):
+                    power_state = node.get_power_state()
+                    if power_state == metal_constants.PowerState.ON:
                         self.free_poweron_node_pool.append(node)
-                    elif (node_info.power_state == 'power off'):
+                    elif power_state == metal_constants.PowerState.OFF:
                         self.free_poweroff_node_pool.append(node)
+                    else:
+                        LOG.info("Ignoring node %s, unknown state: %s",
+                                 node, power_state)
                 else:
                     self.with_vms_node_pool.append(node)
 
     def save_energy(self):
 
-        need_poweron = max(
+        need_poweron = int(max(
             (len(self.with_vms_node_pool) * self.free_used_percent / 100), (
-                self.min_free_hosts_num))
+                self.min_free_hosts_num)))
         len_poweron = len(self.free_poweron_node_pool)
         len_poweroff = len(self.free_poweroff_node_pool)
+        LOG.debug("need_poweron: %s, len_poweron: %s, len_poweroff: %s",
+                  need_poweron, len_poweron, len_poweroff)
         if len_poweron > need_poweron:
             for node in random.sample(self.free_poweron_node_pool,
                                       (len_poweron - need_poweron)):
-                self.add_action_poweronoff_node(node, 'off')
-                LOG.debug("power off %s", node.uuid)
+                self.add_action_poweronoff_node(node,
+                                                metal_constants.PowerState.OFF)
+                LOG.info("power off %s", node.get_id())
         elif len_poweron < need_poweron:
             diff = need_poweron - len_poweron
             for node in random.sample(self.free_poweroff_node_pool,
                                       min(len_poweroff, diff)):
-                self.add_action_poweronoff_node(node, 'on')
-                LOG.debug("power on %s", node.uuid)
+                self.add_action_poweronoff_node(node,
+                                                metal_constants.PowerState.ON)
+                LOG.info("power on %s", node.get_id())
 
     def pre_execute(self):
         self._pre_execute()

watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py

@@ -18,9 +18,13 @@
 # limitations under the License.
 #
 
+import collections
+
 from oslo_log import log
+import oslo_utils
 
 from watcher._i18n import _
+from watcher.applier.actions import migration
 from watcher.common import exception
 from watcher.decision_engine.model import element
 from watcher.decision_engine.strategy.strategies import base
@@ -66,7 +70,8 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
 
     AGGREGATE = 'mean'
     DATASOURCE_METRICS = ['instance_ram_allocated', 'instance_cpu_usage',
-                          'instance_ram_usage', 'instance_root_disk_size']
+                          'instance_ram_usage', 'instance_root_disk_size',
+                          'host_cpu_usage', 'host_ram_usage']
 
     MIGRATION = "migrate"
     CHANGE_NOVA_SERVICE_STATE = "change_nova_service_state"
@@ -76,6 +81,11 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
         self.number_of_migrations = 0
         self.number_of_released_nodes = 0
         self.datasource_instance_data_cache = dict()
+        self.datasource_node_data_cache = dict()
+        # Host metric adjustments that take into account planned
+        # migrations.
+        self.host_metric_delta = collections.defaultdict(
+            lambda: collections.defaultdict(int))
 
     @classmethod
     def get_name(cls):
@@ -196,12 +206,12 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
         :return: None
         """
         instance_state_str = self.get_instance_state_str(instance)
-        if instance_state_str not in (element.InstanceState.ACTIVE.value,
-                                      element.InstanceState.PAUSED.value):
-            # Watcher currently only supports live VM migration and block live
-            # VM migration which both requires migrated VM to be active.
-            # When supported, the cold migration may be used as a fallback
-            # migration mechanism to move non active VMs.
+        if instance_state_str in (element.InstanceState.ACTIVE.value,
+                                  element.InstanceState.PAUSED.value):
+            migration_type = migration.Migrate.LIVE_MIGRATION
+        elif instance_state_str == element.InstanceState.STOPPED.value:
+            migration_type = migration.Migrate.COLD_MIGRATION
+        else:
             LOG.error(
                 'Cannot live migrate: instance_uuid=%(instance_uuid)s, '
                 'state=%(instance_state)s.', dict(
@@ -209,8 +219,6 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
                     instance_state=instance_state_str))
             return
 
-        migration_type = 'live'
-
         # Here will makes repeated actions to enable the same compute node,
         # when migrating VMs to the destination node which is disabled.
         # Whether should we remove the same actions in the solution???
@@ -228,6 +236,18 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
                 destination_node)
             self.number_of_migrations += 1
 
+            instance_util = self.get_instance_utilization(instance)
+            self.host_metric_delta[source_node.hostname]['cpu'] -= (
+                instance_util['cpu'])
+            # We'll deduce the vm allocated memory.
+            self.host_metric_delta[source_node.hostname]['ram'] -= (
+                instance.memory)
+
+            self.host_metric_delta[destination_node.hostname]['cpu'] += (
+                instance_util['cpu'])
+            self.host_metric_delta[destination_node.hostname]['ram'] += (
+                instance.memory)
+
     def disable_unused_nodes(self):
         """Generate actions for disabling unused nodes.
 
@@ -290,6 +310,21 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
             disk=instance_disk_util)
         return self.datasource_instance_data_cache.get(instance.uuid)
 
+    def _get_node_total_utilization(self, node):
+        if node.hostname in self.datasource_node_data_cache:
+            return self.datasource_node_data_cache[node.hostname]
+
+        cpu = self.datasource_backend.get_host_cpu_usage(
+            node, self.period, self.AGGREGATE,
+            self.granularity)
+        ram = self.datasource_backend.get_host_ram_usage(
+            node, self.period, self.AGGREGATE,
+            self.granularity)
+
+        self.datasource_node_data_cache[node.hostname] = dict(
+            cpu=cpu, ram=ram)
+        return self.datasource_node_data_cache[node.hostname]
+
     def get_node_utilization(self, node):
         """Collect cpu, ram and disk utilization statistics of a node.
 
@@ -307,8 +342,36 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
             node_cpu_util += instance_util['cpu']
             node_ram_util += instance_util['ram']
             node_disk_util += instance_util['disk']
+            LOG.debug("instance utilization: %s %s",
+                      instance, instance_util)
 
-        return dict(cpu=node_cpu_util, ram=node_ram_util,
+        total_node_util = self._get_node_total_utilization(node)
+        total_node_cpu_util = total_node_util['cpu'] or 0
+        if total_node_cpu_util:
+            total_node_cpu_util = total_node_cpu_util * node.vcpus / 100
+            # account for planned migrations
+            total_node_cpu_util += self.host_metric_delta[node.hostname]['cpu']
+
+        total_node_ram_util = total_node_util['ram'] or 0
+        if total_node_ram_util:
+            total_node_ram_util /= oslo_utils.units.Ki
+            total_node_ram_util += self.host_metric_delta[node.hostname]['ram']
+
+        LOG.debug(
+            "node utilization: %s. "
+            "total instance cpu: %s, "
+            "total instance ram: %s, "
+            "total instance disk: %s, "
+            "total host cpu: %s, "
+            "total host ram: %s, "
+            "node delta usage: %s.",
+            node,
+            node_cpu_util, node_ram_util, node_disk_util,
+            total_node_cpu_util, total_node_ram_util,
+            self.host_metric_delta[node.hostname])
+
+        return dict(cpu=max(node_cpu_util, total_node_cpu_util),
+                    ram=max(node_ram_util, total_node_ram_util),
                     disk=node_disk_util)
 
     def get_node_capacity(self, node):
@@ -388,8 +451,15 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
         instance_utilization = self.get_instance_utilization(instance)
         metrics = ['cpu', 'ram', 'disk']
         for m in metrics:
-            if (instance_utilization[m] + node_utilization[m] >
-                    node_capacity[m] * cc[m]):
+            fits = (instance_utilization[m] + node_utilization[m] <=
+                    node_capacity[m] * cc[m])
+            LOG.debug(
+                "Instance fits: %s, metric: %s, instance: %s, "
+                "node: %s, instance utilization: %s, "
+                "node utilization: %s, node capacity: %s, cc: %s",
+                fits, m, instance, node, instance_utilization[m],
+                node_utilization[m], node_capacity[m], cc[m])
+            if not fits:
                 return False
         return True
 
@@ -424,6 +494,9 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
                 for a in actions:
                     self.solution.actions.remove(a)
                     self.number_of_migrations -= 1
+                LOG.info("Optimized migrations: %s. "
+                         "Source: %s, destination: %s", actions,
+                         src_name, dst_name)
                 src_node = self.compute_model.get_node_by_name(src_name)
                 dst_node = self.compute_model.get_node_by_name(dst_name)
                 instance = self.compute_model.get_instance_by_uuid(
@@ -460,6 +533,8 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
                         key=lambda x: self.get_instance_utilization(
                             x)['cpu']
                 ):
+                    LOG.info("Node %s overloaded, attempting to reduce load.",
+                             node)
                     # skip exclude instance when migrating
                     if instance.watcher_exclude:
                         LOG.debug("Instance is excluded by scope, "
@@ -468,11 +543,19 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
                     for destination_node in reversed(sorted_nodes):
                         if self.instance_fits(
                                 instance, destination_node, cc):
+                            LOG.info("Offload: found fitting "
+                                     "destination (%s) for instance: %s. "
+                                     "Planning migration.",
+                                     destination_node, instance.uuid)
                             self.add_migration(instance, node,
                                                destination_node)
                             break
                     if not self.is_overloaded(node, cc):
+                        LOG.info("Node %s no longer overloaded.", node)
                         break
+                    else:
+                        LOG.info("Node still overloaded (%s), "
+                                 "continuing offload phase.", node)
 
     def consolidation_phase(self, cc):
         """Perform consolidation phase.
@@ -508,6 +591,10 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
                         break
                     if self.instance_fits(
                             instance, destination_node, cc):
+                        LOG.info("Consolidation: found fitting "
+                                 "destination (%s) for instance: %s. "
+                                 "Planning migration.",
+                                 destination_node, instance.uuid)
                         self.add_migration(instance, node,
                                            destination_node)
                         break

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -295,7 +295,7 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
                         self.threshold)
             return self.solution
 
-        # choose the server with largest cpu_util
+        # choose the server with largest cpu usage
         source_nodes = sorted(source_nodes,
                               reverse=True,
                               key=lambda x: (x[self._meter]))

watcher/decision_engine/strategy/strategies/zone_migration.py

@@ -16,7 +16,7 @@ from dateutil.parser import parse
 
 from oslo_log import log
 
-from cinderclient.v2.volumes import Volume
+from cinderclient.v3.volumes import Volume
 from novaclient.v2.servers import Server
 from watcher._i18n import _
 from watcher.common import cinder_helper
@@ -57,8 +57,19 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
         self.planned_cold_count = 0
         self.volume_count = 0
         self.planned_volume_count = 0
-        self.volume_update_count = 0
-        self.planned_volume_update_count = 0
+
+    # TODO(sean-n-mooney) This is backward compatibility
+    # for calling the swap code paths. Swap is now an alias
+    # for migrate, we should clean this up in a future
+    # cycle.
+    @property
+    def volume_update_count(self):
+        return self.volume_count
+
+    # same as above clean up later.
+    @property
+    def planned_volume_update_count(self):
+        return self.planned_volume_count
 
     @classmethod
     def get_name(cls):
@@ -312,8 +323,8 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
             planned_cold_migrate_instance_count=self.planned_cold_count,
             volume_migrate_count=self.volume_count,
             planned_volume_migrate_count=self.planned_volume_count,
-            volume_update_count=self.volume_update_count,
-            planned_volume_update_count=self.planned_volume_update_count
+            volume_update_count=self.volume_count,
+            planned_volume_update_count=self.planned_volume_count
         )
 
     def set_migration_count(self, targets):
@@ -328,10 +339,7 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
             elif self.is_cold(instance):
                 self.cold_count += 1
         for volume in targets.get('volume', []):
-            if self.is_available(volume):
-                self.volume_count += 1
-            elif self.is_in_use(volume):
-                self.volume_update_count += 1
+            self.volume_count += 1
 
     def is_live(self, instance):
         status = getattr(instance, 'status')
@@ -404,19 +412,16 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
             LOG.debug(src_type)
             LOG.debug("%s %s", dst_pool, dst_type)
 
-            if self.is_available(volume):
-                if src_type == dst_type:
-                    self._volume_migrate(volume, dst_pool)
-                else:
-                    self._volume_retype(volume, dst_type)
-            elif self.is_in_use(volume):
-                self._volume_update(volume, dst_type)
+            if src_type == dst_type:
+                self._volume_migrate(volume, dst_pool)
+            else:
+                self._volume_retype(volume, dst_type)
 
-                # if with_attached_volume is True, migrate attaching instances
-                if self.with_attached_volume:
-                    instances = [self.nova.find_instance(dic.get('server_id'))
-                                 for dic in volume.attachments]
-                    self.instances_migration(instances, action_counter)
+            # if with_attached_volume is True, migrate attaching instances
+            if self.with_attached_volume:
+                instances = [self.nova.find_instance(dic.get('server_id'))
+                             for dic in volume.attachments]
+                self.instances_migration(instances, action_counter)
 
             action_counter.add_pool(pool)
 
@@ -464,16 +469,6 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
             input_parameters=parameters)
         self.planned_cold_count += 1
 
-    def _volume_update(self, volume, dst_type):
-        parameters = {"migration_type": "swap",
-                      "destination_type": dst_type,
-                      "resource_name": volume.name}
-        self.solution.add_action(
-            action_type="volume_migrate",
-            resource_id=volume.id,
-            input_parameters=parameters)
-        self.planned_volume_update_count += 1
-
     def _volume_migrate(self, volume, dst_pool):
         parameters = {"migration_type": "migrate",
                       "destination_node": dst_pool,

watcher/locale/en_GB/LC_MESSAGES/watcher.po

@@ -1,15 +1,16 @@
 # Andi Chandler <andi@gowling.com>, 2017. #zanata
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
 # Andi Chandler <andi@gowling.com>, 2020. #zanata
+# Andi Chandler <andi@gowling.com>, 2022. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: watcher VERSION\n"
 "Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
-"POT-Creation-Date: 2020-10-27 04:14+0000\n"
+"POT-Creation-Date: 2022-08-29 03:03+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2020-10-28 11:02+0000\n"
+"PO-Revision-Date: 2022-05-31 08:38+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -507,6 +508,9 @@ msgstr ""
 msgid "Plugins"
 msgstr "Plugins"
 
+msgid "Policy File JSON to YAML Migration"
+msgstr "Policy File JSON to YAML Migration"
+
 #, python-format
 msgid "Policy doesn't allow %(action)s to be performed."
 msgstr "Policy doesn't allow %(action)s to be performed."

watcher/tests/api/test_base.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from http import HTTPStatus
+
 from watcher.tests.api import base
 
 
@@ -25,6 +27,6 @@ class TestBase(base.FunctionalTest):
         response = self.get_json('/bad/path',
                                  expect_errors=True,
                                  headers={"Accept": "application/json"})
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual("application/json", response.content_type)
         self.assertTrue(response.json['error_message'])

watcher/tests/api/v1/test_actions.py

@@ -14,6 +14,7 @@ import datetime
 import itertools
 from unittest import mock
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from wsme import types as wtypes
@@ -102,7 +103,7 @@ class TestListAction(api_base.FunctionalTest):
 
         response = self.get_json('/actions/%s' % action['uuid'],
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         action = obj_utils.create_test_action(self.context, parents=None)
@@ -125,7 +126,7 @@ class TestListAction(api_base.FunctionalTest):
         action = obj_utils.create_test_action(self.context, parents=None)
         response = self.get_json('/actions/%s/detail' % action['uuid'],
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         action_list = []
@@ -266,7 +267,7 @@ class TestListAction(api_base.FunctionalTest):
         url = '/actions?action_plan_uuid=%s&audit_uuid=%s' % (
             action_plan.uuid, self.audit.uuid)
         response = self.get_json(url, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
     def test_many_with_sort_key_uuid(self):
         action_plan = obj_utils.create_test_action_plan(
@@ -327,7 +328,7 @@ class TestListAction(api_base.FunctionalTest):
         response = self.get_json(
             '/actions?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
     def test_many_with_soft_deleted_action_plan_uuid(self):
         action_plan1 = obj_utils.create_test_action_plan(
@@ -488,7 +489,7 @@ class TestPatch(api_base.FunctionalTest):
             [{'path': '/state', 'value': new_state, 'op': 'replace'}],
             expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertTrue(response.json['error_message'])
 
 
@@ -516,7 +517,7 @@ class TestDelete(api_base.FunctionalTest):
         mock_utcnow.return_value = test_time
         response = self.delete('/actions/%s' % self.action.uuid,
                                expect_errors=True)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -536,7 +537,7 @@ class TestActionPolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:defaut"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_actions_plans.py

@@ -14,6 +14,7 @@ import datetime
 import itertools
 from unittest import mock
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 
@@ -87,7 +88,7 @@ class TestListActionPlan(api_base.FunctionalTest):
 
         response = self.get_json('/action_plans/%s' % action_plan['uuid'],
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         action_plan = obj_utils.create_test_action_plan(self.context)
@@ -113,7 +114,7 @@ class TestListActionPlan(api_base.FunctionalTest):
         response = self.get_json(
             '/action_plan/%s/detail' % action_plan['uuid'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         action_plan_list = []
@@ -260,7 +261,7 @@ class TestListActionPlan(api_base.FunctionalTest):
         response = self.get_json(
             '/action_plans?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
     def test_links(self):
         uuid = utils.generate_uuid()
@@ -317,7 +318,7 @@ class TestDelete(api_base.FunctionalTest):
     def test_delete_action_plan_without_action(self):
         response = self.delete('/action_plans/%s' % self.action_plan.uuid,
                                expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
         self.action_plan.state = objects.action_plan.State.SUCCEEDED
@@ -325,7 +326,7 @@ class TestDelete(api_base.FunctionalTest):
         self.delete('/action_plans/%s' % self.action_plan.uuid)
         response = self.get_json('/action_plans/%s' % self.action_plan.uuid,
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -345,20 +346,20 @@ class TestDelete(api_base.FunctionalTest):
             expect_errors=True)
 
         # The action plan does not exist anymore
-        self.assertEqual(404, ap_response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, ap_response.status_int)
         self.assertEqual('application/json', ap_response.content_type)
         self.assertTrue(ap_response.json['error_message'])
 
         # Nor does the action
         self.assertEqual(0, len(acts_response['actions']))
-        self.assertEqual(404, act_response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, act_response.status_int)
         self.assertEqual('application/json', act_response.content_type)
         self.assertTrue(act_response.json['error_message'])
 
     def test_delete_action_plan_not_found(self):
         uuid = utils.generate_uuid()
         response = self.delete('/action_plans/%s' % uuid, expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -388,7 +389,7 @@ class TestStart(api_base.FunctionalTest):
         uuid = utils.generate_uuid()
         response = self.post('/v1/action_plans/%s/%s' %
                              (uuid, 'start'), expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -401,11 +402,11 @@ class TestStart(api_base.FunctionalTest):
         response = self.post('/v1/action_plans/%s/%s/'
                              % (self.action_plan.uuid, 'start'),
                              expect_errors=True)
-        self.assertEqual(200, response.status_int)
+        self.assertEqual(HTTPStatus.OK, response.status_int)
         act_response = self.get_json(
             '/actions/%s' % action.uuid,
             expect_errors=True)
-        self.assertEqual(200, act_response.status_int)
+        self.assertEqual(HTTPStatus.OK, act_response.status_int)
         self.assertEqual('PENDING', act_response.json['state'])
         self.assertEqual('application/json', act_response.content_type)
 
@@ -445,7 +446,7 @@ class TestPatch(api_base.FunctionalTest):
             expect_errors=True)
 
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertTrue(response.json['error_message'])
 
     def test_replace_non_existent_action_plan_denied(self):
@@ -455,7 +456,7 @@ class TestPatch(api_base.FunctionalTest):
               'value': objects.action_plan.State.PENDING,
               'op': 'replace'}],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -465,7 +466,7 @@ class TestPatch(api_base.FunctionalTest):
             [{'path': '/foo', 'value': 'bar', 'op': 'add'}],
             expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertTrue(response.json['error_message'])
 
     def test_remove_denied(self):
@@ -480,7 +481,7 @@ class TestPatch(api_base.FunctionalTest):
             expect_errors=True)
 
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertTrue(response.json['error_message'])
 
     def test_remove_uuid_denied(self):
@@ -488,7 +489,7 @@ class TestPatch(api_base.FunctionalTest):
             '/action_plans/%s' % self.action_plan.uuid,
             [{'path': '/uuid', 'op': 'remove'}],
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -497,7 +498,7 @@ class TestPatch(api_base.FunctionalTest):
             '/action_plans/%s' % self.action_plan.uuid,
             [{'path': '/non-existent', 'op': 'remove'}],
             expect_errors=True)
-        self.assertEqual(400, response.status_code)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_code)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -512,7 +513,7 @@ class TestPatch(api_base.FunctionalTest):
             [{'path': '/state', 'value': new_state,
               'op': 'replace'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
         applier_mock.assert_called_once_with(mock.ANY,
                                              self.action_plan.uuid)
 
@@ -579,7 +580,7 @@ class TestPatchStateTransitionDenied(api_base.FunctionalTest):
 
         self.assertNotEqual(self.new_state, initial_ap['state'])
         self.assertEqual(self.original_state, updated_ap['state'])
-        self.assertEqual(400, response.status_code)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_code)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -618,7 +619,7 @@ class TestPatchStateTransitionOk(api_base.FunctionalTest):
         self.assertNotEqual(self.new_state, initial_ap['state'])
         self.assertEqual(self.new_state, updated_ap['state'])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
 
 class TestActionPlanPolicyEnforcement(api_base.FunctionalTest):
@@ -635,7 +636,7 @@ class TestActionPlanPolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:defaut"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_audit_templates.py

@@ -15,6 +15,7 @@ import itertools
 from unittest import mock
 from urllib import parse as urlparse
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from oslo_utils import timeutils
@@ -126,7 +127,7 @@ class TestListAuditTemplate(FunctionalTestWithSetup):
         response = self.get_json(
             '/audit_templates/%s' % audit_template['uuid'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         audit_template = obj_utils.create_test_audit_template(self.context)
@@ -152,7 +153,7 @@ class TestListAuditTemplate(FunctionalTestWithSetup):
         response = self.get_json(
             '/audit_templates/%s/detail' % audit_template['uuid'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         audit_template_list = []
@@ -336,7 +337,7 @@ class TestListAuditTemplate(FunctionalTestWithSetup):
         response = self.get_json(
             '/audit_templates?sort_key=%s' % 'goal_bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
 
 class TestPatch(FunctionalTestWithSetup):
@@ -362,7 +363,7 @@ class TestPatch(FunctionalTestWithSetup):
             [{'path': '/goal', 'value': new_goal_uuid,
               'op': 'replace'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
         response = self.get_json(
             '/audit_templates/%s' % self.audit_template.uuid)
@@ -386,7 +387,7 @@ class TestPatch(FunctionalTestWithSetup):
             [{'path': '/goal', 'value': new_goal_uuid,
               'op': 'replace'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
         response = self.get_json(
             '/audit_templates/%s' % self.audit_template.name)
@@ -401,7 +402,7 @@ class TestPatch(FunctionalTestWithSetup):
             [{'path': '/goal', 'value': self.fake_goal1.uuid,
               'op': 'replace'}],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -416,7 +417,7 @@ class TestPatch(FunctionalTestWithSetup):
                 [{'path': '/goal', 'value': utils.generate_uuid(),
                   'op': 'replace'}],
                 expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         assert not cn_mock.called
 
     def test_add_goal_uuid(self):
@@ -426,7 +427,7 @@ class TestPatch(FunctionalTestWithSetup):
               'value': self.fake_goal2.uuid,
               'op': 'add'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_int)
+        self.assertEqual(HTTPStatus.OK, response.status_int)
 
         response = self.get_json(
             '/audit_templates/%s' % self.audit_template.uuid)
@@ -439,7 +440,7 @@ class TestPatch(FunctionalTestWithSetup):
               'value': self.fake_strategy1.uuid,
               'op': 'add'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_int)
+        self.assertEqual(HTTPStatus.OK, response.status_int)
 
         response = self.get_json(
             '/audit_templates/%s' % self.audit_template.uuid)
@@ -452,7 +453,7 @@ class TestPatch(FunctionalTestWithSetup):
               'value': self.fake_strategy2['uuid'],
               'op': 'replace'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_int)
+        self.assertEqual(HTTPStatus.OK, response.status_int)
 
         response = self.get_json(
             '/audit_templates/%s' % self.audit_template.uuid)
@@ -466,7 +467,7 @@ class TestPatch(FunctionalTestWithSetup):
               'value': utils.generate_uuid(),  # Does not exist
               'op': 'replace'}], expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertTrue(response.json['error_message'])
 
     def test_add_non_existent_property(self):
@@ -475,7 +476,7 @@ class TestPatch(FunctionalTestWithSetup):
             [{'path': '/foo', 'value': 'bar', 'op': 'add'}],
             expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertTrue(response.json['error_message'])
 
     def test_remove_strategy(self):
@@ -492,7 +493,7 @@ class TestPatch(FunctionalTestWithSetup):
             '/audit_templates/%s' % self.audit_template.uuid,
             [{'path': '/strategy', 'op': 'remove'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
     def test_remove_goal(self):
         response = self.get_json(
@@ -503,7 +504,7 @@ class TestPatch(FunctionalTestWithSetup):
             '/audit_templates/%s' % self.audit_template.uuid,
             [{'path': '/goal', 'op': 'remove'}],
             expect_errors=True)
-        self.assertEqual(403, response.status_code)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_code)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -512,7 +513,7 @@ class TestPatch(FunctionalTestWithSetup):
             '/audit_templates/%s' % self.audit_template.uuid,
             [{'path': '/uuid', 'op': 'remove'}],
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -521,7 +522,7 @@ class TestPatch(FunctionalTestWithSetup):
             '/audit_templates/%s' % self.audit_template.uuid,
             [{'path': '/non-existent', 'op': 'remove'}],
             expect_errors=True)
-        self.assertEqual(400, response.status_code)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_code)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -538,7 +539,7 @@ class TestPost(FunctionalTestWithSetup):
 
         response = self.post_json('/audit_templates', audit_template_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         # Check location header
         self.assertIsNotNone(response.location)
         expected_location = \
@@ -565,7 +566,7 @@ class TestPost(FunctionalTestWithSetup):
 
         response = self.post_json('/audit_templates', audit_template_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         # Check location header
         self.assertIsNotNone(response.location)
         expected_location = \
@@ -613,7 +614,8 @@ class TestPost(FunctionalTestWithSetup):
             strategy=self.fake_strategy1.uuid, scope=scope)
         response = self.post_json('/audit_templates',
                                   audit_template_dict, expect_errors=True)
-        self.assertEqual(500, response.status_int)
+        self.assertEqual(HTTPStatus.INTERNAL_SERVER_ERROR,
+                         response.status_int)
 
     def test_create_audit_template_does_autogenerate_id(self):
         audit_template_dict = post_get_test_audit_template(
@@ -635,7 +637,7 @@ class TestPost(FunctionalTestWithSetup):
 
         response = self.post_json('/audit_templates', audit_template_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertTrue(utils.is_uuid_like(response.json['uuid']))
 
     def test_create_audit_template_with_invalid_goal(self):
@@ -648,7 +650,7 @@ class TestPost(FunctionalTestWithSetup):
                 goal_uuid=utils.generate_uuid())
             response = self.post_json('/audit_templates',
                                       audit_template_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         assert not cn_mock.called
 
     def test_create_audit_template_with_invalid_strategy(self):
@@ -662,7 +664,7 @@ class TestPost(FunctionalTestWithSetup):
                 strategy_uuid=utils.generate_uuid())
             response = self.post_json('/audit_templates',
                                       audit_template_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         assert not cn_mock.called
 
     def test_create_audit_template_with_unrelated_strategy(self):
@@ -676,7 +678,7 @@ class TestPost(FunctionalTestWithSetup):
                 strategy=self.fake_strategy2['uuid'])
             response = self.post_json('/audit_templates',
                                       audit_template_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         assert not cn_mock.called
 
     def test_create_audit_template_with_uuid(self):
@@ -689,7 +691,7 @@ class TestPost(FunctionalTestWithSetup):
             response = self.post_json('/audit_templates', audit_template_dict,
                                       expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         assert not cn_mock.called
 
     def test_create_audit_template_with_old_scope(self):
@@ -710,7 +712,7 @@ class TestPost(FunctionalTestWithSetup):
             strategy=self.fake_strategy1.uuid, scope=scope)
         response = self.post_json('/audit_templates',
                                   audit_template_dict)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
 
 
 class TestDelete(api_base.FunctionalTest):
@@ -730,7 +732,7 @@ class TestDelete(api_base.FunctionalTest):
         response = self.get_json(
             urlparse.quote('/audit_templates/%s' % self.audit_template.uuid),
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -753,7 +755,7 @@ class TestDelete(api_base.FunctionalTest):
         response = self.get_json(
             urlparse.quote('/audit_templates/%s' % self.audit_template.name),
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -771,7 +773,7 @@ class TestDelete(api_base.FunctionalTest):
         uuid = utils.generate_uuid()
         response = self.delete(
             '/audit_templates/%s' % uuid, expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -784,7 +786,7 @@ class TestAuditTemplatePolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:defaut"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_audits.py

@@ -16,6 +16,7 @@ import itertools
 from unittest import mock
 from urllib import parse as urlparse
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from oslo_utils import timeutils
@@ -130,7 +131,7 @@ class TestListAudit(api_base.FunctionalTest):
 
         response = self.get_json('/audits/%s' % audit['uuid'],
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         audit = obj_utils.create_test_audit(self.context)
@@ -153,7 +154,7 @@ class TestListAudit(api_base.FunctionalTest):
         audit = obj_utils.create_test_audit(self.context)
         response = self.get_json('/audits/%s/detail' % audit['uuid'],
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         audit_list = []
@@ -225,7 +226,7 @@ class TestListAudit(api_base.FunctionalTest):
         response = self.get_json(
             '/audits?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
     def test_links(self):
         uuid = utils.generate_uuid()
@@ -295,7 +296,7 @@ class TestPatch(api_base.FunctionalTest):
             [{'path': '/state', 'value': new_state,
              'op': 'replace'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
         response = self.get_json('/audits/%s' % self.audit.uuid)
         self.assertEqual(new_state, response['state'])
@@ -308,7 +309,7 @@ class TestPatch(api_base.FunctionalTest):
             '/audits/%s' % utils.generate_uuid(),
             [{'path': '/state', 'value': objects.audit.State.SUCCEEDED,
               'op': 'replace'}], expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -318,7 +319,7 @@ class TestPatch(api_base.FunctionalTest):
             '/audits/%s' % self.audit.uuid,
             [{'path': '/state', 'value': new_state, 'op': 'add'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_int)
+        self.assertEqual(HTTPStatus.OK, response.status_int)
 
         response = self.get_json('/audits/%s' % self.audit.uuid)
         self.assertEqual(new_state, response['state'])
@@ -329,7 +330,7 @@ class TestPatch(api_base.FunctionalTest):
             [{'path': '/foo', 'value': 'bar', 'op': 'add'}],
             expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertTrue(response.json['error_message'])
 
     def test_remove_ok(self):
@@ -339,7 +340,7 @@ class TestPatch(api_base.FunctionalTest):
         response = self.patch_json('/audits/%s' % self.audit.uuid,
                                    [{'path': '/interval', 'op': 'remove'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
         response = self.get_json('/audits/%s' % self.audit.uuid)
         self.assertIsNone(response['interval'])
@@ -348,7 +349,7 @@ class TestPatch(api_base.FunctionalTest):
         response = self.patch_json('/audits/%s' % self.audit.uuid,
                                    [{'path': '/uuid', 'op': 'remove'}],
                                    expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -357,7 +358,7 @@ class TestPatch(api_base.FunctionalTest):
             '/audits/%s' % self.audit.uuid,
             [{'path': '/non-existent', 'op': 'remove'}],
             expect_errors=True)
-        self.assertEqual(400, response.status_code)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_code)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -415,7 +416,7 @@ class TestPatchStateTransitionDenied(api_base.FunctionalTest):
               'op': 'replace'}],
             expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_code)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_code)
         self.assertTrue(response.json['error_message'])
 
         response = self.get_json('/audits/%s' % self.audit.uuid)
@@ -462,7 +463,7 @@ class TestPatchStateTransitionOk(api_base.FunctionalTest):
             [{'path': '/state', 'value': self.new_state,
              'op': 'replace'}])
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(200, response.status_code)
+        self.assertEqual(HTTPStatus.OK, response.status_code)
 
         response = self.get_json('/audits/%s' % self.audit.uuid)
         self.assertEqual(self.new_state, response['state'])
@@ -502,7 +503,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         # Check location header
         self.assertIsNotNone(response.location)
         expected_location = '/v1/audits/%s' % response.json['uuid']
@@ -527,7 +528,7 @@ class TestPost(api_base.FunctionalTest):
         audit_dict = post_get_test_audit(state=objects.audit.State.SUCCEEDED)
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -542,7 +543,7 @@ class TestPost(api_base.FunctionalTest):
                                'next_run_time', 'hostname'])
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -557,7 +558,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertTrue(utils.is_uuid_like(response.json['uuid']))
@@ -573,7 +574,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertTrue(utils.is_uuid_like(response.json['uuid']))
@@ -590,7 +591,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertTrue(utils.is_uuid_like(response.json['uuid']))
@@ -608,7 +609,7 @@ class TestPost(api_base.FunctionalTest):
             '01234567-8910-1112-1314-151617181920')
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual("application/json", response.content_type)
         expected_error_msg = ('The audit template UUID or name specified is '
                               'invalid')
@@ -643,7 +644,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertTrue(utils.is_uuid_like(response.json['uuid']))
@@ -660,7 +661,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertEqual(audit_dict['interval'], response.json['interval'])
@@ -679,7 +680,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertEqual(audit_dict['interval'], response.json['interval'])
@@ -698,9 +699,9 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(500, response.status_int)
+        self.assertEqual(HTTPStatus.INTERNAL_SERVER_ERROR, response.status_int)
         expected_error_msg = ('Exactly 5 or 6 columns has to be '
-                              'specified for iteratorexpression.')
+                              'specified for iterator expression.')
         self.assertTrue(response.json['error_message'])
         self.assertIn(expected_error_msg, response.json['error_message'])
 
@@ -714,7 +715,7 @@ class TestPost(api_base.FunctionalTest):
         audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         expected_error_msg = ('Interval of audit must be specified '
                               'for CONTINUOUS.')
@@ -731,7 +732,7 @@ class TestPost(api_base.FunctionalTest):
         audit_dict['audit_type'] = objects.audit.AuditType.ONESHOT.value
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         expected_error_msg = 'Interval of audit must not be set for ONESHOT.'
         self.assertTrue(response.json['error_message'])
@@ -755,7 +756,7 @@ class TestPost(api_base.FunctionalTest):
         del audit_dict['scope']
         response = self.post_json('/audits', audit_dict, expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         assert not mock_trigger_audit.called
 
     @mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
@@ -769,7 +770,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         expected_error_msg = ('Specify parameters but no predefined '
                               'strategy for audit, or no '
                               'parameter spec in predefined strategy')
@@ -792,7 +793,7 @@ class TestPost(api_base.FunctionalTest):
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         expected_error_msg = ('Specify parameters but no predefined '
                               'strategy for audit, or no '
                               'parameter spec in predefined strategy')
@@ -816,7 +817,7 @@ class TestPost(api_base.FunctionalTest):
             del audit_dict[k]
 
         response = self.post_json('/audits', audit_dict, expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual("application/json", response.content_type)
         expected_error_msg = 'Audit parameter fake2 are not allowed'
         self.assertTrue(response.json['error_message'])
@@ -875,13 +876,13 @@ class TestPost(api_base.FunctionalTest):
         audit_dict['name'] = normal_name
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(normal_name, response.json['name'])
 
         audit_dict['name'] = long_name
         response = self.post_json('/audits', audit_dict)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertNotEqual(long_name, response.json['name'])
 
     @mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
@@ -905,7 +906,7 @@ class TestPost(api_base.FunctionalTest):
             audit_dict,
             headers={'OpenStack-API-Version': 'infra-optim 1.1'})
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertEqual(objects.audit.State.PENDING,
                          response.json['state'])
         self.assertEqual(audit_dict['interval'], response.json['interval'])
@@ -944,7 +945,7 @@ class TestPost(api_base.FunctionalTest):
             headers={'OpenStack-API-Version': 'infra-optim 1.0'},
             expect_errors=True)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(406, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_ACCEPTABLE, response.status_int)
         expected_error_msg = 'Request not acceptable.'
         self.assertTrue(response.json['error_message'])
         self.assertIn(expected_error_msg, response.json['error_message'])
@@ -963,7 +964,7 @@ class TestPost(api_base.FunctionalTest):
             audit_dict,
             headers={'OpenStack-API-Version': 'infra-optim 1.2'})
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertFalse(response.json['force'])
 
     @mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
@@ -980,7 +981,7 @@ class TestPost(api_base.FunctionalTest):
             audit_dict,
             headers={'OpenStack-API-Version': 'infra-optim 1.2'})
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(201, response.status_int)
+        self.assertEqual(HTTPStatus.CREATED, response.status_int)
         self.assertTrue(response.json['force'])
 
 
@@ -1013,7 +1014,7 @@ class TestDelete(api_base.FunctionalTest):
              'op': 'replace'}])
         response = self.delete('/audits/%s' % self.audit.uuid,
                                expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -1025,7 +1026,7 @@ class TestDelete(api_base.FunctionalTest):
         self.delete('/audits/%s' % self.audit.uuid)
         response = self.get_json('/audits/%s' % self.audit.uuid,
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -1041,7 +1042,7 @@ class TestDelete(api_base.FunctionalTest):
     def test_delete_audit_not_found(self):
         uuid = utils.generate_uuid()
         response = self.delete('/audits/%s' % uuid, expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -1058,7 +1059,7 @@ class TestAuditPolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:defaut"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_data_model.py

@@ -15,6 +15,7 @@
 
 from unittest import mock
 
+from http import HTTPStatus
 from oslo_serialization import jsonutils
 
 from watcher.decision_engine import rpcapi as deapi
@@ -42,7 +43,7 @@ class TestListDataModel(api_base.FunctionalTest):
             '/data_model/?data_model_type=compute',
             headers={'OpenStack-API-Version': 'infra-optim 1.2'},
             expect_errors=True)
-        self.assertEqual(406, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_ACCEPTABLE, response.status_int)
 
 
 class TestDataModelPolicyEnforcement(api_base.FunctionalTest):
@@ -59,7 +60,7 @@ class TestDataModelPolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:defaut"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_goals.py

@@ -10,6 +10,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from urllib import parse as urlparse
@@ -59,7 +60,7 @@ class TestListGoal(api_base.FunctionalTest):
         response = self.get_json(
             '/goals/%s' % goal['uuid'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         goal = obj_utils.create_test_goal(self.context)
@@ -71,7 +72,7 @@ class TestListGoal(api_base.FunctionalTest):
         goal = obj_utils.create_test_goal(self.context)
         response = self.get_json('/goals/%s/detail' % goal.uuid,
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         goal_list = []
@@ -139,7 +140,7 @@ class TestListGoal(api_base.FunctionalTest):
         response = self.get_json(
             '/goals?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
 
 class TestGoalPolicyEnforcement(api_base.FunctionalTest):
@@ -150,7 +151,7 @@ class TestGoalPolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:default"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_microversions.py

@@ -10,6 +10,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+from http import HTTPStatus
+
 from watcher.api.controllers.v1 import versions
 from watcher.tests.api import base as api_base
 
@@ -38,7 +40,7 @@ class TestMicroversions(api_base.FunctionalTest):
                                                         '10'])},
             expect_errors=True, return_json=False)
         self.assertEqual('application/json', response.content_type)
-        self.assertEqual(406, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_ACCEPTABLE, response.status_int)
         expected_error_msg = ('Invalid value for'
                               ' OpenStack-API-Version header')
         self.assertTrue(response.json['error_message'])
@@ -98,7 +100,7 @@ class TestMicroversions(api_base.FunctionalTest):
             headers={'OpenStack-API-Version': ' '.join([SERVICE_TYPE,
                                                         '1.999'])},
             expect_errors=True)
-        self.assertEqual(406, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_ACCEPTABLE, response.status_int)
         self.assertEqual(response.headers[H_MIN_VER], MIN_VER)
         self.assertEqual(response.headers[H_MAX_VER], MAX_VER)
         expected_error_msg = ('Version 1.999 was requested but the minor '

watcher/tests/api/v1/test_scoring_engines.py

@@ -10,6 +10,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from watcher.common import utils
@@ -44,7 +45,7 @@ class TestListScoringEngine(api_base.FunctionalTest):
         response = self.get_json(
             '/scoring_engines/%s' % scoring_engine['name'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         obj_utils.create_test_goal(self.context)
@@ -63,7 +64,7 @@ class TestListScoringEngine(api_base.FunctionalTest):
         response = self.get_json(
             '/scoring_engines/%s/detail' % scoring_engine.id,
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         scoring_engine_list = []
@@ -131,7 +132,7 @@ class TestListScoringEngine(api_base.FunctionalTest):
         response = self.get_json(
             '/goals?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
 
 class TestScoringEnginePolicyEnforcement(api_base.FunctionalTest):
@@ -142,7 +143,7 @@ class TestScoringEnginePolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:default"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_services.py

@@ -10,6 +10,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from urllib import parse as urlparse
@@ -57,7 +58,7 @@ class TestListService(api_base.FunctionalTest):
         response = self.get_json(
             '/services/%s' % service['id'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         service = obj_utils.create_test_service(self.context)
@@ -74,7 +75,7 @@ class TestListService(api_base.FunctionalTest):
         service = obj_utils.create_test_service(self.context)
         response = self.get_json('/services/%s/detail' % service.id,
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         service_list = []
@@ -149,7 +150,7 @@ class TestListService(api_base.FunctionalTest):
         response = self.get_json(
             '/services?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
 
 class TestServicePolicyEnforcement(api_base.FunctionalTest):
@@ -160,7 +161,7 @@ class TestServicePolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:default"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_strategies.py

@@ -13,6 +13,7 @@
 from unittest import mock
 from urllib import parse as urlparse
 
+from http import HTTPStatus
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 
@@ -88,7 +89,7 @@ class TestListStrategy(api_base.FunctionalTest):
         response = self.get_json(
             '/strategies/%s' % strategy['uuid'],
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_detail(self):
         strategy = obj_utils.create_test_strategy(self.context)
@@ -104,7 +105,7 @@ class TestListStrategy(api_base.FunctionalTest):
         strategy = obj_utils.create_test_strategy(self.context)
         response = self.get_json('/strategies/%s/detail' % strategy.uuid,
                                  expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
 
     def test_many(self):
         strategy_list = []
@@ -240,7 +241,7 @@ class TestListStrategy(api_base.FunctionalTest):
         response = self.get_json(
             '/strategies?sort_key=%s' % 'bad_name',
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
 
 
 class TestStrategyPolicyEnforcement(api_base.FunctionalTest):
@@ -256,7 +257,7 @@ class TestStrategyPolicyEnforcement(api_base.FunctionalTest):
             "default": "rule:admin_api",
             rule: "rule:defaut"})
         response = func(*arg, **kwarg)
-        self.assertEqual(403, response.status_int)
+        self.assertEqual(HTTPStatus.FORBIDDEN, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(
             "Policy doesn't allow %s to be performed." % rule,

watcher/tests/api/v1/test_types.py

@@ -18,6 +18,8 @@ import webtest
 import wsme
 from wsme import types as wtypes
 
+from http import HTTPStatus
+
 from watcher.api.controllers.v1 import types
 from watcher.common import exception
 from watcher.common import utils
@@ -120,68 +122,68 @@ class TestJsonPatchType(base.TestCase):
                          {'path': '/dict', 'op': 'add',
                           'value': {'cat': 'meow'}}]
         ret = self._patch_json(valid_patches, False)
-        self.assertEqual(200, ret.status_int)
+        self.assertEqual(HTTPStatus.OK, ret.status_int)
         self.assertEqual(valid_patches, ret.json)
 
     def test_cannot_update_internal_attr(self):
         patch = [{'path': '/internal', 'op': 'replace', 'value': 'foo'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_cannot_update_internal_dict_attr(self):
         patch = [{'path': '/internal', 'op': 'replace',
                  'value': 'foo'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_mandatory_attr(self):
         patch = [{'op': 'replace', 'path': '/mandatory', 'value': 'foo'}]
         ret = self._patch_json(patch, False)
-        self.assertEqual(200, ret.status_int)
+        self.assertEqual(HTTPStatus.OK, ret.status_int)
         self.assertEqual(patch, ret.json)
 
     def test_cannot_remove_mandatory_attr(self):
         patch = [{'op': 'remove', 'path': '/mandatory'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_missing_required_fields_path(self):
         missing_path = [{'op': 'remove'}]
         ret = self._patch_json(missing_path, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_missing_required_fields_op(self):
         missing_op = [{'path': '/foo'}]
         ret = self._patch_json(missing_op, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_invalid_op(self):
         patch = [{'path': '/foo', 'op': 'invalid'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_invalid_path(self):
         patch = [{'path': 'invalid-path', 'op': 'remove'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_cannot_add_with_no_value(self):
         patch = [{'path': '/extra/foo', 'op': 'add'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
     def test_cannot_replace_with_no_value(self):
         patch = [{'path': '/foo', 'op': 'replace'}]
         ret = self._patch_json(patch, True)
-        self.assertEqual(400, ret.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, ret.status_int)
         self.assertTrue(ret.json['faultstring'])
 
 

watcher/tests/api/v1/test_webhooks.py

@@ -12,6 +12,8 @@
 
 from unittest import mock
 
+from http import HTTPStatus
+
 from watcher.decision_engine import rpcapi as deapi
 from watcher import objects
 from watcher.tests.api import base as api_base
@@ -34,7 +36,7 @@ class TestPost(api_base.FunctionalTest):
         response = self.post_json(
             '/webhooks/%s' % audit['uuid'], {},
             headers={'OpenStack-API-Version': 'infra-optim 1.4'})
-        self.assertEqual(202, response.status_int)
+        self.assertEqual(HTTPStatus.ACCEPTED, response.status_int)
         mock_trigger_audit.assert_called_once_with(
             mock.ANY, audit['uuid'])
 
@@ -43,7 +45,7 @@ class TestPost(api_base.FunctionalTest):
             '/webhooks/no-audit', {},
             headers={'OpenStack-API-Version': 'infra-optim 1.4'},
             expect_errors=True)
-        self.assertEqual(404, response.status_int)
+        self.assertEqual(HTTPStatus.NOT_FOUND, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -53,7 +55,7 @@ class TestPost(api_base.FunctionalTest):
             '/webhooks/%s' % audit['uuid'], {},
             headers={'OpenStack-API-Version': 'infra-optim 1.4'},
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])
 
@@ -66,6 +68,6 @@ class TestPost(api_base.FunctionalTest):
             '/webhooks/%s' % audit['uuid'], {},
             headers={'OpenStack-API-Version': 'infra-optim 1.4'},
             expect_errors=True)
-        self.assertEqual(400, response.status_int)
+        self.assertEqual(HTTPStatus.BAD_REQUEST, response.status_int)
         self.assertEqual('application/json', response.content_type)
         self.assertTrue(response.json['error_message'])

watcher/tests/applier/actions/test_change_node_power_state.py

@@ -19,134 +19,151 @@ import jsonschema
 
 from watcher.applier.actions import base as baction
 from watcher.applier.actions import change_node_power_state
-from watcher.common import clients
+from watcher.common.metal_helper import constants as m_constants
+from watcher.common.metal_helper import factory as m_helper_factory
 from watcher.tests import base
+from watcher.tests.decision_engine import fake_metal_helper
 
 COMPUTE_NODE = "compute-1"
 
 
-@mock.patch.object(clients.OpenStackClients, 'nova')
-@mock.patch.object(clients.OpenStackClients, 'ironic')
 class TestChangeNodePowerState(base.TestCase):
 
     def setUp(self):
         super(TestChangeNodePowerState, self).setUp()
 
+        p_m_factory = mock.patch.object(m_helper_factory, 'get_helper')
+        m_factory = p_m_factory.start()
+        self._metal_helper = m_factory.return_value
+        self.addCleanup(p_m_factory.stop)
+
+        # Let's avoid unnecessary sleep calls while running the test.
+        p_sleep = mock.patch('time.sleep')
+        p_sleep.start()
+        self.addCleanup(p_sleep.stop)
+
         self.input_parameters = {
             baction.BaseAction.RESOURCE_ID: COMPUTE_NODE,
-            "state": change_node_power_state.NodeState.POWERON.value,
+            "state": m_constants.PowerState.ON.value,
         }
         self.action = change_node_power_state.ChangeNodePowerState(
             mock.Mock())
         self.action.input_parameters = self.input_parameters
 
-    def test_parameters_down(self, mock_ironic, mock_nova):
+    def test_parameters_down(self):
         self.action.input_parameters = {
             baction.BaseAction.RESOURCE_ID: COMPUTE_NODE,
             self.action.STATE:
-                change_node_power_state.NodeState.POWEROFF.value}
+                m_constants.PowerState.OFF.value}
         self.assertTrue(self.action.validate_parameters())
 
-    def test_parameters_up(self, mock_ironic, mock_nova):
+    def test_parameters_up(self):
         self.action.input_parameters = {
             baction.BaseAction.RESOURCE_ID: COMPUTE_NODE,
             self.action.STATE:
-                change_node_power_state.NodeState.POWERON.value}
+                m_constants.PowerState.ON.value}
         self.assertTrue(self.action.validate_parameters())
 
-    def test_parameters_exception_wrong_state(self, mock_ironic, mock_nova):
+    def test_parameters_exception_wrong_state(self):
         self.action.input_parameters = {
             baction.BaseAction.RESOURCE_ID: COMPUTE_NODE,
             self.action.STATE: 'error'}
         self.assertRaises(jsonschema.ValidationError,
                           self.action.validate_parameters)
 
-    def test_parameters_resource_id_empty(self, mock_ironic, mock_nova):
+    def test_parameters_resource_id_empty(self):
         self.action.input_parameters = {
             self.action.STATE:
-                change_node_power_state.NodeState.POWERON.value,
+                m_constants.PowerState.ON.value,
         }
         self.assertRaises(jsonschema.ValidationError,
                           self.action.validate_parameters)
 
-    def test_parameters_applies_add_extra(self, mock_ironic, mock_nova):
+    def test_parameters_applies_add_extra(self):
         self.action.input_parameters = {"extra": "failed"}
         self.assertRaises(jsonschema.ValidationError,
                           self.action.validate_parameters)
 
-    def test_change_service_state_pre_condition(self, mock_ironic, mock_nova):
+    def test_change_service_state_pre_condition(self):
         try:
             self.action.pre_condition()
         except Exception as exc:
             self.fail(exc)
 
-    def test_change_node_state_post_condition(self, mock_ironic, mock_nova):
+    def test_change_node_state_post_condition(self):
         try:
             self.action.post_condition()
         except Exception as exc:
             self.fail(exc)
 
-    def test_execute_node_service_state_with_poweron_target(
-            self, mock_ironic, mock_nova):
-        mock_irclient = mock_ironic.return_value
+    def test_execute_node_service_state_with_poweron_target(self):
         self.action.input_parameters["state"] = (
-            change_node_power_state.NodeState.POWERON.value)
-        mock_irclient.node.get.side_effect = [
-            mock.MagicMock(power_state='power off'),
-            mock.MagicMock(power_state='power on')]
+            m_constants.PowerState.ON.value)
+        mock_nodes = [
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.OFF),
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.ON)
+        ]
+        self._metal_helper.get_node.side_effect = mock_nodes
 
         result = self.action.execute()
         self.assertTrue(result)
 
-        mock_irclient.node.set_power_state.assert_called_once_with(
-            COMPUTE_NODE, change_node_power_state.NodeState.POWERON.value)
+        mock_nodes[0].set_power_state.assert_called_once_with(
+            m_constants.PowerState.ON.value)
 
-    def test_execute_change_node_state_with_poweroff_target(
-            self, mock_ironic, mock_nova):
-        mock_irclient = mock_ironic.return_value
-        mock_nvclient = mock_nova.return_value
-        mock_get = mock.MagicMock()
-        mock_get.to_dict.return_value = {'running_vms': 0}
-        mock_nvclient.hypervisors.get.return_value = mock_get
+    def test_execute_change_node_state_with_poweroff_target(self):
         self.action.input_parameters["state"] = (
-            change_node_power_state.NodeState.POWEROFF.value)
-        mock_irclient.node.get.side_effect = [
-            mock.MagicMock(power_state='power on'),
-            mock.MagicMock(power_state='power on'),
-            mock.MagicMock(power_state='power off')]
+            m_constants.PowerState.OFF.value)
+
+        mock_nodes = [
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.ON),
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.ON),
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.OFF)
+        ]
+        self._metal_helper.get_node.side_effect = mock_nodes
+
         result = self.action.execute()
         self.assertTrue(result)
 
-        mock_irclient.node.set_power_state.assert_called_once_with(
-            COMPUTE_NODE, change_node_power_state.NodeState.POWEROFF.value)
+        mock_nodes[0].set_power_state.assert_called_once_with(
+            m_constants.PowerState.OFF.value)
 
-    def test_revert_change_node_state_with_poweron_target(
-            self, mock_ironic, mock_nova):
-        mock_irclient = mock_ironic.return_value
-        mock_nvclient = mock_nova.return_value
-        mock_get = mock.MagicMock()
-        mock_get.to_dict.return_value = {'running_vms': 0}
-        mock_nvclient.hypervisors.get.return_value = mock_get
+    def test_revert_change_node_state_with_poweron_target(self):
         self.action.input_parameters["state"] = (
-            change_node_power_state.NodeState.POWERON.value)
-        mock_irclient.node.get.side_effect = [
-            mock.MagicMock(power_state='power on'),
-            mock.MagicMock(power_state='power on'),
-            mock.MagicMock(power_state='power off')]
+            m_constants.PowerState.ON.value)
+
+        mock_nodes = [
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.ON),
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.ON),
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.OFF)
+        ]
+        self._metal_helper.get_node.side_effect = mock_nodes
+
         self.action.revert()
 
-        mock_irclient.node.set_power_state.assert_called_once_with(
-            COMPUTE_NODE, change_node_power_state.NodeState.POWEROFF.value)
+        mock_nodes[0].set_power_state.assert_called_once_with(
+            m_constants.PowerState.OFF.value)
 
-    def test_revert_change_node_state_with_poweroff_target(
-            self, mock_ironic, mock_nova):
-        mock_irclient = mock_ironic.return_value
+    def test_revert_change_node_state_with_poweroff_target(self):
         self.action.input_parameters["state"] = (
-            change_node_power_state.NodeState.POWEROFF.value)
-        mock_irclient.node.get.side_effect = [
-            mock.MagicMock(power_state='power off'),
-            mock.MagicMock(power_state='power on')]
+            m_constants.PowerState.OFF.value)
+        mock_nodes = [
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.OFF),
+            fake_metal_helper.get_mock_metal_node(
+                power_state=m_constants.PowerState.ON)
+        ]
+        self._metal_helper.get_node.side_effect = mock_nodes
+
         self.action.revert()
 
-        mock_irclient.node.set_power_state.assert_called_once_with(
-            COMPUTE_NODE, change_node_power_state.NodeState.POWERON.value)
+        mock_nodes[0].set_power_state.assert_called_once_with(
+            m_constants.PowerState.ON.value)

watcher/tests/applier/actions/test_volume_migration.py

@@ -22,7 +22,6 @@ from watcher.common import cinder_helper
 from watcher.common import clients
 from watcher.common import keystone_helper
 from watcher.common import nova_helper
-from watcher.common import utils as w_utils
 from watcher.tests import base
 
 
@@ -102,12 +101,15 @@ class TestMigration(base.TestCase):
 
     @staticmethod
     def fake_volume(**kwargs):
+        # FIXME(sean-k-mooney): we should be using real objects in this
+        # test or at lease something more Representative of the real data
         volume = mock.MagicMock()
         volume.id = kwargs.get('id', TestMigration.VOLUME_UUID)
         volume.size = kwargs.get('size', '1')
         volume.status = kwargs.get('status', 'available')
         volume.snapshot_id = kwargs.get('snapshot_id', None)
         volume.availability_zone = kwargs.get('availability_zone', 'nova')
+        volume.attachments = kwargs.get('attachments', [])
         return volume
 
     @staticmethod
@@ -175,42 +177,14 @@ class TestMigration(base.TestCase):
             "storage1-typename",
         )
 
-    def test_swap_success(self):
-        volume = self.fake_volume(
-            status='in-use', attachments=[{'server_id': 'server_id'}])
-        self.m_n_helper.find_instance.return_value = self.fake_instance()
-
-        new_volume = self.fake_volume(id=w_utils.generate_uuid())
-        user = mock.Mock()
-        session = mock.MagicMock()
-        self.m_k_helper.create_user.return_value = user
-        self.m_k_helper.create_session.return_value = session
-        self.m_c_helper.get_volume.return_value = volume
-        self.m_c_helper.create_volume.return_value = new_volume
-
-        result = self.action_swap.execute()
-        self.assertTrue(result)
-
-        self.m_n_helper.swap_volume.assert_called_once_with(
-            volume,
-            new_volume
-        )
-        self.m_k_helper.delete_user.assert_called_once_with(user)
-
-    def test_swap_fail(self):
-        # _can_swap fail
-        instance = self.fake_instance(status='STOPPED')
-        self.m_n_helper.find_instance.return_value = instance
-
-        result = self.action_swap.execute()
-        self.assertFalse(result)
-
     def test_can_swap_success(self):
         volume = self.fake_volume(
-            status='in-use', attachments=[{'server_id': 'server_id'}])
-        instance = self.fake_instance()
+            status='in-use', attachments=[
+                {'server_id': TestMigration.INSTANCE_UUID}])
 
+        instance = self.fake_instance()
         self.m_n_helper.find_instance.return_value = instance
+
         result = self.action_swap._can_swap(volume)
         self.assertTrue(result)
 
@@ -219,16 +193,33 @@ class TestMigration(base.TestCase):
         result = self.action_swap._can_swap(volume)
         self.assertTrue(result)
 
-        instance = self.fake_instance(status='RESIZED')
-        self.m_n_helper.find_instance.return_value = instance
-        result = self.action_swap._can_swap(volume)
-        self.assertTrue(result)
-
     def test_can_swap_fail(self):
 
         volume = self.fake_volume(
-            status='in-use', attachments=[{'server_id': 'server_id'}])
+            status='in-use', attachments=[
+                {'server_id': TestMigration.INSTANCE_UUID}])
         instance = self.fake_instance(status='STOPPED')
         self.m_n_helper.find_instance.return_value = instance
         result = self.action_swap._can_swap(volume)
         self.assertFalse(result)
+
+        instance = self.fake_instance(status='RESIZED')
+        self.m_n_helper.find_instance.return_value = instance
+        result = self.action_swap._can_swap(volume)
+        self.assertFalse(result)
+
+    def test_swap_success(self):
+        volume = self.fake_volume(
+            status='in-use', attachments=[
+                {'server_id': TestMigration.INSTANCE_UUID}])
+        self.m_c_helper.get_volume.return_value = volume
+
+        instance = self.fake_instance()
+        self.m_n_helper.find_instance.return_value = instance
+
+        result = self.action_swap.execute()
+        self.assertTrue(result)
+        self.m_c_helper.migrate.assert_called_once_with(
+            volume,
+            "storage1-poolname"
+        )

watcher/tests/common/metal_helper/test_base.py

@@ -0,0 +1,96 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+from watcher.common import exception
+from watcher.common.metal_helper import base as m_helper_base
+from watcher.common.metal_helper import constants as m_constants
+from watcher.tests import base
+
+
+# The base classes have abstract methods, we'll need to
+# stub them.
+class MockMetalNode(m_helper_base.BaseMetalNode):
+    def get_power_state(self):
+        raise NotImplementedError()
+
+    def get_id(self):
+        raise NotImplementedError()
+
+    def power_on(self):
+        raise NotImplementedError()
+
+    def power_off(self):
+        raise NotImplementedError()
+
+
+class MockMetalHelper(m_helper_base.BaseMetalHelper):
+    def list_compute_nodes(self):
+        pass
+
+    def get_node(self, node_id):
+        pass
+
+
+class TestBaseMetalNode(base.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self._nova_node = mock.Mock()
+        self._node = MockMetalNode(self._nova_node)
+
+    def test_get_hypervisor_node(self):
+        self.assertEqual(
+            self._nova_node,
+            self._node.get_hypervisor_node())
+
+    def test_get_hypervisor_node_missing(self):
+        node = MockMetalNode()
+        self.assertRaises(
+            exception.Invalid,
+            node.get_hypervisor_node)
+
+    def test_get_hypervisor_hostname(self):
+        self.assertEqual(
+            self._nova_node.hypervisor_hostname,
+            self._node.get_hypervisor_hostname())
+
+    @mock.patch.object(MockMetalNode, 'power_on')
+    @mock.patch.object(MockMetalNode, 'power_off')
+    def test_set_power_state(self,
+                             mock_power_off, mock_power_on):
+        self._node.set_power_state(m_constants.PowerState.ON)
+        mock_power_on.assert_called_once_with()
+
+        self._node.set_power_state(m_constants.PowerState.OFF)
+        mock_power_off.assert_called_once_with()
+
+        self.assertRaises(
+            exception.UnsupportedActionType,
+            self._node.set_power_state,
+            m_constants.PowerState.UNKNOWN)
+
+
+class TestBaseMetalHelper(base.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self._osc = mock.Mock()
+        self._helper = MockMetalHelper(self._osc)
+
+    def test_nova_client_attr(self):
+        self.assertEqual(self._osc.nova.return_value,
+                         self._helper.nova_client)

watcher/tests/common/metal_helper/test_factory.py

@@ -0,0 +1,38 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+from watcher.common import clients
+from watcher.common.metal_helper import factory
+from watcher.common.metal_helper import ironic
+from watcher.common.metal_helper import maas
+from watcher.tests import base
+
+
+class TestMetalHelperFactory(base.TestCase):
+
+    @mock.patch.object(clients, 'OpenStackClients')
+    @mock.patch.object(maas, 'MaasHelper')
+    @mock.patch.object(ironic, 'IronicHelper')
+    def test_factory(self, mock_ironic, mock_maas, mock_osc):
+        self.assertEqual(
+            mock_ironic.return_value,
+            factory.get_helper())
+
+        self.config(url="fake_maas_url", group="maas_client")
+        self.assertEqual(
+            mock_maas.return_value,
+            factory.get_helper())

watcher/tests/common/metal_helper/test_ironic.py

@@ -0,0 +1,128 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+from watcher.common.metal_helper import constants as m_constants
+from watcher.common.metal_helper import ironic
+from watcher.tests import base
+
+
+class TestIronicNode(base.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self._wrapped_node = mock.Mock()
+        self._nova_node = mock.Mock()
+        self._ironic_client = mock.Mock()
+
+        self._node = ironic.IronicNode(
+            self._wrapped_node, self._nova_node, self._ironic_client)
+
+    def test_get_power_state(self):
+        states = (
+            "power on",
+            "power off",
+            "rebooting",
+            "soft power off",
+            "soft reboot",
+            'SomeOtherState')
+        type(self._wrapped_node).power_state = mock.PropertyMock(
+            side_effect=states)
+
+        expected_states = (
+            m_constants.PowerState.ON,
+            m_constants.PowerState.OFF,
+            m_constants.PowerState.ON,
+            m_constants.PowerState.OFF,
+            m_constants.PowerState.ON,
+            m_constants.PowerState.UNKNOWN)
+
+        for expected_state in expected_states:
+            actual_state = self._node.get_power_state()
+            self.assertEqual(expected_state, actual_state)
+
+    def test_get_id(self):
+        self.assertEqual(
+            self._wrapped_node.uuid,
+            self._node.get_id())
+
+    def test_power_on(self):
+        self._node.power_on()
+        self._ironic_client.node.set_power_state.assert_called_once_with(
+            self._wrapped_node.uuid, "on")
+
+    def test_power_off(self):
+        self._node.power_off()
+        self._ironic_client.node.set_power_state.assert_called_once_with(
+            self._wrapped_node.uuid, "off")
+
+
+class TestIronicHelper(base.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self._mock_osc = mock.Mock()
+        self._mock_nova_client = self._mock_osc.nova.return_value
+        self._mock_ironic_client = self._mock_osc.ironic.return_value
+        self._helper = ironic.IronicHelper(osc=self._mock_osc)
+
+    def test_list_compute_nodes(self):
+        mock_machines = [
+            mock.Mock(
+                extra=dict(compute_node_id=mock.sentinel.compute_node_id)),
+            mock.Mock(
+                extra=dict(compute_node_id=mock.sentinel.compute_node_id2)),
+            mock.Mock(
+                extra=dict())
+        ]
+        mock_hypervisor = mock.Mock()
+
+        self._mock_ironic_client.node.list.return_value = mock_machines
+        self._mock_ironic_client.node.get.side_effect = mock_machines
+        self._mock_nova_client.hypervisors.get.side_effect = (
+            mock_hypervisor, None)
+
+        out_nodes = self._helper.list_compute_nodes()
+        self.assertEqual(1, len(out_nodes))
+
+        out_node = out_nodes[0]
+        self.assertIsInstance(out_node, ironic.IronicNode)
+        self.assertEqual(mock_hypervisor, out_node._nova_node)
+        self.assertEqual(mock_machines[0], out_node._ironic_node)
+        self.assertEqual(self._mock_ironic_client, out_node._ironic_client)
+
+    def test_get_node(self):
+        mock_machine = mock.Mock(
+            extra=dict(compute_node_id=mock.sentinel.compute_node_id))
+        self._mock_ironic_client.node.get.return_value = mock_machine
+
+        out_node = self._helper.get_node(mock.sentinel.id)
+
+        self.assertEqual(self._mock_nova_client.hypervisors.get.return_value,
+                         out_node._nova_node)
+        self.assertEqual(self._mock_ironic_client, out_node._ironic_client)
+        self.assertEqual(mock_machine, out_node._ironic_node)
+
+    def test_get_node_not_a_hypervisor(self):
+        mock_machine = mock.Mock(extra=dict(compute_node_id=None))
+        self._mock_ironic_client.node.get.return_value = mock_machine
+
+        out_node = self._helper.get_node(mock.sentinel.id)
+
+        self._mock_nova_client.hypervisors.get.assert_not_called()
+        self.assertIsNone(out_node._nova_node)
+        self.assertEqual(self._mock_ironic_client, out_node._ironic_client)
+        self.assertEqual(mock_machine, out_node._ironic_node)

watcher/tests/common/metal_helper/test_maas.py

@@ -0,0 +1,126 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+try:
+    from maas.client import enum as maas_enum
+except ImportError:
+    maas_enum = None
+
+from watcher.common.metal_helper import constants as m_constants
+from watcher.common.metal_helper import maas
+from watcher.tests import base
+
+
+class TestMaasNode(base.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self._wrapped_node = mock.Mock()
+        self._nova_node = mock.Mock()
+        self._maas_client = mock.Mock()
+
+        self._node = maas.MaasNode(
+            self._wrapped_node, self._nova_node, self._maas_client)
+
+    def test_get_power_state(self):
+        if not maas_enum:
+            self.skipTest("python-libmaas not intalled.")
+
+        self._wrapped_node.query_power_state.side_effect = (
+            maas_enum.PowerState.ON,
+            maas_enum.PowerState.OFF,
+            maas_enum.PowerState.ERROR,
+            maas_enum.PowerState.UNKNOWN,
+            'SomeOtherState')
+
+        expected_states = (
+            m_constants.PowerState.ON,
+            m_constants.PowerState.OFF,
+            m_constants.PowerState.ERROR,
+            m_constants.PowerState.UNKNOWN,
+            m_constants.PowerState.UNKNOWN)
+
+        for expected_state in expected_states:
+            actual_state = self._node.get_power_state()
+            self.assertEqual(expected_state, actual_state)
+
+    def test_get_id(self):
+        self.assertEqual(
+            self._wrapped_node.system_id,
+            self._node.get_id())
+
+    def test_power_on(self):
+        self._node.power_on()
+        self._wrapped_node.power_on.assert_called_once_with()
+
+    def test_power_off(self):
+        self._node.power_off()
+        self._wrapped_node.power_off.assert_called_once_with()
+
+
+class TestMaasHelper(base.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self._mock_osc = mock.Mock()
+        self._mock_nova_client = self._mock_osc.nova.return_value
+        self._mock_maas_client = self._mock_osc.maas.return_value
+        self._helper = maas.MaasHelper(osc=self._mock_osc)
+
+    def test_list_compute_nodes(self):
+        compute_fqdn = "compute-0"
+        # some other MAAS node, not a Nova node
+        ctrl_fqdn = "ctrl-1"
+
+        mock_machines = [
+            mock.Mock(fqdn=compute_fqdn,
+                      system_id=mock.sentinel.compute_node_id),
+            mock.Mock(fqdn=ctrl_fqdn,
+                      system_id=mock.sentinel.ctrl_node_id),
+        ]
+        mock_hypervisors = [
+            mock.Mock(hypervisor_hostname=compute_fqdn),
+        ]
+
+        self._mock_maas_client.machines.list.return_value = mock_machines
+        self._mock_nova_client.hypervisors.list.return_value = mock_hypervisors
+
+        out_nodes = self._helper.list_compute_nodes()
+        self.assertEqual(1, len(out_nodes))
+
+        out_node = out_nodes[0]
+        self.assertIsInstance(out_node, maas.MaasNode)
+        self.assertEqual(mock.sentinel.compute_node_id, out_node.get_id())
+        self.assertEqual(compute_fqdn, out_node.get_hypervisor_hostname())
+
+    def test_get_node(self):
+        mock_machine = mock.Mock(fqdn='compute-0')
+        self._mock_maas_client.machines.get.return_value = mock_machine
+
+        mock_compute_nodes = [
+            mock.Mock(hypervisor_hostname="compute-011"),
+            mock.Mock(hypervisor_hostname="compute-0"),
+            mock.Mock(hypervisor_hostname="compute-01"),
+        ]
+        self._mock_nova_client.hypervisors.search.return_value = (
+            mock_compute_nodes)
+
+        out_node = self._helper.get_node(mock.sentinel.id)
+
+        self.assertEqual(mock_compute_nodes[1], out_node._nova_node)
+        self.assertEqual(self._mock_maas_client, out_node._maas_client)
+        self.assertEqual(mock_machine, out_node._maas_node)

watcher/tests/common/test_cinder_helper.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 #
 
-import time
 from unittest import mock
 
+from http import HTTPStatus
+import time
+
 from cinderclient import exceptions as cinder_exception
 
 from watcher.common import cinder_helper
@@ -291,7 +293,8 @@ class TestCinderHelper(base.TestCase):
 
         volume = self.fake_volume()
         cinder_util.get_volume = mock.MagicMock()
-        cinder_util.get_volume.side_effect = cinder_exception.NotFound(404)
+        cinder_util.get_volume.side_effect =\
+            cinder_exception.NotFound(HTTPStatus.NOT_FOUND)
         result = cinder_util._can_get_volume(volume.id)
         self.assertFalse(result)
 
@@ -339,7 +342,7 @@ class TestCinderHelper(base.TestCase):
         cinder_util = cinder_helper.CinderHelper()
 
         volume = self.fake_volume()
-        side_effect = cinder_exception.NotFound(404)
+        side_effect = cinder_exception.NotFound(HTTPStatus.NOT_FOUND)
         cinder_util.cinder.volumes.get.side_effect = side_effect
         cinder_util.cinder.volumes.find.return_value = False
         result = cinder_util.get_volume(volume)

watcher/tests/common/test_clients.py

@@ -10,8 +10,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from ceilometerclient import client as ceclient
-import ceilometerclient.v2.client as ceclient_v2
 from unittest import mock
 
 from cinderclient import client as ciclient
@@ -270,56 +268,6 @@ class TestClients(base.TestCase):
         cinder_cached = osc.cinder()
         self.assertEqual(cinder, cinder_cached)
 
-    @mock.patch.object(ceclient, 'Client')
-    @mock.patch.object(clients.OpenStackClients, 'session')
-    def test_clients_ceilometer(self, mock_session, mock_call):
-        osc = clients.OpenStackClients()
-        osc._ceilometer = None
-        osc.ceilometer()
-        mock_call.assert_called_once_with(
-            CONF.ceilometer_client.api_version,
-            None,
-            endpoint_type=CONF.ceilometer_client.endpoint_type,
-            region_name=CONF.ceilometer_client.region_name,
-            session=mock_session)
-
-    @mock.patch.object(clients.OpenStackClients, 'session')
-    @mock.patch.object(ceclient_v2.Client, '_get_redirect_client')
-    def test_clients_ceilometer_diff_vers(self, mock_get_redirect_client,
-                                          mock_session):
-        '''ceilometerclient currently only has one version (v2)'''
-        mock_get_redirect_client.return_value = [mock.Mock(), mock.Mock()]
-        CONF.set_override('api_version', '2',
-                          group='ceilometer_client')
-        osc = clients.OpenStackClients()
-        osc._ceilometer = None
-        osc.ceilometer()
-        self.assertEqual(ceclient_v2.Client,
-                         type(osc.ceilometer()))
-
-    @mock.patch.object(clients.OpenStackClients, 'session')
-    @mock.patch.object(ceclient_v2.Client, '_get_redirect_client')
-    def test_clients_ceilometer_diff_endpoint(self, mock_get_redirect_client,
-                                              mock_session):
-        mock_get_redirect_client.return_value = [mock.Mock(), mock.Mock()]
-        CONF.set_override('endpoint_type', 'publicURL',
-                          group='ceilometer_client')
-        osc = clients.OpenStackClients()
-        osc._ceilometer = None
-        osc.ceilometer()
-        self.assertEqual('publicURL', osc.ceilometer().http_client.interface)
-
-    @mock.patch.object(clients.OpenStackClients, 'session')
-    @mock.patch.object(ceclient_v2.Client, '_get_redirect_client')
-    def test_clients_ceilometer_cached(self, mock_get_redirect_client,
-                                       mock_session):
-        mock_get_redirect_client.return_value = [mock.Mock(), mock.Mock()]
-        osc = clients.OpenStackClients()
-        osc._ceilometer = None
-        ceilometer = osc.ceilometer()
-        ceilometer_cached = osc.ceilometer()
-        self.assertEqual(ceilometer, ceilometer_cached)
-
     @mock.patch.object(netclient, 'Client')
     @mock.patch.object(clients.OpenStackClients, 'session')
     def test_clients_neutron(self, mock_session, mock_call):

watcher/tests/common/test_placement_helper.py

@@ -11,6 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from http import HTTPStatus
 from unittest import mock
 
 from watcher.common import placement_helper
@@ -55,10 +56,10 @@ class TestPlacementHelper(base.TestCase):
         kss_req.assert_called_once_with(url, method, **kwargs)
 
     def test_get(self, kss_req):
-        kss_req.return_value = fake_requests.FakeResponse(200)
+        kss_req.return_value = fake_requests.FakeResponse(HTTPStatus.OK)
         url = '/resource_providers'
         resp = self.client.get(url)
-        self.assertEqual(200, resp.status_code)
+        self.assertEqual(HTTPStatus.OK, resp.status_code)
         self._assert_keystone_called_once(kss_req, url, 'GET')
 
     def test_get_resource_providers_OK(self, kss_req):
@@ -76,7 +77,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_resource_providers(rp_name)
 
@@ -99,7 +100,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_resource_providers(rp_name)
 
@@ -110,7 +111,8 @@ class TestPlacementHelper(base.TestCase):
     def test_get_resource_providers_fail(self, kss_req):
         rp_name = 'compute'
         kss_req.return_value = fake_requests.FakeResponse(
-            400, content=jsonutils.dump_as_bytes(self.fake_err_msg))
+            HTTPStatus.BAD_REQUEST,
+            content=jsonutils.dump_as_bytes(self.fake_err_msg))
         result = self.client.get_resource_providers(rp_name)
         self.assertIsNone(result)
 
@@ -149,7 +151,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_inventories(rp_uuid)
 
@@ -160,7 +162,8 @@ class TestPlacementHelper(base.TestCase):
     def test_get_inventories_fail(self, kss_req):
         rp_uuid = uuidutils.generate_uuid()
         kss_req.return_value = fake_requests.FakeResponse(
-            404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
+            HTTPStatus.NOT_FOUND,
+            content=jsonutils.dump_as_bytes(self.fake_err_msg))
         result = self.client.get_inventories(rp_uuid)
         self.assertIsNone(result)
 
@@ -175,7 +178,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_provider_traits(rp_uuid)
 
@@ -186,7 +189,8 @@ class TestPlacementHelper(base.TestCase):
     def test_get_provider_traits_fail(self, kss_req):
         rp_uuid = uuidutils.generate_uuid()
         kss_req.return_value = fake_requests.FakeResponse(
-            404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
+            HTTPStatus.NOT_FOUND,
+            content=jsonutils.dump_as_bytes(self.fake_err_msg))
         result = self.client.get_provider_traits(rp_uuid)
         self.assertIsNone(result)
 
@@ -216,7 +220,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_allocations_for_consumer(c_uuid)
 
@@ -227,7 +231,8 @@ class TestPlacementHelper(base.TestCase):
     def test_get_allocations_for_consumer_fail(self, kss_req):
         c_uuid = uuidutils.generate_uuid()
         kss_req.return_value = fake_requests.FakeResponse(
-            404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
+            HTTPStatus.NOT_FOUND,
+            content=jsonutils.dump_as_bytes(self.fake_err_msg))
         result = self.client.get_allocations_for_consumer(c_uuid)
         self.assertIsNone(result)
 
@@ -245,7 +250,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_usages_for_resource_provider(rp_uuid)
 
@@ -256,7 +261,8 @@ class TestPlacementHelper(base.TestCase):
     def test_get_usages_for_resource_provider_fail(self, kss_req):
         rp_uuid = uuidutils.generate_uuid()
         kss_req.return_value = fake_requests.FakeResponse(
-            404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
+            HTTPStatus.NOT_FOUND,
+            content=jsonutils.dump_as_bytes(self.fake_err_msg))
         result = self.client.get_usages_for_resource_provider(rp_uuid)
         self.assertIsNone(result)
 
@@ -296,7 +302,7 @@ class TestPlacementHelper(base.TestCase):
         }
 
         kss_req.return_value = fake_requests.FakeResponse(
-            200, content=jsonutils.dump_as_bytes(mock_json_data))
+            HTTPStatus.OK, content=jsonutils.dump_as_bytes(mock_json_data))
 
         result = self.client.get_candidate_providers(resources)
 
@@ -307,6 +313,7 @@ class TestPlacementHelper(base.TestCase):
     def test_get_candidate_providers_fail(self, kss_req):
         rp_uuid = uuidutils.generate_uuid()
         kss_req.return_value = fake_requests.FakeResponse(
-            404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
+            HTTPStatus.NOT_FOUND,
+            content=jsonutils.dump_as_bytes(self.fake_err_msg))
         result = self.client.get_candidate_providers(rp_uuid)
         self.assertIsNone(result)

watcher/tests/common/test_service.py

@@ -80,13 +80,13 @@ class TestService(base.TestCase):
         super(TestService, self).setUp()
 
     @mock.patch.object(om.rpc.server, "RPCServer")
-    def test_start(self, m_handler):
+    def _test_start(self, m_handler):
         dummy_service = service.Service(DummyManager)
         dummy_service.start()
         self.assertEqual(1, m_handler.call_count)
 
     @mock.patch.object(om.rpc.server, "RPCServer")
-    def test_stop(self, m_handler):
+    def _test_stop(self, m_handler):
         dummy_service = service.Service(DummyManager)
         dummy_service.stop()
         self.assertEqual(1, m_handler.call_count)

watcher/tests/common/test_utils.py

@@ -0,0 +1,52 @@
+# Copyright 2023 Cloudbase Solutions
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import asyncio
+import time
+from unittest import mock
+
+from watcher.common import utils
+from watcher.tests import base
+
+
+class TestCommonUtils(base.TestCase):
+    async def test_coro(self, sleep=0, raise_exc=None):
+        time.sleep(sleep)
+        if raise_exc:
+            raise raise_exc
+        return mock.sentinel.ret_val
+
+    def test_async_compat(self):
+        ret_val = utils.async_compat_call(self.test_coro)
+        self.assertEqual(mock.sentinel.ret_val, ret_val)
+
+    def test_async_compat_exc(self):
+        self.assertRaises(
+            IOError,
+            utils.async_compat_call,
+            self.test_coro,
+            raise_exc=IOError('fake error'))
+
+    def test_async_compat_timeout(self):
+        # Timeout not reached.
+        ret_val = utils.async_compat_call(self.test_coro, timeout=10)
+        self.assertEqual(mock.sentinel.ret_val, ret_val)
+
+        # Timeout reached.
+        self.assertRaises(
+            asyncio.TimeoutError,
+            utils.async_compat_call,
+            self.test_coro,
+            sleep=0.5, timeout=0.1)

watcher/tests/db/base.py

@@ -17,9 +17,10 @@
 
 import fixtures
 from oslo_config import cfg
+from oslo_db.sqlalchemy import enginefacade
+
 
 from watcher.db import api as dbapi
-from watcher.db.sqlalchemy import api as sqla_api
 from watcher.db.sqlalchemy import migration
 from watcher.db.sqlalchemy import models
 from watcher.tests import base
@@ -35,16 +36,16 @@ _DB_CACHE = None
 
 class Database(fixtures.Fixture):
 
-    def __init__(self, db_api, db_migrate, sql_connection):
+    def __init__(self, engine, db_migrate, sql_connection):
         self.sql_connection = sql_connection
 
-        self.engine = db_api.get_engine()
+        self.engine = engine
         self.engine.dispose()
-        conn = self.engine.connect()
-        self.setup_sqlite(db_migrate)
-        self.post_migrations()
 
-        self._DB = "".join(line for line in conn.connection.iterdump())
+        with self.engine.connect() as conn:
+            self.setup_sqlite(db_migrate)
+            self.post_migrations()
+            self._DB = "".join(line for line in conn.connection.iterdump())
         self.engine.dispose()
 
     def setup_sqlite(self, db_migrate):
@@ -55,9 +56,8 @@ class Database(fixtures.Fixture):
 
     def setUp(self):
         super(Database, self).setUp()
-
-        conn = self.engine.connect()
-        conn.connection.executescript(self._DB)
+        with self.engine.connect() as conn:
+            conn.connection.executescript(self._DB)
         self.addCleanup(self.engine.dispose)
 
     def post_migrations(self):
@@ -80,7 +80,9 @@ class DbTestCase(base.TestCase):
 
         global _DB_CACHE
         if not _DB_CACHE:
-            _DB_CACHE = Database(sqla_api, migration,
+            engine = enginefacade.writer.get_engine()
+            _DB_CACHE = Database(engine, migration,
                                  sql_connection=CONF.database.connection)
+            engine.dispose()
         self.useFixture(_DB_CACHE)
         self._id_gen = utils.id_generator()

watcher/tests/decision_engine/audit/test_audit_handlers.py

@@ -263,7 +263,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
         cfg.CONF.set_override("host", "hostname1")
 
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'add_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
     @mock.patch.object(objects.audit.Audit, 'list')
@@ -286,7 +286,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
         self.assertIsNone(self.audits[1].next_run_time)
 
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'add_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
     @mock.patch.object(objects.audit.Audit, 'list')
@@ -309,7 +309,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
 
     @mock.patch.object(continuous.ContinuousAuditHandler, '_next_cron_time')
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'add_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
     @mock.patch.object(objects.audit.Audit, 'list')
@@ -328,7 +328,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
                           audit_handler.launch_audits_periodically)
 
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'add_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
     @mock.patch.object(objects.audit.Audit, 'list')
@@ -349,7 +349,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
         m_add_job.assert_has_calls(calls)
 
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'add_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
     @mock.patch.object(objects.audit.Audit, 'list')
@@ -384,7 +384,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
         self.assertTrue(is_inactive)
 
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
     @mock.patch.object(objects.audit.AuditStateTransitionManager,
                        'is_inactive')
@@ -406,7 +406,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
         self.assertIsNotNone(self.audits[0].next_run_time)
 
     @mock.patch.object(objects.service.Service, 'list')
-    @mock.patch.object(sq_api, 'get_engine')
+    @mock.patch.object(sq_api.enginefacade.writer, 'get_engine')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'remove_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'add_job')
     @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')