Merge "use cinder migrate for swap volume" into stable/2024.1

Adds a tempest configuration for min and max microversions supported
by watcher. This help us to define the correct range of microversion
to be tested on each stable branch.
New microversion proposals should also increase the default
max_microversion, in order to work with watcher-tempest-plugin
microversion testing.

Change-Id: I0b695ba4530eb89ed17b3935b87e938cadec84cc
(cherry picked from commit adfe3858aa)
(cherry picked from commit defd3953d8)
(cherry picked from commit b5b1bc5473)
Signed-off-by: Douglas Viroel <viroel@gmail.com>

.gitreview

@@ -1,4 +1,5 @@
 [gerrit]
-host=review.openstack.org
+host=review.opendev.org
 port=29418
 project=openstack/watcher.git
+defaultbranch=stable/2024.1

.zuul.yaml

@@ -1,57 +1,45 @@
 - project:
+    queue: watcher
     templates:
       - check-requirements
       - openstack-cover-jobs
-      - openstack-lower-constraints-jobs
+      - openstack-python3-jobs
-      - openstack-python-jobs
-      - openstack-python35-jobs
-      - openstack-python36-jobs
       - publish-openstack-docs-pti
       - release-notes-jobs-python3
     check:
       jobs:
         - watcher-tempest-functional
         - watcher-grenade
-        - watcher-tempest-dummy_optim
+        - watcher-tempest-strategies
         - watcher-tempest-actuator
-        - watcher-tempest-basic_optim
-        - watcher-tempest-vm_workload_consolidation
-        - watcher-tempest-workload_balancing
         - watcherclient-tempest-functional
-        - watcher-tempest-zone_migration
+        - watcher-tempest-functional-ipv6-only
-        - watcher-tempest-host_maintenance
-        - watcher-tempest-storage_balance
-        - watcher-tls-test
     gate:
-      queue: watcher
       jobs:
         - watcher-tempest-functional
+        - watcher-tempest-functional-ipv6-only
 
 - job:
     name: watcher-tempest-dummy_optim
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_dummy_optim
 
 - job:
     name: watcher-tempest-actuator
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_actuator
 
 - job:
     name: watcher-tempest-basic_optim
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_basic_optim
 
 - job:
     name: watcher-tempest-vm_workload_consolidation
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation
       devstack_local_conf:
@@ -63,28 +51,24 @@
 - job:
     name: watcher-tempest-workload_balancing
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_workload_balancing
 
 - job:
     name: watcher-tempest-zone_migration
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_zone_migration
 
 - job:
     name: watcher-tempest-host_maintenance
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance
 
 - job:
     name: watcher-tempest-storage_balance
     parent: watcher-tempest-multinode
-    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_storage_balance
       devstack_local_conf:
@@ -96,20 +80,17 @@
               multi_backend: true
 
 - job:
-    name: watcher-tls-test
+    name: watcher-tempest-strategies
     parent: watcher-tempest-multinode
-    group-vars:
-      subnode:
-          devstack_services:
-            tls-proxy: true
     vars:
-      devstack_services:
+      tempest_concurrency: 1
-        tls-proxy: true
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_strategies
+      tempest_exclude_regex: .*\[.*\breal_load\b.*\].*
 
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
-    nodeset: openstack-two-node
+    nodeset: openstack-two-node-jammy
     roles:
       - zuul: openstack/tempest
     group-vars:
@@ -127,8 +108,7 @@
           watcher-api: false
           watcher-decision-engine: true
           watcher-applier: false
-          # We need to add TLS support for watcher plugin
+          c-bak: false
-          tls-proxy: false
           ceilometer: false
           ceilometer-acompute: false
           ceilometer-acentral: false
@@ -152,29 +132,30 @@
           $TEMPEST_CONFIG:
             compute:
               min_compute_nodes: 2
+              min_microversion: 2.56
             compute-feature-enabled:
               live_migration: true
               block_migration_for_live_migration: true
+            placement:
+              min_microversion: 1.29
       devstack_plugins:
-        ceilometer: https://git.openstack.org/openstack/ceilometer
+        ceilometer: https://opendev.org/openstack/ceilometer
 
 - job:
     name: watcher-tempest-functional
     parent: devstack-tempest
     timeout: 7200
-    required-projects:
+    required-projects: &base_required_projects
       - openstack/ceilometer
-      - openstack-infra/devstack-gate
       - openstack/python-openstackclient
       - openstack/python-watcherclient
       - openstack/watcher
       - openstack/watcher-tempest-plugin
       - openstack/tempest
-    vars:
+    vars: &base_vars
       devstack_plugins:
-        watcher: https://git.openstack.org/openstack/watcher
+        watcher: https://opendev.org/openstack/watcher
       devstack_services:
-        tls-proxy: false
         watcher-api: true
         watcher-decision-engine: true
         watcher-applier: true
@@ -183,22 +164,29 @@
         s-container: false
         s-object: false
         s-proxy: false
-      devstack_localrc:
+      tempest_plugins:
-        TEMPEST_PLUGINS: /opt/stack/watcher-tempest-plugin
+        - watcher-tempest-plugin
       tempest_test_regex: watcher_tempest_plugin.tests.api
       tox_envlist: all
-      tox_environment:
-        # Do we really need to set this? It's cargo culted
-        PYTHONUNBUFFERED: 'true'
       zuul_copy_output:
         /etc/hosts: logs
 
+- job:
+    name: watcher-tempest-functional-ipv6-only
+    parent: devstack-tempest-ipv6
+    description: |
+      Watcher devstack tempest tests job for IPv6-only deployment
+    required-projects: *base_required_projects
+    vars: *base_vars
+
 - job:
     name: watcher-grenade
-    parent: legacy-dsvm-base
+    parent: grenade
-    timeout: 10800
+    required-projects:
-    run: playbooks/legacy/grenade-devstack-watcher/run.yaml
+      - openstack/watcher
-    post-run: playbooks/legacy/grenade-devstack-watcher/post.yaml
+      - openstack/python-watcherclient
+      - openstack/watcher-tempest-plugin
+    vars: *base_vars
     irrelevant-files:
       - ^(test-|)requirements.txt$
       - ^.*\.rst$
@@ -210,18 +198,11 @@
       - ^setup.cfg$
       - ^tools/.*$
       - ^tox.ini$
-    required-projects:
-      - openstack-dev/grenade
-      - openstack-infra/devstack-gate
-      - openstack/watcher
-      - openstack/python-watcherclient
-      - openstack/watcher-tempest-plugin
 
 - job:
     # This job is used in python-watcherclient repo
     name: watcherclient-tempest-functional
     parent: watcher-tempest-functional
-    voting: false
     timeout: 4200
     vars:
       tempest_concurrency: 1

README.rst

@@ -1,6 +1,6 @@
-========================
+=======
-Team and repository tags
+Watcher
-========================
+=======
 
 .. image:: https://governance.openstack.org/tc/badges/watcher.svg
     :target: https://governance.openstack.org/tc/reference/tags/index.html
@@ -13,10 +13,6 @@ Team and repository tags
 
           https://creativecommons.org/licenses/by/3.0/
 
-=======
-Watcher
-=======
-
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
 Watcher provides a robust framework to realize a wide range of cloud
@@ -26,7 +22,8 @@ migration, increased energy efficiency and more!
 
 * Free software: Apache license
 * Wiki: https://wiki.openstack.org/wiki/Watcher
-* Source: https://github.com/openstack/watcher
+* Source: https://opendev.org/openstack/watcher
 * Bugs: https://bugs.launchpad.net/watcher
 * Documentation: https://docs.openstack.org/watcher/latest/
 * Release notes: https://docs.openstack.org/releasenotes/watcher/
+* Design specifications: https://specs.openstack.org/openstack/watcher-specs/

api-ref/source/conf.py

@@ -22,9 +22,6 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-from watcher import version as watcher_version
-
-
 extensions = [
     'openstackdocstheme',
     'os_api_ref',
@@ -46,21 +43,13 @@ project = u'Infrastructure Optimization API Reference'
 copyright = u'2010-present, OpenStack Foundation'
 
 # openstackdocstheme options
-repository_name = 'openstack/watcher'
+openstackdocs_repo_name = 'openstack/watcher'
-bug_project = 'watcher'
+openstackdocs_auto_name = False
-bug_tag = ''
+openstackdocs_bug_project = 'watcher'
-
+openstackdocs_bug_tag = ''
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The full version, including alpha/beta/rc tags.
-release = watcher_version.version_info.release_string()
-# The short X.Y version.
-version = watcher_version.version_string
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # -- Options for HTML output --------------------------------------------------
 
@@ -75,10 +64,6 @@ html_theme_options = {
     "sidebar_mode": "toc",
 }
 
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%Y-%m-%d %H:%M'
-
 # -- Options for LaTeX output -------------------------------------------------
 
 # Grouping the document tree into LaTeX files. List of tuples

api-ref/source/index.rst

@@ -15,3 +15,5 @@ Watcher API
 .. include:: watcher-api-v1-strategies.inc
 .. include:: watcher-api-v1-services.inc
 .. include:: watcher-api-v1-scoring_engines.inc
+.. include:: watcher-api-v1-datamodel.inc
+.. include:: watcher-api-v1-webhooks.inc

api-ref/source/parameters.yaml

@@ -129,6 +129,14 @@ r_strategy:
   in: query
   required: false
   type: string
+r_type:
+  description: |
+    Type of data model user want to list. Default type is compute.
+    Supported values: compute.
+    Future support values: storage, baremetal.
+  in: query
+  required: false
+  type: string
 sort_dir:
   description: |
     Sorts the response by the requested sort direction.
@@ -245,6 +253,13 @@ audit_endtime_resp:
   required: false
   type: string
   min_version: 1.1
+audit_force:
+  description: |
+    Launch audit even if action plan is ongoing.
+  in: body
+  required: false
+  type: boolean
+  min_version: 1.2
 audit_goal:
   description: |
     The UUID or name of the Goal.
@@ -404,6 +419,62 @@ links:
   required: true
   type: array
 
+# Data Model Node
+node_disk:
+  description: |
+    The Disk of the node(in GiB).
+  in: body
+  required: true
+  type: integer
+node_disk_ratio:
+  description: |
+    The Disk Ratio of the node.
+  in: body
+  required: true
+  type: float
+node_hostname:
+  description: |
+    The Host Name of the node.
+  in: body
+  required: true
+  type: string
+node_memory:
+  description: |
+    The Memory of the node(in MiB).
+  in: body
+  required: true
+  type: integer
+node_memory_ratio:
+  description: |
+    The Memory Ratio of the node.
+  in: body
+  required: true
+  type: float
+node_state:
+  description: |
+    The State of the node. The value is up or down.
+  in: body
+  required: true
+  type: string
+node_uuid:
+  description: |
+    The Unique UUID of the node.
+  in: body
+  required: true
+  type: string
+node_vcpu_ratio:
+  description: |
+    The Vcpu ratio of the node.
+  in: body
+  required: true
+  type: float
+node_vcpus:
+  description: |
+    The Vcpu of the node.
+  in: body
+  required: true
+  type: integer
+
 # Scoring Engine
 scoring_engine_description:
   description: |
@@ -423,34 +494,72 @@ scoring_engine_name:
   in: body
   required: true
   type: string
+
+# Data Model Server
+server_disk:
+  description: |
+    The Disk of the server.
+  in: body
+  required: true
+  type: integer
+server_memory:
+  description: |
+    The Memory of server.
+  in: body
+  required: true
+  type: integer
+server_name:
+  description: |
+    The Name of the server.
+  in: body
+  required: true
+  type: string
+server_state:
+  description: |
+    The State of the server.
+  in: body
+  required: true
+  type: string
+server_uuid:
+  description: |
+    The Unique UUID of the server.
+  in: body
+  required: true
+  type: string
+server_vcpus:
+  description: |
+    The Vcpu of the server.
+  in: body
+  required: true
+  type: integer
 # Service
 service_host:
   description: |
-    Name of host where service is placed on.
+    The Name of host where service is placed on.
   in: body
   required: true
   type: string
 service_id:
   description: |
-    ID of service.
+    The ID of service.
   in: body
   required: true
   type: integer
 service_last_seen_up:
   description: |
-    Time when Watcher service sent latest heartbeat.
+    The Time when Watcher service sent latest heartbeat.
   in: body
   required: true
   type: string
 service_name:
   description: |
-    Name of service like ``watcher-applier``.
+    The Name of service like ``watcher-applier``.
   in: body
   required: true
   type: string
 service_status:
   description: |
-    State of service. It can be either in ACTIVE or FAILED state.
+    The State of service. It can be either in ACTIVE or FAILED state.
   in: body
   required: true
   type: string

api-ref/source/samples/audit-cancel-response.json

@@ -28,6 +28,7 @@
         }
     },
     "auto_trigger": false,
+    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],

api-ref/source/samples/audit-create-request-continuous.json

@@ -1,5 +1,6 @@
 {
     "auto_trigger": false,
+    "force": false,
     "audit_template_uuid": "76fddfee-a9c4-40b0-8da0-c19ad6904f09",
     "name": "test_audit",
     "parameters": {

api-ref/source/samples/audit-create-request-oneshot.json

@@ -1,5 +1,6 @@
 {
     "audit_type": "ONESHOT",
     "auto_trigger": false,
+    "force": true,
     "audit_template_uuid": "5e70a156-ced7-4012-b1c6-88fcb02ee0c1"
-}
+}

api-ref/source/samples/audit-create-response.json

@@ -28,6 +28,7 @@
         }
     },
     "auto_trigger": false,
+    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],

api-ref/source/samples/audit-list-detailed-response.json

@@ -30,6 +30,7 @@
                 }
             },
             "auto_trigger": false,
+	    "force": false,
             "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
             "goal_name": "workload_balancing",
             "scope": [],

api-ref/source/samples/audit-show-response.json

@@ -28,6 +28,7 @@
         }
     },
     "auto_trigger": false,
+    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],

api-ref/source/samples/audit-update-response.json

@@ -28,6 +28,7 @@
         }
     },
     "auto_trigger": false,
+    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],

api-ref/source/samples/datamodel-list-response.json

@@ -0,0 +1,38 @@
+{
+    "context": [
+        {
+            "server_uuid": "1bf91464-9b41-428d-a11e-af691e5563bb",
+            "server_name": "chenke-test1",
+            "server_vcpus": "1",
+            "server_memory": "512",
+            "server_disk": "1",
+            "server_state": "active",
+            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112",
+            "node_hostname": "localhost.localdomain",
+            "node_vcpus": "4",
+            "node_vcpu_ratio": "16.0",
+            "node_memory": "16383",
+            "node_memory_ratio": "1.5",
+            "node_disk": "37"
+            "node_disk_ratio": "1.0",
+            "node_state": "up",
+        },
+        {
+            "server_uuid": "e2cb5f6f-fa1d-4ba2-be1e-0bf02fa86ba4",
+            "server_name": "chenke-test2",
+            "server_vcpus": "1",
+            "server_memory": "512",
+            "server_disk": "1",
+            "server_state": "active",
+            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112",
+            "node_hostname": "localhost.localdomain",
+            "node_vcpus": "4",
+            "node_vcpu_ratio": "16.0",
+            "node_memory": "16383",
+            "node_memory_ratio": "1.5",
+            "node_disk": "37"
+            "node_disk_ratio": "1.0",
+            "node_state": "up",
+        }
+    ]
+}

api-ref/source/watcher-api-v1-audits.inc

@@ -47,6 +47,7 @@ Request
    - auto_trigger: audit_autotrigger
    - start_time: audit_starttime_req
    - end_time: audit_endtime_req
+   - force: audit_force
 
 **Example ONESHOT Audit creation request:**
 
@@ -83,6 +84,7 @@ version 1:
     - hostname: audit_hostname
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
+    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -181,6 +183,7 @@ Response
     - hostname: audit_hostname
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
+    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -227,6 +230,7 @@ Response
     - hostname: audit_hostname
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
+    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -281,6 +285,7 @@ version 1:
     - hostname: audit_hostname
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
+    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -335,6 +340,7 @@ Response
     - hostname: audit_hostname
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
+    - force: audit_force
 
 **Example JSON representation of an Audit:**
 

api-ref/source/watcher-api-v1-datamodel.inc

@@ -0,0 +1,57 @@
+.. -*- rst -*-
+
+==========
+Data Model
+==========
+
+.. versionadded:: 1.3
+
+``Data Model`` is very important for Watcher to generate resource
+optimization solutions. Users can easily view the data model by the
+API.
+
+
+List Data Model
+===============
+
+.. rest_method::  GET /v1/data_model
+
+Returns the information about Data Model.
+
+Normal response codes: 200
+
+Error codes: 400,401,406
+
+Request
+-------
+
+.. rest_parameters:: parameters.yaml
+
+   - audit: r_audit
+   - type: r_type
+
+Response
+--------
+
+.. rest_parameters:: parameters.yaml
+
+    - server_uuid: server_uuid
+    - server_name: server_name
+    - server_vcpus: server_vcpus
+    - server_memory: server_memory
+    - server_disk: server_disk
+    - server_state: server_state
+    - node_uuid: node_uuid
+    - node_hostname: node_hostname
+    - node_vcpus: node_vcpus
+    - node_vcpu_ratio: node_vcpu_ratio
+    - node_memory: node_memory
+    - node_memory_ratio: node_memory_ratio
+    - node_disk: node_disk
+    - node_disk_ratio: node_disk_ratio
+    - node_state: node_state
+
+**Example JSON representation of a Data Model:**
+
+.. literalinclude:: samples/datamodel-list-response.json
+   :language: javascript

api-ref/source/watcher-api-v1-webhooks.inc

@@ -0,0 +1,26 @@
+.. -*- rst -*-
+
+========
+Webhooks
+========
+
+.. versionadded:: 1.4
+
+Triggers an event based Audit.
+
+
+Trigger EVENT Audit
+===================
+
+.. rest_method::  POST /v1/webhooks/{audit_ident}
+
+Normal response codes: 202
+
+Error codes: 400,404
+
+Request
+-------
+
+.. rest_parameters:: parameters.yaml
+
+   - audit_ident: audit_ident

babel.cfg

@@ -1,2 +0,0 @@
-[python: **.py]
-

devstack/lib/watcher

@@ -51,7 +51,19 @@ if is_ssl_enabled_service "watcher" || is_service_enabled tls-proxy; then
     WATCHER_SERVICE_PROTOCOL="https"
 fi
 
-WATCHER_USE_MOD_WSGI=$(trueorfalse True WATCHER_USE_MOD_WSGI)
+# Support entry points installation of console scripts
+if [[ -d $WATCHER_DIR/bin ]]; then
+    WATCHER_BIN_DIR=$WATCHER_DIR/bin
+else
+    WATCHER_BIN_DIR=$(get_python_exec_prefix)
+fi
+
+# There are 2 modes, which is "uwsgi" which runs with an apache
+# proxy uwsgi in front of it, or "mod_wsgi", which runs in
+# apache. mod_wsgi is deprecated, don't use it.
+WATCHER_USE_WSGI_MODE=${WATCHER_USE_WSGI_MODE:-$WSGI_MODE}
+WATCHER_UWSGI=$WATCHER_BIN_DIR/watcher-api-wsgi
+WATCHER_UWSGI_CONF=$WATCHER_CONF_DIR/watcher-uwsgi.ini
 
 if is_suse; then
     WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/srv/www/htdocs/watcher}
@@ -59,16 +71,15 @@ else
     WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/var/www/watcher}
 fi
 # Public facing bits
-WATCHER_SERVICE_HOST=${WATCHER_SERVICE_HOST:-$HOST_IP}
+WATCHER_SERVICE_HOST=${WATCHER_SERVICE_HOST:-$SERVICE_HOST}
 WATCHER_SERVICE_PORT=${WATCHER_SERVICE_PORT:-9322}
 WATCHER_SERVICE_PORT_INT=${WATCHER_SERVICE_PORT_INT:-19322}
 WATCHER_SERVICE_PROTOCOL=${WATCHER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 
-# Support entry points installation of console scripts
+if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-if [[ -d $WATCHER_DIR/bin ]]; then
+    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
-    WATCHER_BIN_DIR=$WATCHER_DIR/bin
 else
-    WATCHER_BIN_DIR=$(get_python_exec_prefix)
+    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
 fi
 
 # Entry Points
@@ -93,7 +104,9 @@ function _cleanup_watcher_apache_wsgi {
 # runs that a clean run would need to clean up
 function cleanup_watcher {
     sudo rm -rf $WATCHER_STATE_PATH $WATCHER_AUTH_CACHE_DIR
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
+    else
         _cleanup_watcher_apache_wsgi
     fi
 }
@@ -139,15 +152,15 @@ function create_watcher_accounts {
         "infra-optim" "Watcher Infrastructure Optimization Service")
     get_or_create_endpoint $watcher_service \
         "$REGION_NAME" \
-        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" \
+        "$WATCHER_API_URL"\
-        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" \
+        "$WATCHER_API_URL"\
-        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
+        "$WATCHER_API_URL"
 }
 
 # _config_watcher_apache_wsgi() - Set WSGI config files of watcher
 function _config_watcher_apache_wsgi {
     local watcher_apache_conf
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
         local service_port=$WATCHER_SERVICE_PORT
         if is_service_enabled tls-proxy; then
             service_port=$WATCHER_SERVICE_PORT_INT
@@ -165,8 +178,6 @@ function _config_watcher_apache_wsgi {
             s|%APACHE_NAME%|$APACHE_NAME|g;
         " -i $watcher_apache_conf
         enable_apache_site watcher-api
-        tail_log watcher-access /var/log/$APACHE_NAME/watcher-api-access.log
-        tail_log watcher-api /var/log/$APACHE_NAME/watcher-api.log
    fi
 
 }
@@ -182,13 +193,17 @@ function create_watcher_conf {
     iniset_rpc_backend watcher $WATCHER_CONF
 
     iniset $WATCHER_CONF database connection $(database_connection_url watcher)
-    iniset $WATCHER_CONF api host "$WATCHER_SERVICE_HOST"
+    iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
 
     if is_service_enabled tls-proxy; then
+        iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
         iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT_INT"
         # iniset $WATCHER_CONF api enable_ssl_api "True"
     else
-        iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
+        if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
+            iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
+            iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
+        fi
     fi
 
     iniset $WATCHER_CONF oslo_policy policy_file $WATCHER_POLICY_YAML
@@ -214,15 +229,12 @@ function create_watcher_conf {
     fi
 
     # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
+    setup_logging $WATCHER_CONF
-        setup_colorized_logging $WATCHER_CONF DEFAULT
-    else
-        # Show user_name and project_name instead of user_id and project_id
-        iniset $WATCHER_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(project_domain)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
-    fi
 
     #config apache files
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
+    else
         _config_watcher_apache_wsgi
     fi
     # Register SSL certificates if provided
@@ -234,10 +246,6 @@ function create_watcher_conf {
 
         iniset $WATCHER_CONF DEFAULT enabled_ssl_apis "$WATCHER_ENABLED_APIS"
     fi
-
-    if is_service_enabled ceilometer; then
-        iniset $WATCHER_CONF watcher_messaging notifier_driver "messaging"
-    fi
 }
 
 # create_watcher_cache_dir() - Part of the init_watcher() process
@@ -273,7 +281,7 @@ function install_watcherclient {
 function install_watcher {
     git_clone $WATCHER_REPO $WATCHER_DIR $WATCHER_BRANCH
     setup_develop $WATCHER_DIR
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
         install_apache_wsgi
     fi
 }
@@ -284,24 +292,26 @@ function start_watcher_api {
 
     local service_port=$WATCHER_SERVICE_PORT
     local service_protocol=$WATCHER_SERVICE_PROTOCOL
+    local watcher_url
     if is_service_enabled tls-proxy; then
         service_port=$WATCHER_SERVICE_PORT_INT
         service_protocol="http"
     fi
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
+        watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
+    else
+        watcher_url=$service_protocol://$SERVICE_HOST:$service_port
         enable_apache_site watcher-api
         restart_apache_server
-    else
+        # Start proxies if enabled
-        run_process watcher-api "$WATCHER_BIN_DIR/watcher-api --config-file $WATCHER_CONF"
+        if is_service_enabled tls-proxy; then
-    fi
+            start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
-
+        fi
-    # Start proxies if enabled
-    if is_service_enabled tls-proxy; then
-        start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
     fi
 
     echo "Waiting for watcher-api to start..."
-    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$WATCHER_SERVICE_HOST:$service_port; then
+    if ! wait_for_service $SERVICE_TIMEOUT $watcher_url; then
         die $LINENO "watcher-api did not start"
     fi
 
@@ -317,17 +327,30 @@ function start_watcher {
 
 # stop_watcher() - Stop running processes (non-screen)
 function stop_watcher {
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        stop_process watcher-api
+    else
         disable_apache_site watcher-api
         restart_apache_server
-    else
-        stop_process watcher-api
     fi
     for serv in watcher-decision-engine watcher-applier; do
         stop_process $serv
     done
 }
 
+# configure_tempest_for_watcher() - Configure Tempest for watcher
+function configure_tempest_for_watcher {
+    # Set default microversion for watcher-tempest-plugin
+    # Please make sure to update this when the microversion is updated, otherwise
+    # new tests may be skipped.
+    TEMPEST_WATCHER_MIN_MICROVERSION=${TEMPEST_WATCHER_MIN_MICROVERSION:-"1.0"}
+    TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.4"}
+
+    # Set microversion options in tempest.conf
+    iniset $TEMPEST_CONFIG optimize min_microversion $TEMPEST_WATCHER_MIN_MICROVERSION
+    iniset $TEMPEST_CONFIG optimize max_microversion $TEMPEST_WATCHER_MAX_MICROVERSION
+}
+
 # Restore xtrace
 $_XTRACE_WATCHER
 

devstack/local.conf.compute

@@ -10,6 +10,7 @@ SERVICE_PASSWORD=$ADMIN_PASSWORD
 SERVICE_TOKEN=azertytoken
 
 HOST_IP=192.168.42.2 # Change this to this compute node's IP address
+#HOST_IPV6=2001:db8::7
 FLAT_INTERFACE=eth0
 
 FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is
@@ -30,12 +31,12 @@ ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client
 NOVA_VNC_ENABLED=True
 NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_auto.html"
 VNCSERVER_LISTEN=0.0.0.0
-VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP
+VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP # or HOST_IPV6
 
 NOVA_INSTANCES_PATH=/opt/stack/data/instances
 
 # Enable the Ceilometer plugin for the compute agent
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://opendev.org/openstack/ceilometer
 disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
 
 LOGFILE=$DEST/logs/stack.sh.log
@@ -44,3 +45,9 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
+[notifications]
+# Enable both versioned and unversioned notifications. Watcher only
+# uses versioned notifications but ceilometer uses unversioned. We
+# can change this to just versioned when ceilometer handles versioned
+# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
+notification_format=both

devstack/local.conf.controller

@@ -10,6 +10,7 @@ SERVICE_PASSWORD=$ADMIN_PASSWORD
 SERVICE_TOKEN=azertytoken
 
 HOST_IP=192.168.42.1 # Change this to your controller node IP address
+#HOST_IPV6=2001:db8::7
 FLAT_INTERFACE=eth0
 
 FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is
@@ -25,13 +26,13 @@ MULTI_HOST=1
 disable_service n-cpu
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
-enable_plugin watcher git://git.openstack.org/openstack/watcher
+enable_plugin watcher https://opendev.org/openstack/watcher
 
 # Enable the Ceilometer plugin
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://opendev.org/openstack/ceilometer
 
 # This is the controller node, so disable the ceilometer compute agent
 disable_service ceilometer-acompute
@@ -48,3 +49,9 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
+[notifications]
+# Enable both versioned and unversioned notifications. Watcher only
+# uses versioned notifications but ceilometer uses unversioned. We
+# can change this to just versioned when ceilometer handles versioned
+# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
+notification_format=both

devstack/override-defaults

@@ -0,0 +1,9 @@
+# Plug-in overrides
+# https://docs.openstack.org/devstack/latest/plugins.html#plugin-interface
+
+# Enable both versioned and unversioned notifications. Watcher only
+# uses versioned notifications but ceilometer uses unversioned. We
+# can change this to just versioned when ceilometer handles
+# versioned notifications from nova:
+# https://bugs.launchpad.net/ceilometer/+bug/1665449
+NOVA_NOTIFICATION_FORMAT=both

devstack/plugin.sh

@@ -38,6 +38,9 @@ if is_service_enabled watcher-api watcher-decision-engine watcher-applier; then
         # Start the watcher components
         echo_summary "Starting watcher"
         start_watcher
+    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
+        echo_summary "Configuring tempest for watcher"
+        configure_tempest_for_watcher
     fi
 
     if [[ "$1" == "unstack" ]]; then

devstack/upgrade/settings

@@ -1,11 +1,18 @@
 register_project_for_upgrade watcher
 register_db_to_save watcher
 
-devstack_localrc base enable_plugin watcher https://git.openstack.org/openstack/watcher stable/rocky
+devstack_localrc base enable_plugin watcher https://opendev.org/openstack/watcher $BASE_DEVSTACK_BRANCH
-devstack_localrc target enable_plugin watcher https://git.openstack.org/openstack/watcher
+devstack_localrc target enable_plugin watcher https://opendev.org/openstack/watcher
 
 devstack_localrc base enable_service watcher-api watcher-decision-engine watcher-applier
 devstack_localrc target enable_service watcher-api watcher-decision-engine watcher-applier
 
 BASE_RUN_SMOKE=False
 TARGET_RUN_SMOKE=False
+
+# Enable both versioned and unversioned notifications. Watcher only
+# uses versioned notifications but ceilometer uses unversioned. We
+# can change this to just versioned when ceilometer handles
+# versioned notifications from nova:
+# https://bugs.launchpad.net/ceilometer/+bug/1665449
+devstack_localrc base NOVA_NOTIFICATION_FORMAT=both

devstack/upgrade/upgrade.sh

@@ -40,6 +40,10 @@ set -o errexit
 source $TARGET_DEVSTACK_DIR/stackrc
 source $TARGET_DEVSTACK_DIR/lib/apache
 source $TARGET_DEVSTACK_DIR/lib/tls
+source $TARGET_DEVSTACK_DIR/lib/keystone
+
+source $TOP_DIR/openrc admin admin
+
 source $(dirname $(dirname $BASH_SOURCE))/settings
 source $(dirname $(dirname $BASH_SOURCE))/plugin.sh
 
@@ -56,6 +60,15 @@ install_watcher
 # calls upgrade-watcher for specific release
 upgrade_project watcher $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
 
+if [[ ! -f "$WATCHER_UWSGI_CONF" ]] && [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]
+then write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
+     endpoints=$(openstack endpoint list --service watcher -c ID -f value)
+     for id in $endpoints; do
+         openstack endpoint delete $id
+     done
+     create_watcher_accounts
+fi
+
 # Migrate the database
 watcher-db-manage upgrade || die $LINO "DB migration error"
 

doc/ext/term.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import unicode_literals
-
 import importlib
 import inspect
 

doc/requirements.txt

@@ -1,9 +1,10 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
-openstackdocstheme>=1.20.0 # Apache-2.0
+openstackdocstheme>=2.2.1 # Apache-2.0
-sphinx>=1.6.5,!=1.6.6,!=1.6.7 # BSD
+sphinx>=2.0.0,!=2.1.0 # BSD
 sphinxcontrib-pecanwsme>=0.8.0 # Apache-2.0
-reno>=2.7.0 # Apache-2.0
+sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD
+reno>=3.1.0 # Apache-2.0
 sphinxcontrib-apidoc>=0.2.0  # BSD
 os-api-ref>=1.4.0 # Apache-2.0

doc/source/admin/index.rst

@@ -8,5 +8,7 @@ Administrator Guide
    apache-mod-wsgi
    gmr
    policy
-   ways-to-install
    ../strategies/index
+   ../datasources/index
+   ../contributor/notifications
+   ../contributor/concurrency

doc/source/admin/policy.rst

@@ -17,6 +17,14 @@
 Policies
 ========
 
+.. warning::
+
+   JSON formatted policy file is deprecated since Watcher 6.0.0 (Wallaby).
+   This `oslopolicy-convert-json-to-yaml`__ tool will migrate your existing
+   JSON-formatted policy file to YAML in a backward-compatible way.
+
+.. __: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html
+
 Watcher's public API calls may be restricted to certain sets of users using a
 policy configuration file. This document explains exactly how policies are
 configured and what they apply to.

doc/source/architecture.rst

@@ -281,11 +281,13 @@ previously created :ref:`Audit template <audit_template_definition>`:
    :width: 100%
 
 The :ref:`Administrator <administrator_definition>` also can specify type of
-Audit and interval (in case of CONTINUOUS type). There is two types of Audit:
+Audit and interval (in case of CONTINUOUS type). There is three types of Audit:
-ONESHOT and CONTINUOUS. Oneshot Audit is launched once and if it succeeded
+ONESHOT, CONTINUOUS and EVENT. ONESHOT Audit is launched once and if it
-executed new action plan list will be provided. Continuous Audit creates
+succeeded executed new action plan list will be provided; CONTINUOUS Audit
-action plans with specified interval (in seconds); if action plan
+creates action plans with specified interval (in seconds or cron format, cron
-has been created, all previous action plans get CANCELLED state.
+inteval can be used like: `*/5 * * * *`), if action plan
+has been created, all previous action plans get CANCELLED state;
+EVENT audit is launched when receiving webhooks API.
 
 A message is sent on the :ref:`AMQP bus <amqp_bus_definition>` which triggers
 the Audit in the
@@ -479,4 +481,4 @@ change to a new value:
 
 
 
-.. _Watcher API: https://developer.openstack.org/api-ref/resource-optimization/
+.. _Watcher API: https://docs.openstack.org/api-ref/resource-optimization/

doc/source/conf.py

@@ -14,7 +14,6 @@
 import os
 import sys
 
-from watcher import version as watcher_version
 from watcher import objects
 
 objects.register_all()
@@ -32,17 +31,16 @@ sys.path.insert(0, os.path.abspath('./'))
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'oslo_config.sphinxext',
-    'sphinxcontrib.apidoc',
     'sphinx.ext.viewcode',
     'sphinxcontrib.httpdomain',
     'sphinxcontrib.pecanwsme.rest',
     'stevedore.sphinxext',
-    'wsmeext.sphinxext',
     'ext.term',
     'ext.versioned_notifications',
     'oslo_config.sphinxconfiggen',
     'openstackdocstheme',
     'sphinx.ext.napoleon',
+    'sphinxcontrib.rsvgconverter',
 ]
 
 wsme_protocols = ['restjson']
@@ -51,22 +49,6 @@ config_generator_config_file = [(
     '_static/watcher')]
 sample_config_basename = 'watcher'
 
-# autodoc generation is a bit aggressive and a nuisance when doing heavy
-# text edit cycles.
-# execute "export SPHINX_DEBUG=1" in your terminal to disable
-
-# sphinxcontrib.apidoc options
-apidoc_module_dir = '../../watcher'
-apidoc_output_dir = 'api'
-apidoc_excluded_paths = [
-    'tests/*',
-    'db',
-    'decision_engine',
-    'doc',
-    'objects',
-]
-apidoc_separate_modules = True
-
 # The suffix of source filenames.
 source_suffix = '.rst'
 
@@ -74,18 +56,8 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Watcher'
+project = 'Watcher'
-copyright = u'OpenStack Foundation'
+copyright = 'OpenStack Foundation'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-# The full version, including alpha/beta/rc tags.
-release = watcher_version.version_info.release_string()
-# The short X.Y version.
-version = watcher_version.version_string
 
 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['watcher.']
@@ -111,7 +83,7 @@ add_module_names = True
 suppress_warnings = ['app.add_directive']
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # -- Options for man page output --------------------------------------------
 
@@ -119,14 +91,14 @@ pygments_style = 'sphinx'
 # List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'
 
 man_pages = [
-    ('man/watcher-api', 'watcher-api', u'Watcher API Server',
+    ('man/watcher-api', 'watcher-api', 'Watcher API Server',
-     [u'OpenStack'], 1),
+     ['OpenStack'], 1),
-    ('man/watcher-applier', 'watcher-applier', u'Watcher Applier',
+    ('man/watcher-applier', 'watcher-applier', 'Watcher Applier',
-     [u'OpenStack'], 1),
+     ['OpenStack'], 1),
     ('man/watcher-db-manage', 'watcher-db-manage',
-     u'Watcher Db Management Utility', [u'OpenStack'], 1),
+     'Watcher Db Management Utility', ['OpenStack'], 1),
     ('man/watcher-decision-engine', 'watcher-decision-engine',
-     u'Watcher Decision Engine', [u'OpenStack'], 1),
+     'Watcher Decision Engine', ['OpenStack'], 1),
 ]
 
 # -- Options for HTML output --------------------------------------------------
@@ -142,22 +114,34 @@ html_theme = 'openstackdocs'
 # Output file base name for HTML help builder.
 htmlhelp_basename = '%sdoc' % project
 
-html_last_updated_fmt = '%Y-%m-%d %H:%M'
 
 #openstackdocstheme options
-repository_name = 'openstack/watcher'
+openstackdocs_repo_name = 'openstack/watcher'
-bug_project = 'watcher'
+openstackdocs_pdf_link = True
-bug_tag = ''
+openstackdocs_auto_name = False
+openstackdocs_bug_project = 'watcher'
+openstackdocs_bug_tag = ''
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass
 # [howto/manual]).
 latex_documents = [
     ('index',
-     '%s.tex' % project,
+     'doc-watcher.tex',
-     u'%s Documentation' % project,
+     'Watcher Documentation',
-     u'OpenStack Foundation', 'manual'),
+     'OpenStack Foundation', 'manual'),
 ]
 
+# If false, no module index is generated.
+latex_domain_indices = False
+
+latex_elements = {
+    'makeindex': '',
+    'printindex': '',
+    'preamble': r'\setcounter{tocdepth}{3}',
+}
+
+# Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664
+latex_use_xindy = False
 # Example configuration for intersphinx: refer to the Python standard library.
 # intersphinx_mapping = {'http://docs.python.org/': None}

doc/source/configuration/configuring.rst

@@ -178,7 +178,7 @@ You can easily generate and update a sample configuration file
 named :ref:`watcher.conf.sample <watcher_sample_configuration_files>` by using
 these following commands::
 
-    $ git clone git://git.openstack.org/openstack/watcher
+    $ git clone https://opendev.org/openstack/watcher.git
     $ cd watcher/
     $ tox -e genconfig
     $ vi etc/watcher/watcher.conf.sample
@@ -372,7 +372,7 @@ You can configure and install Ceilometer by following the documentation below :
 #. https://docs.openstack.org/ceilometer/latest
 
 The built-in strategy 'basic_consolidation' provided by watcher requires
-"**compute.node.cpu.percent**" and "**cpu_util**" measurements to be collected
+"**compute.node.cpu.percent**" and "**cpu**" measurements to be collected
 by Ceilometer.
 The measurements available depend on the hypervisors that OpenStack manages on
 the specific implementation.

doc/source/contributor/concurrency.rst

@@ -0,0 +1,248 @@
+===========
+Concurrency
+===========
+
+Introduction
+************
+
+Modern processors typically contain multiple cores all capable of executing
+instructions in parallel. Ensuring applications can fully utilize modern
+underlying hardware requires developing with these concepts in mind. The
+OpenStack foundation maintains a number of libraries to facilitate this
+utilization, combined with constructs like CPython's GIL_ the proper use of
+these concepts becomes more straightforward compared to other programming
+languages.
+
+The primary libraries maintained by OpenStack to facilitate concurrency are
+futurist_ and taskflow_. Here futurist is a more straightforward and
+lightweight library while taskflow is more advanced supporting features like
+rollback mechanisms. Within Watcher both libraries are used to facilitate
+concurrency.
+
+.. _GIL: https://wiki.python.org/moin/GlobalInterpreterLock
+.. _futurist: https://docs.openstack.org/futurist/latest/
+.. _taskflow: https://docs.openstack.org/taskflow/latest/
+
+Threadpool
+**********
+
+A threadpool is a collection of one or more threads typically called *workers*
+to which tasks can be submitted. These submitted tasks will be scheduled by a
+threadpool and subsequently executed. In the case of Python tasks typically are
+bounded or unbounded methods while other programming languages like Java
+require implementing an interface.
+
+The order and amount of concurrency with which these tasks are executed is up
+to the threadpool to decide. Some libraries like taskflow allow for either
+strong or loose ordering of tasks while others like futurist might only support
+loose ordering. Taskflow supports building tree-based hierarchies of dependent
+tasks for example.
+
+Upon submission of a task to a threadpool a so called future_ is returned.
+These objects allow to determine information about the task such as if it is
+currently being executed or if it has finished execution. When the task has
+finished execution the future can also be used to retrieve what was returned by
+the method.
+
+Some libraries like futurist provide synchronization primitives for collections
+of futures such as wait_for_any_. The following sections will cover different
+types of concurrency used in various services of Watcher.
+
+.. _future: https://docs.python.org/3/library/concurrent.futures.html
+.. _wait_for_any: https://docs.openstack.org/futurist/latest/reference/index.html#waiters
+
+
+Decision engine concurrency
+***************************
+
+The concurrency in the decision engine is governed by two independent
+threadpools. Both of these threadpools are GreenThreadPoolExecutor_ from the
+futurist_ library. One of these is used automatically and most contributors
+will not interact with it while developing new features. The other threadpool
+can frequently be used while developing new features or updating existing ones.
+It is known as the DecisionEngineThreadpool and allows to achieve performance
+improvements in network or I/O bound operations.
+
+.. _GreenThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#executors
+
+AuditEndpoint
+#############
+
+The first threadpool is used to allow multiple audits to be run in parallel.
+In practice, however, only one audit can be run in parallel. This is due to
+the data model used by audits being a singleton. To prevent audits destroying
+each others data model one must wait for the other to complete before being
+allowed to access this data model. A performance improvement could be achieved
+by being more intelligent in the use, caching and construction of these
+data models.
+
+DecisionEngineThreadPool
+########################
+
+The second threadpool is used for generic tasks, typically networking and I/O
+could benefit the most of this threadpool. Upon execution of an audit this
+threadpool can be utilized to retrieve information from the Nova compute
+service for instance. This second threadpool is a singleton and is shared
+amongst concurrently running audits as a result the amount of workers is static
+and independent from the amount of workers in the first threadpool. The use of
+the :class:`~.DecisionEngineThreadpool` while building the Nova compute data
+model is demonstrated to show how it can effectively be used.
+
+In the following example a reference to the
+:class:`~.DecisionEngineThreadpool` is stored in ``self.executor``. Here two
+tasks are submitted one with function ``self._collect_aggregates`` and the
+other function ``self._collect_zones``. With both ``self.executor.submit``
+calls subsequent arguments are passed to the function. All subsequent arguments
+are passed to the function being submitted as task following the common
+``(fn, *args, **kwargs)`` signature. One of the original signatures would be
+``def _collect_aggregates(host_aggregates, compute_nodes)`` for example.
+
+.. code-block:: python
+
+    zone_aggregate_futures = {
+        self.executor.submit(
+            self._collect_aggregates, host_aggregates, compute_nodes),
+        self.executor.submit(
+            self._collect_zones, availability_zones, compute_nodes)
+    }
+    waiters.wait_for_all(zone_aggregate_futures)
+
+The last statement of the example above waits on all futures to complete.
+Similarly, ``waiters.wait_for_any`` will wait for any future of the specified
+collection to complete. To simplify the usage of ``wait_for_any`` the
+:class:`~.DecisiongEngineThreadpool` defines a ``do_while_futures`` method.
+This method will iterate in a do_while loop over a collection of futures until
+all of them have completed. The advantage of ``do_while_futures`` is that it
+allows to immediately call a method as soon as a future finishes. The arguments
+for this callback method can be supplied when calling ``do_while_futures``,
+however, the first argument to the callback is always the future itself! If
+the collection of futures can safely be modified ``do_while_futures_modify``
+can be used and should have slightly better performance. The following example
+will show how ``do_while_futures`` is used in the decision engine.
+
+.. code-block:: python
+
+    # For every compute node from compute_nodes submit a task to gather the node it's information.
+    # List comprehension is used to store all the futures of the submitted tasks in node_futures.
+    node_futures = [self.executor.submit(
+        self.nova_helper.get_compute_node_by_name,
+        node, servers=True, detailed=True)
+        for node in compute_nodes]
+    LOG.debug("submitted {0} jobs".format(len(compute_nodes)))
+
+    future_instances = []
+    # do_while iterate over node_futures and upon completion of a future call
+    # self._compute_node_future with the future and future_instances as arguments.
+    self.executor.do_while_futures_modify(
+        node_futures, self._compute_node_future, future_instances)
+
+    # Wait for all instance jobs to finish
+    waiters.wait_for_all(future_instances)
+
+Finally, let's demonstrate how powerful this ``do_while_futures`` can be by
+showing what the ``compute_node_future`` callback does. First, it retrieves the
+result from the future and adds the compute node to the data model. Afterwards,
+it checks if the compute node has any associated instances and if so it submits
+an additional task to the :class:`~.DecisionEngineThreadpool`. The future is
+appended to the ``future_instances`` so ``waiters.wait_for_all`` can be called
+on this list. This is important as otherwise the building of the data model
+might return before all tasks for instances have finished.
+
+.. code-block:: python
+
+    # Get the result from the future.
+    node_info = future.result()[0]
+
+    # Filter out baremetal nodes.
+    if node_info.hypervisor_type == 'ironic':
+        LOG.debug("filtering out baremetal node: %s", node_info)
+        return
+
+    # Add the compute node to the data model.
+    self.add_compute_node(node_info)
+    # Get the instances from the compute node.
+    instances = getattr(node_info, "servers", None)
+    # Do not submit job if there are no instances on compute node.
+    if instances is None:
+        LOG.info("No instances on compute_node: {0}".format(node_info))
+        return
+    # Submit a job to retrieve detailed information about the instances.
+    future_instances.append(
+        self.executor.submit(
+            self.add_instance_node, node_info, instances)
+    )
+
+Without ``do_while_futures`` an additional ``waiters.wait_for_all`` would be
+required in between the compute node tasks and the instance tasks. This would
+cause the progress of the decision engine to stall as less and less tasks
+remain active before the instance tasks could be submitted. This demonstrates
+how ``do_while_futures`` can be used to achieve more constant utilization of
+the underlying hardware.
+
+Applier concurrency
+*******************
+
+The applier does not use the futurist_ GreenThreadPoolExecutor_ directly but
+instead uses taskflow_. However, taskflow still utilizes a greenthreadpool.
+This threadpool is initialized in the workflow engine called
+:class:`~.DefaultWorkFlowEngine`. Currently Watcher supports one workflow
+engine but the base class allows contributors to develop other workflow engines
+as well. In taskflow tasks are created using different types of flows such as a
+linear, unordered or a graph flow. The linear and graph flow allow for strong
+ordering between individual tasks and it is for this reason that the workflow
+engine utilizes a graph flow. The creation of tasks, subsequently linking them
+into a graph like structure and submitting them is shown below.
+
+.. code-block:: python
+
+    self.execution_rule = self.get_execution_rule(actions)
+    flow = gf.Flow("watcher_flow")
+    actions_uuid = {}
+    for a in actions:
+        task = TaskFlowActionContainer(a, self)
+        flow.add(task)
+        actions_uuid[a.uuid] = task
+
+    for a in actions:
+        for parent_id in a.parents:
+            flow.link(actions_uuid[parent_id], actions_uuid[a.uuid],
+                      decider=self.decider)
+
+    e = engines.load(
+        flow, executor='greenthreaded', engine='parallel',
+        max_workers=self.config.max_workers)
+    e.run()
+
+    return flow
+
+In the applier tasks are contained in a :class:`~.TaskFlowActionContainer`
+which allows them to trigger events in the workflow engine. This way the
+workflow engine can halt or take other actions while the action plan is being
+executed based on the success or failure of individual actions. However, the
+base workflow engine simply uses these notifies to store the result of
+individual actions in the database. Additionally, since taskflow uses a graph
+flow if any of the tasks would fail all childs of this tasks not be executed
+while ``do_revert`` will be triggered for all parents.
+
+.. code-block:: python
+
+    class TaskFlowActionContainer(...):
+        ...
+        def do_execute(self, *args, **kwargs):
+            ...
+            result = self.action.execute()
+            if result is True:
+                return self.engine.notify(self._db_action,
+                                          objects.action.State.SUCCEEDED)
+            else:
+                self.engine.notify(self._db_action,
+                                   objects.action.State.FAILED)
+
+    class BaseWorkFlowEngine(...):
+        ...
+        def notify(self, action, state):
+            db_action = objects.Action.get_by_uuid(self.context, action.uuid,
+                                                   eager=True)
+            db_action.state = state
+            db_action.save()
+            return db_action

doc/source/contributor/contributing.rst

@@ -1,72 +1,111 @@
-..
+============================
-      Except where otherwise noted, this document is licensed under Creative
+So You Want to Contribute...
-      Commons Attribution 3.0 License.  You can view the license at:
+============================
 
-          https://creativecommons.org/licenses/by/3.0/
+For general information on contributing to OpenStack, please check out the
+`contributor guide <https://docs.openstack.org/contributors/>`_ to get started.
+It covers all the basics that are common to all OpenStack projects:
+the accounts you need, the basics of interacting with our Gerrit review system,
+how we communicate as a community, etc.
 
-.. _contributing:
+Below will cover the more project specific information you need to get started
+with Watcher.
 
-=======================
+Communication
-Contributing to Watcher
+~~~~~~~~~~~~~~
-=======================
+.. This would be a good place to put the channel you chat in as a project; when/
-
+   where your meeting is, the tags you prepend to your ML threads, etc.
-If you're interested in contributing to the Watcher project,
-the following will help get you started.
-
-Contributor License Agreement
------------------------------
-
-.. index::
-   single: license; agreement
-
-In order to contribute to the Watcher project, you need to have
-signed OpenStack's contributor's agreement.
-
-.. seealso::
-
-   * https://docs.openstack.org/infra/manual/developers.html
-   * https://wiki.openstack.org/CLA
-
-LaunchPad Project
------------------
-
-Most of the tools used for OpenStack depend on a launchpad.net ID for
-authentication. After signing up for a launchpad account, join the
-"openstack" team to have access to the mailing list and receive
-notifications of important events.
-
-.. seealso::
-
-   * https://launchpad.net
-   * https://launchpad.net/watcher
-   * https://launchpad.net/openstack
-
-
-Project Hosting Details
------------------------
-
-Bug tracker
-    https://launchpad.net/watcher
-
-Mailing list (prefix subjects with ``[watcher]`` for faster responses)
-    http://lists.openstack.org/pipermail/openstack-discuss/
-
-Wiki
-    https://wiki.openstack.org/Watcher
-
-Code Hosting
-    https://git.openstack.org/cgit/openstack/watcher
-
-Code Review
-    https://review.openstack.org/#/q/status:open+project:openstack/watcher,n,z
 
 IRC Channel
     ``#openstack-watcher`` (changelog_)
 
+Mailing list(prefix subjects with ``[watcher]``)
+    http://lists.openstack.org/pipermail/openstack-discuss/
+
 Weekly Meetings
-    On Wednesdays at 14:00 UTC on even weeks in the ``#openstack-meeting-4``
+    Bi-weekly, on Wednesdays at 08:00 UTC on odd weeks in the
-    IRC channel, 08:00 UTC on odd weeks in the ``#openstack-meeting-alt``
+    ``#openstack-meeting-alt`` IRC channel (`meetings logs`_)
-    IRC channel (`meetings logs`_)
+
+Meeting Agenda
+    https://wiki.openstack.org/wiki/Watcher_Meeting_Agenda
 
 .. _changelog: http://eavesdrop.openstack.org/irclogs/%23openstack-watcher/
 .. _meetings logs:  http://eavesdrop.openstack.org/meetings/watcher/
+
+Contacting the Core Team
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. This section should list the core team, their irc nicks, emails, timezones etc.
+   If all this info is maintained elsewhere (i.e. a wiki), you can link to that
+   instead of enumerating everyone here.
+
++--------------------+---------------+------------------------------------+
+| Name               | IRC           | Email                              |
++====================+===============+====================================+
+| `Li Canwei`_       | licanwei      | li.canwei2@zte.com.cn              |
++--------------------+---------------+------------------------------------+
+| `chen ke`_         | chenke        | chen.ke14@zte.com.cn               |
++--------------------+---------------+------------------------------------+
+| `Corne Lukken`_    | dantalion     | info@dantalion.nl                  |
++--------------------+---------------+------------------------------------+
+| `su zhengwei`_     | suzhengwei    | sugar-2008@163.com                 |
++--------------------+---------------+------------------------------------+
+| `Yumeng Bao`_      | Yumeng        | yumeng_bao@yahoo.com               |
++--------------------+---------------+------------------------------------+
+
+.. _Corne Lukken: https://launchpad.net/~dantalion
+.. _Li Canwei: https://launchpad.net/~li-canwei2
+.. _su zhengwei: https://launchpad.net/~sue.sam
+.. _Yumeng Bao: https://launchpad.net/~yumeng-bao
+.. _chen ke: https://launchpad.net/~chenker
+
+New Feature Planning
+~~~~~~~~~~~~~~~~~~~~
+.. This section is for talking about the process to get a new feature in. Some
+   projects use blueprints, some want specs, some want both! Some projects
+   stick to a strict schedule when selecting what new features will be reviewed
+   for a release.
+
+New feature will be discussed via IRC or ML (with [Watcher] prefix).
+Watcher team uses blueprints in `Launchpad`_ to manage the new features.
+
+.. _Launchpad: https://launchpad.net/watcher
+
+Task Tracking
+~~~~~~~~~~~~~~
+.. This section is about where you track tasks- launchpad? storyboard?
+   is there more than one launchpad project? what's the name of the project
+   group in storyboard?
+
+We track our tasks in Launchpad.
+If you're looking for some smaller, easier work item to pick up and get started
+on, search for the 'low-hanging-fruit' tag.
+
+.. NOTE: If your tag is not 'low-hanging-fruit' please change the text above.
+
+Reporting a Bug
+~~~~~~~~~~~~~~~
+.. Pretty self explanatory section, link directly to where people should report bugs for
+   your project.
+
+You found an issue and want to make sure we are aware of it? You can do so
+`HERE`_.
+
+.. _HERE: https://bugs.launchpad.net/watcher
+
+Getting Your Patch Merged
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. This section should have info about what it takes to get something merged.
+   Do you require one or two +2's before +W? Do some of your repos require
+   unit test changes with all patches? etc.
+
+Due to the small number of core reviewers of the Watcher project,
+we only need one +2 before +W (merge). All patches excepting for documentation
+or typos fixes must have unit test.
+
+Project Team Lead Duties
+------------------------
+.. this section is where you can put PTL specific duties not already listed in
+   the common PTL guide (linked below)  or if you already have them written
+   up elsewhere, you can link to that doc here.
+
+All common PTL duties are enumerated here in the `PTL guide <https://docs.openstack.org/project-team-guide/ptl.html>`_.

doc/source/contributor/devstack.rst

@@ -19,7 +19,7 @@ model. To enable the Watcher plugin with DevStack, add the following to the
 `[[local|localrc]]` section of your controller's `local.conf` to enable the
 Watcher plugin::
 
-    enable_plugin watcher git://git.openstack.org/openstack/watcher
+    enable_plugin watcher https://opendev.org/openstack/watcher
 
 For more detailed instructions, see `Detailed DevStack Instructions`_. Check
 out the `DevStack documentation`_ for more information regarding DevStack.
@@ -27,6 +27,38 @@ out the `DevStack documentation`_ for more information regarding DevStack.
 .. _PluginModelDocs: https://docs.openstack.org/devstack/latest/plugins.html
 .. _DevStack documentation: https://docs.openstack.org/devstack/latest
 
+Quick Devstack Instructions with Datasources
+============================================
+
+Watcher requires a datasource to collect metrics from compute nodes and
+instances in order to execute most strategies. To enable this a
+`[[local|localrc]]` to setup DevStack for some of the supported datasources
+is provided. These examples specify the minimal configuration parameters to
+get both Watcher and the datasource working but can be expanded is desired.
+
+Gnocchi
+-------
+
+With the Gnocchi datasource most of the metrics for compute nodes and
+instances will work with the provided configuration but metrics that
+require Ironic such as `host_airflow and` `host_power` will still be
+unavailable as well as `instance_l3_cpu_cache`::
+
+        [[local|localrc]]
+        enable_plugin watcher https://opendev.org/openstack/watcher
+
+        enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard
+
+        enable_plugin ceilometer https://opendev.org/openstack/ceilometer.git
+        CEILOMETER_BACKEND=gnocchi
+
+        enable_plugin aodh https://opendev.org/openstack/aodh
+        enable_plugin panko https://opendev.org/openstack/panko
+
+        [[post-config|$NOVA_CONF]]
+        [DEFAULT]
+        compute_monitors=cpu.virt_driver
+
 Detailed DevStack Instructions
 ==============================
 
@@ -44,7 +76,7 @@ Detailed DevStack Instructions
 
        sudo apt-get update
        sudo apt-get install git
-       git clone https://git.openstack.org/openstack-dev/devstack
+       git clone https://opendev.org/openstack/devstack.git
        sudo ./devstack/tools/create-stack-user.sh
 
    Now you have a stack user that is used to run the DevStack processes. You
@@ -56,7 +88,7 @@ Detailed DevStack Instructions
 
        sudo su stack
        cd ~
-       git clone https://git.openstack.org/openstack-dev/devstack
+       git clone https://opendev.org/openstack/devstack.git
 
 #. For each compute node, copy the provided `local.conf.compute`_ example file
    to the compute node's system at ~/devstack/local.conf. Make sure the

doc/source/contributor/environment.rst

@@ -16,7 +16,7 @@ for development purposes.
 To install Watcher from packaging, refer instead to Watcher `User
 Documentation`_.
 
-.. _`Git Repository`: https://git.openstack.org/cgit/openstack/watcher
+.. _`Git Repository`: https://opendev.org/openstack/watcher
 .. _`User Documentation`: https://docs.openstack.org/watcher/latest/
 
 Prerequisites
@@ -47,7 +47,7 @@ Make a clone of the code from our `Git repository`:
 
 .. code-block:: bash
 
-    $ git clone https://git.openstack.org/openstack/watcher.git
+    $ git clone https://opendev.org/openstack/watcher.git
 
 When that is complete, you can:
 
@@ -99,7 +99,7 @@ useful to keep a clean environment for working on Watcher.
 .. code-block:: bash
 
     $ mkvirtualenv watcher
-    $ git clone https://git.openstack.org/openstack/watcher
+    $ git clone https://opendev.org/openstack/watcher.git
 
     # Use 'python setup.py' to link Watcher into Python's site-packages
     $ cd watcher && python setup.py install

doc/source/contributor/index.rst

@@ -1,8 +1,12 @@
-.. toctree::
+==================
-   :maxdepth: 1
+Contribution Guide
+==================
 
+.. toctree::
+   :maxdepth: 2
+
+   contributing
    environment
    devstack
-   notifications
    testing
    rally_link

doc/source/contributor/plugin/index.rst

@@ -1,3 +1,7 @@
+============
+Plugin Guide
+============
+
 .. toctree::
    :maxdepth: 1
 

doc/source/contributor/plugin/strategy-plugin.rst

@@ -56,9 +56,6 @@ Here is an example showing how you can write a plugin called ``NewStrategy``:
     # filepath: thirdparty/new.py
     # import path: thirdparty.new
     import abc
-
-    import six
-
     from watcher._i18n import _
     from watcher.decision_engine.strategy.strategies import base
 
@@ -285,8 +282,15 @@ The following code snippet shows how datasource_backend is defined:
     @property
     def datasource_backend(self):
         if not self._datasource_backend:
+
+            # Load the global preferred datasources order but override it
+            # if the strategy has a specific datasources config
+            datasources = CONF.watcher_datasources
+            if self.config.datasources:
+                datasources = self.config
+
             self._datasource_backend = ds_manager.DataSourceManager(
-                config=self.config,
+                config=datasources,
                 osc=self.osc
             ).get_backend(self.DATASOURCE_METRICS)
         return self._datasource_backend
@@ -296,6 +300,6 @@ Using that you can now query the values for that specific metric:
 .. code-block:: py
 
     avg_meter = self.datasource_backend.statistic_aggregation(
-        instance.uuid, 'cpu_util', self.periods['instance'],
+        instance.uuid, 'instance_cpu_usage', self.periods['instance'],
         self.granularity,
         aggregation=self.aggregation_method['instance'])

doc/source/contributor/testing.rst

@@ -4,9 +4,9 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-=======
+=================
-Testing
+Developer Testing
-=======
+=================
 
 .. _unit_tests:
 
@@ -15,10 +15,10 @@ Unit tests
 
 All unit tests should be run using `tox`_. Before running the unit tests, you
 should download the latest `watcher`_ from the github. To run the same unit
-tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
+tests that are executing onto `Gerrit`_ which includes ``py36``, ``py37`` and
 ``pep8``, you can issue the following command::
 
-    $ git clone https://git.openstack.org/openstack/watcher
+    $ git clone https://opendev.org/openstack/watcher
     $ cd watcher
     $ pip install tox
     $ tox
@@ -26,19 +26,19 @@ tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
 If you only want to run one of the aforementioned, you can then issue one of
 the following::
 
-    $ tox -e py35
+    $ tox -e py36
-    $ tox -e py27
+    $ tox -e py37
     $ tox -e pep8
 
 .. _tox: https://tox.readthedocs.org/
-.. _watcher: https://git.openstack.org/cgit/openstack/watcher
+.. _watcher: https://opendev.org/openstack/watcher
-.. _Gerrit: https://review.openstack.org/
+.. _Gerrit: https://review.opendev.org/
 
 If you only want to run specific unit test code and don't like to waste time
 waiting for all unit tests to execute, you can add parameters ``--`` followed
 by a regex string::
 
-    $ tox -e py27 -- watcher.tests.api
+    $ tox -e py37 -- watcher.tests.api
 
 .. _tempest_tests:
 
@@ -48,4 +48,4 @@ Tempest tests
 Tempest tests for Watcher has been migrated to the external repo
 `watcher-tempest-plugin`_.
 
-.. _watcher-tempest-plugin: https://git.openstack.org/cgit/openstack/watcher-tempest-plugin
+.. _watcher-tempest-plugin: https://opendev.org/openstack/watcher-tempest-plugin

doc/source/datasources/grafana.rst

@@ -0,0 +1,426 @@
+==================
+Grafana datasource
+==================
+
+Synopsis
+--------
+
+Grafana can interface with many different types of storage backends that
+Grafana calls datasources_. Since the term datasources causes significant
+confusion by overlapping definitions used in Watcher these **datasources are
+called projects instead**. Some examples of supported projects are InfluxDB
+or Elasticsearch while others might be more familiar such as Monasca or
+Gnocchi. The Grafana datasource provides the functionality to retrieve metrics
+from Grafana for different projects. This functionality is achieved by using
+the proxy interface exposed in Grafana to communicate with Grafana projects
+directly.
+
+Background
+**********
+
+Since queries to retrieve metrics from Grafana are proxied to the project the
+format of these queries will change significantly depending on the type of
+project. The structure of the projects themselves will also change
+significantly as they are structured by users and administrators. For instance,
+some developers might decide to store metrics about compute_nodes in MySQL and
+use the UUID as primary key while others use InfluxDB and use the hostname as
+primary key. Furthermore, datasources in Watcher should return metrics in
+specific units strictly defined in the baseclass_ depending on how the units
+are stored in the projects they might require conversion before being returned.
+The flexible configuration parameters of the Grafana datasource allow to
+specify exactly how the deployment is configured and this will enable to
+correct retrieval of metrics and with the correct units.
+
+.. _datasources: https://grafana.com/plugins?type=datasource
+.. _baseclass: https://github.com/openstack/watcher/blob/584eeefdc8/watcher/datasources/base.py
+
+Requirements
+------------
+
+The use of the Grafana datasource requires a reachable Grafana endpoint and an
+authentication token for access to the desired projects. The projects behind
+Grafana will need to contain the metrics for compute_nodes_ or instances_ and
+these need to be identifiable by an attribute of the Watcher datamodel_ for
+instance hostname or UUID.
+
+.. _compute_nodes: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/element/node.py
+.. _instances: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/element/instance.py
+.. _datamodel: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/element
+
+Limitations
+***********
+
+* Only the InfluxDB project is currently supported [#f1]_.
+* All metrics must be retrieved from the same Grafana endpoint (same URL).
+* All metrics must be retrieved with the same authentication token.
+
+.. [#f1] A base class for projects is available_ and easily extensible.
+.. _available: https://review.opendev.org/#/c/649341/24/watcher/datasources/grafana_translator/base.py
+
+Configuration
+-------------
+
+Several steps are required in order to use the Grafana datasource, Most steps
+are related configuring Watcher to match the deployed Grafana setup such as
+queries proxied to the project or the type of project for any given metric.
+Most of the configuration can either be supplied via the traditional
+configuration file or in a `special yaml`_ file.
+
+.. _special yaml: https://specs.openstack.org/openstack/watcher-specs/specs/train/approved/file-based-metricmap.html
+
+token
+*****
+
+First step is to generate an access token with access to the required projects.
+This can be done from the api_ or from the web interface_. Tokens generated
+from the web interface will have the same access to projects as the user that
+created them while using the cli allows to generate a key for a specific
+role.The token will only be displayed once so store it well. This token will go
+into the configuration file later and this parameter can not be placed in the
+yaml.
+
+.. _api: https://grafana.com/docs/http_api/auth/#create-api-key
+.. _interface: https://grafana.com/docs/http_api/auth/#create-api-token
+
+base_url
+********
+
+Next step is supplying the base url of the Grafana endpoint. The base url
+parameter will need to specify the type of http protocol and the use of
+plain text http is strongly discouraged due to the transmission of the access
+token. Additionally the path to the proxy interface needs to be supplied as
+well in case Grafana is placed in a sub directory of the web server. An example
+would be: `https://mygrafana.org/api/datasource/proxy/` were
+`/api/datasource/proxy` is the default path without any subdirectories.
+Likewise, this parameter can not be placed in the yaml.
+
+To prevent many errors from occurring and potentially filing the logs files it
+is advised to specify the desired datasource in the configuration as it would
+prevent the datasource manager from having to iterate and try possible
+datasources with the launch of each audit. To do this specify `datasources` in
+the `[watcher_datasources]` group.
+
+The current configuration that is required to be placed in the traditional
+configuration file would look like the following:
+
+.. code-block:: shell
+
+    [grafana_client]
+    token = 0JLbF0oB4R3Q2Fl337Gh4Df5VN12D3adBE3f==
+    base_url = https://mygranfa.org/api/datasource/proxy
+
+    [watcher_datasources]
+    datasources = grafana
+
+metric parameters
+*****************
+
+The last five remaining configuration parameters can all be placed both in the
+traditional configuration file or in the yaml, however, it is not advised to
+mix and match but in the case it does occur the yaml would override the
+settings from the traditional configuration file. All five of these parameters
+are dictionaries mapping specific metrics to a configuration parameter. For
+instance the `project_id_map` will specify the specific project id in Grafana
+to be used. The parameters are named as follow:
+
+* project_id_map
+* database_map
+* translator_map
+* attribute_map
+* query_map
+
+These five parameters are named differently if configured using the yaml
+configuration file. The parameters are named as follows and are in
+identical order as to the list of the traditional configuration file:
+
+* project
+* db
+* translator
+* attribute
+* query
+
+When specified in the yaml the parameters are no longer dictionaries instead
+each parameter needs to be defined per metric as sub-parameters. Examples of
+these parameters configured for both the yaml and traditional configuration
+are described at the end of this document.
+
+project_id
+**********
+
+The project id's can only be determined by someone with the admin role in
+Grafana as that role is required to open the list of projects. The list of
+projects can be found on `/datasources` in the web interface but
+unfortunately it does not immediately display the project id. To display
+the id one can best hover the mouse over the projects and the url will show the
+project id's for example `/datasources/edit/7563`. Alternatively the entire
+list of projects can be retrieved using the `REST api`_. To easily make
+requests to the REST api a tool such as Postman can be used.
+
+.. _REST api: https://grafana.com/docs/http_api/data_source/#get-all-datasources
+
+database
+********
+
+The database is the parameter for the schema / database that is actually
+defined in the project. For instance, if the project would be based on MySQL
+this is were the name of schema used within the MySQL server would be
+specified. For many different projects it is possible to list all the databases
+currently available. Tools like Postman can be used to list all the available
+databases per project. For InfluxDB based projects this would be with the
+following path and query, however be sure to construct these request in Postman
+as the header needs to contain the authorization token:
+
+.. code-block:: shell
+
+    https://URL.DOMAIN/api/datasources/proxy/PROJECT_ID/query?q=SHOW%20DATABASES
+
+translator
+**********
+
+Each translator is for a specific type of project will have a uniquely
+identifiable name and the baseclass allows to easily support new types of
+projects such as elasticsearch or prometheus. Currently only InfluxDB based
+projects are supported as a result the only valid value for this parameter is `
+influxdb`.
+
+attribute
+*********
+
+The attribute parameter specifies which attribute to use from Watcher's
+data model in order to construct the query. The available attributes differ
+per type of object in the data model but the following table shows the
+attributes for ComputeNodes, Instances and IronicNodes.
+
++-----------------+-----------------+--------------------+
+| ComputeNode     | Instance        | IronicNode         |
++=================+=================+====================+
+| uuid            | uuid            | uuid               |
++-----------------+-----------------+--------------------+
+| id              | name            | human_id           |
++-----------------+-----------------+--------------------+
+| hostname        | project_id      | power_state        |
++-----------------+-----------------+--------------------+
+| status          | watcher_exclude | maintenance        |
++-----------------+-----------------+--------------------+
+| disabled_reason | locked          | maintenance_reason |
++-----------------+-----------------+--------------------+
+| state           | metadata        | extra              |
++-----------------+-----------------+--------------------+
+| memory          | state           |                    |
++-----------------+-----------------+--------------------+
+| disk            | memory          |                    |
++-----------------+-----------------+--------------------+
+| disk_capacity   | disk            |                    |
++-----------------+-----------------+--------------------+
+| vcpus           | disk_capacity   |                    |
++-----------------+-----------------+--------------------+
+|                 | vcpus           |                    |
++-----------------+-----------------+--------------------+
+
+Many if not all of these attributes map to attributes of the objects that are
+fetched from clients such as Nova. To see how these attributes are put into the
+data model the following source files can be analyzed for Nova_ and Ironic_.
+
+.. _Nova: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/collector/nova.py#L304
+.. _Ironic: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/collector/ironic.py#L85
+
+query
+*****
+
+The query is the single most important parameter it will be passed to the
+project and should return the desired metric for the specific host and return
+the value in the correct unit. The units for all available metrics are
+documented in the `datasource baseclass`_. This might mean the query specified
+in this parameter is responsible for converting the unit. The following query
+demonstrates how such a conversion could be achieved and demonstrates the
+conversion from bytes to megabytes.
+
+.. code-block:: shell
+
+    SELECT value/1000000 FROM memory...
+
+Queries will be formatted using the .format string method within Python. This
+format will currently have give attributes exposed to it labeled `{0}` to
+`{4}`. Every occurrence of these characters within the string will be replaced
+with the specific attribute.
+
+- {0} is the aggregate typically `mean`, `min`, `max` but `count` is also
+  supported.
+- {1} is the attribute as specified in the attribute parameter.
+- {2} is the period of time to aggregate data over in seconds.
+- {3} is the granularity or the interval between data points in seconds.
+- {4} is translator specific and in the case of InfluxDB it will be used for
+  retention_periods.
+
+**InfluxDB**
+
+Constructing the queries or rather anticipating how the results should look to
+be correctly interpreted by Watcher can be a challenge. The following json
+example demonstrates how what the result should look like and the query used to
+get this result.
+
+.. code-block:: json
+
+    {
+    "results": [
+        {
+            "statement_id": 0,
+            "series": [
+                {
+                    "name": "vmstats",
+                    "tags": {
+                        "host": "autoserver01"
+                    },
+                    "columns": [
+                        "time",
+                        "mean"
+                    ],
+                    "values": [
+                        [
+                            1560848284284,
+                            7680000
+                        ]
+                    ]
+                }
+            ]
+        }
+    ]
+    }
+
+.. code-block:: shell
+
+    SELECT {0}("{0}_value") FROM "vmstats" WHERE host =~ /^{1}$/ AND
+    "type_instance" =~ /^mem$/ AND time >= now() - {2}s GROUP BY host
+
+.. _datasource baseclass: https://opendev.org/openstack/watcher/src/branch/master/watcher/datasources/base.py
+
+Example configuration
+---------------------
+
+The example configurations will show both how to achieve the entire
+configuration in the config file or use a combination of the regular file and
+yaml. Using yaml to define all the parameters for each metric is recommended
+since it has better human readability and supports mutli-line option
+definitions.
+
+Configuration file
+******************
+
+**It is important to note that the line breaks shown in between assignments of
+parameters can not be used in the actual configuration and these are simply here
+for readability reasons.**
+
+.. code-block:: shell
+
+    [grafana_client]
+    # Authentication token to gain access (string value)
+    # Note: This option can be changed without restarting.
+    token = eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk==
+
+    # first part of the url (including https:// or http://) up until project id
+    # part. Example: https://secure.org/api/datasource/proxy/ (string value)
+    # Note: This option can be changed without restarting.
+    base_url = https://monitoring-grafana.com/api/datasources/proxy/
+
+    # Project id as in url (integer value)
+    # Note: This option can be changed without restarting.
+    project_id_map = host_cpu_usage:1337,host_ram_usage:6969,
+    instance_cpu_usage:1337,instance_ram_usage:9696
+
+    # Mapping of grafana databases to datasource metrics. (dict value)
+    # Note: This option can be changed without restarting.
+    database_map = host_cpu_usage:monit_production,
+    host_ram_usage:monit_production,instance_cpu_usage:prod_cloud,
+    instance_ram_usage:prod_cloud
+
+    translator_map = host_cpu_usage:influxdb,host_ram_usage:influxdb,
+    instance_cpu_usage:influxdb,instance_ram_usage:influxdb
+
+    attribute_map = host_cpu_usage:hostname,host_ram_usage:hostname,
+    instance_cpu_usage:name,instance_ram_usage:name
+
+    query_map = host_cpu_usage:SELECT 100-{0}("{0}_value") FROM {4}.cpu WHERE
+    ("host" =~ /^{1}$/ AND "type_instance" =~/^idle$/ AND time > now()-{2}s),
+    host_ram_usage:SELECT {0}("{0}_value")/1000000 FROM {4}.memory WHERE
+     ("host" =~ /^{1}$/) AND "type_instance" =~ /^used$/ AND time >= now()-{2}s
+     GROUP BY "type_instance",instance_cpu_usage:SELECT {0}("{0}_value") FROM
+     "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^cpu$/ AND time >=
+     now() - {2}s GROUP BY host,instance_ram_usage:SELECT {0}("{0}_value") FROM
+     "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^mem$/ AND time >=
+     now() - {2}s GROUP BY host
+
+    [grafana_translators]
+
+    retention_periods = one_week:10080,one_month:302400,five_years:525600
+
+    [watcher_datasources]
+    datasources = grafana
+
+yaml
+****
+
+When using the yaml configuration file some parameters still need to be defined
+using the regular configuration such as the path for the yaml file these
+parameters are detailed below:
+
+.. code-block:: shell
+
+    [grafana_client]
+    token = eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk==
+
+    base_url = https://monitoring-grafana.com/api/datasources/proxy/
+
+    [watcher_datasources]
+    datasources = grafana
+
+    [watcher_decision_engine]
+    metric_map_path = /etc/watcher/metric_map.yaml
+
+Using the yaml allows to more effectively define the parameters per metric with
+greater human readability due to the availability of multi line options. These
+multi line options are demonstrated in the query parameters.
+
+.. code-block:: yaml
+
+  grafana:
+    host_cpu_usage:
+      project: 1337
+      db: monit_production
+      translator: influxdb
+      attribute: hostname
+      query: >
+          SELECT 100-{0}("{0}_value") FROM {4}.cpu
+          WHERE ("host" =~ /^{1}$/ AND "type_instance" =~/^idle$/ AND
+          time > now()-{2}s)
+    host_ram_usage:
+      project: 6969
+      db: monit_production
+      translator: influxdb
+      attribute: hostname
+      query: >
+          SELECT {0}("{0}_value")/1000000 FROM {4}.memory WHERE
+           ("host" =~ /^{1}$/) AND "type_instance" =~ /^used$/ AND time >=
+           now()-{2}s GROUP BY "type_instance"
+    instance_cpu_usage:
+      project: 1337
+      db: prod_cloud
+      translator: influxdb
+      attribute: name
+      query: >
+          SELECT {0}("{0}_value") FROM
+           "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^cpu$/ AND
+           time >= now() - {2}s GROUP BY host
+    instance_ram_usage:
+      project: 9696
+      db: prod_cloud
+      translator: influxdb
+      attribute: name
+      query: >
+          SELECT {0}("{0}_value") FROM
+           "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^mem$/ AND
+           time >= now() - {2}s GROUP BY host
+
+External Links
+--------------
+
+- `List of Grafana datasources <https://grafana.com/plugins?type=datasource>`_

doc/source/datasources/index.rst

@@ -0,0 +1,8 @@
+Datasources
+===========
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   ./*

doc/source/glossary.rst

@@ -8,7 +8,7 @@
 Glossary
 ========
 
-.. glossary::
+.. _glossary:
    :sorted:
 
 This page explains the different terms used in the Watcher system.

doc/source/index.rst

@@ -28,102 +28,25 @@ The documentation provided here is continually kept up-to-date based
 on the latest code, and may not represent the state of the project at any
 specific prior release.
 
-.. _watcher: https://git.openstack.org/cgit/openstack/watcher/
+.. _watcher: https://opendev.org/openstack/watcher/
-.. _python-watcherclient: https://git.openstack.org/cgit/openstack/python-watcherclient/
+.. _python-watcherclient: https://opendev.org/openstack/python-watcherclient/
-.. _watcher-dashboard: https://git.openstack.org/cgit/openstack/watcher-dashboard/
+.. _watcher-dashboard: https://opendev.org/openstack/watcher-dashboard/
-
-Developer Guide
-===============
-
-Introduction
-------------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
-  glossary
   architecture
-  contributor/contributing
-
-
-Getting Started
----------------
-
-.. toctree::
-  :maxdepth: 1
-
   contributor/index
-
-Installation
-============
-.. toctree::
-  :maxdepth: 2
-
   install/index
-
-Admin Guide
-===========
-
-.. toctree::
-  :maxdepth: 2
-
   admin/index
-
-User Guide
-==========
-
-.. toctree::
-  :maxdepth: 2
-
   user/index
-
-API References
-==============
-
-.. toctree::
-  :maxdepth: 1
-
-  API Reference <https://developer.openstack.org/api-ref/resource-optimization/>
-  Watcher API Microversion History </contributor/api_microversion_history>
-
-Plugins
--------
-
-.. toctree::
-  :maxdepth: 1
-
-  contributor/plugin/index
-
-Watcher Configuration Options
-=============================
-
-.. toctree::
-  :maxdepth: 2
-
   configuration/index
-
+  contributor/plugin/index
-Watcher Manual Pages
+  man/index
-====================
 
 .. toctree::
-   :glob:
+  :maxdepth: 1
-   :maxdepth: 1
 
-   man/index
+  API Reference <https://docs.openstack.org/api-ref/resource-optimization/>
-
+  Watcher API Microversion History </contributor/api_microversion_history>
-.. # NOTE(mriedem): This is the section where we hide things that we don't
+  glossary
-   # actually want in the table of contents but sphinx build would fail if
-   # they aren't in the toctree somewhere. For example, we hide api/autoindex
-   # since that's already covered with modindex below.
-.. toctree::
-   :hidden:
-
-   api/modules
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

doc/source/install/index.rst

@@ -1,6 +1,6 @@
-===================================
+=============
-Infrastructure Optimization service
+Install Guide
-===================================
+=============
 
 .. toctree::
    :maxdepth: 2

doc/source/man/index.rst

@@ -1,3 +1,7 @@
+====================
+Watcher Manual Pages
+====================
+
 .. toctree::
    :glob:
    :maxdepth: 1

doc/source/man/watcher-status.rst

@@ -78,6 +78,10 @@ Upgrade
 
   **History of Checks**
 
-  **1.12.0 (Stein)**
+  **2.0.0 (Stein)**
 
   * Sample check to be filled in with checks as they are added in Stein.
+
+  **3.0.0 (Train)**
+
+  * A check was added to enforce the minimum required version of nova API used.

doc/source/strategies/basic-server-consolidation.rst

@@ -19,12 +19,15 @@ Metrics
 
 The *basic* strategy requires the following metrics:
 
-============================ ============ ======= =======
+============================ ============ ======= ===========================
 metric                       service name plugins comment
-============================ ============ ======= =======
+============================ ============ ======= ===========================
-``compute.node.cpu.percent`` ceilometer_  none
+``compute.node.cpu.percent`` ceilometer_  none    need to set the
-``cpu_util``                 ceilometer_  none
+                                                  ``compute_monitors`` option
-============================ ============ ======= =======
+                                                  to ``cpu.virt_driver`` in
+                                                  the nova.conf.
+``cpu``                      ceilometer_  none
+============================ ============ ======= ===========================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 

doc/source/strategies/node_resource_consolidation.rst

@@ -0,0 +1,95 @@
+====================================
+Node Resource Consolidation Strategy
+====================================
+
+Synopsis
+--------
+
+**display name**: ``Node Resource Consolidation Strategy``
+
+**goal**: ``Server Consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.node_resource_consolidation.NodeResourceConsolidation
+
+Requirements
+------------
+
+None.
+
+Metrics
+*******
+
+None
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.weight.WeightPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+==================== ====== =======================================
+parameter            type   default Value description
+==================== ====== =======================================
+``host_choice``      String The way to select the server migration
+                            destination node, The value auto means
+                            that Nova schedular selects the
+                            destination node, and specify means
+                            the strategy specifies the destination.
+==================== ====== =======================================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Node Resource Consolidation Strategy please refer
+to: https://specs.openstack.org/openstack/watcher-specs/specs/train/approved/node-resource-consolidation.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 server_consolidation \
+      --strategy node_resource_consolidation
+
+    $ openstack optimize audit create \
+      -a at1 -p host_choice=auto
+
+External Links
+--------------
+
+None.

doc/source/strategies/saving_energy.rst

@@ -89,9 +89,9 @@ step 2: Create audit to do optimization
 .. code-block:: shell
 
     $ openstack optimize audittemplate create \
-      at1 saving_energy --strategy saving_energy
+      saving_energy_template1 saving_energy --strategy saving_energy
 
-    $ openstack optimize audit create -a at1 \
+    $ openstack optimize audit create -a saving_energy_audit1 \
       -p free_used_percent=20.0
 
 External Links

doc/source/strategies/vm_workload_consolidation.rst

@@ -19,16 +19,22 @@ Metrics
 
 The *vm_workload_consolidation* strategy requires the following metrics:
 
-============================ ============ ======= =======
+============================ ============ ======= =========================
 metric                       service name plugins comment
-============================ ============ ======= =======
+============================ ============ ======= =========================
-``cpu_util``                 ceilometer_  none
+``cpu``                      ceilometer_  none
 ``memory.resident``          ceilometer_  none
 ``memory``                   ceilometer_  none
 ``disk.root.size``           ceilometer_  none
-============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none    (optional) need to set the
+                                                  ``compute_monitors`` option
+                                                  to ``cpu.virt_driver`` in the
+                                                  nova.conf.
+``hardware.memory.used``     ceilometer_  SNMP_   (optional)
+============================ ============ ======= =========================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
+.. _SNMP: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#snmp-based-meters
 
 Cluster data model
 ******************

doc/source/strategies/workload-stabilization.rst

@@ -19,17 +19,20 @@ Metrics
 
 The *workload_stabilization* strategy requires the following metrics:
 
-============================ ============ ======= =======
+============================ ============ ======= =============================
 metric                       service name plugins comment
-============================ ============ ======= =======
+============================ ============ ======= =============================
-``compute.node.cpu.percent`` ceilometer_  none
+``compute.node.cpu.percent`` ceilometer_  none    need to set the
+                                                  ``compute_monitors`` option
+                                                  to ``cpu.virt_driver`` in the
+                                                  nova.conf.
 ``hardware.memory.used``     ceilometer_  SNMP_
-``cpu_util``                 ceilometer_  none
+``cpu``                      ceilometer_  none
-``memory.resident``          ceilometer_  none
+``instance_ram_usage``       ceilometer_  none
-============================ ============ ======= =======
+============================ ============ ======= =============================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
-.. _SNMP: http://docs.openstack.org/admin-guide/telemetry-measurements.html
+.. _SNMP: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#snmp-based-meters
 
 Cluster data model
 ******************
@@ -103,10 +106,10 @@ parameter            type   default Value         description
                                                   period of all received ones.
 ==================== ====== ===================== =============================
 
-.. |metrics| replace:: ["cpu_util", "memory.resident"]
+.. |metrics| replace:: ["instance_cpu_usage", "instance_ram_usage"]
-.. |thresholds| replace:: {"cpu_util": 0.2, "memory.resident": 0.2}
+.. |thresholds| replace:: {"instance_cpu_usage": 0.2, "instance_ram_usage": 0.2}
-.. |weights| replace:: {"cpu_util_weight": 1.0, "memory.resident_weight": 1.0}
+.. |weights| replace:: {"instance_cpu_usage_weight": 1.0, "instance_ram_usage_weight": 1.0}
-.. |instance_metrics| replace:: {"cpu_util": "compute.node.cpu.percent", "memory.resident": "hardware.memory.used"}
+.. |instance_metrics| replace:: {"instance_cpu_usage": "compute.node.cpu.percent", "instance_ram_usage": "hardware.memory.used"}
 .. |periods| replace:: {"instance": 720, "node": 600}
 
 Efficacy Indicator
@@ -132,8 +135,8 @@ How to use it ?
       at1 workload_balancing --strategy workload_stabilization
 
     $ openstack optimize audit create -a at1 \
-      -p thresholds='{"memory.resident": 0.05}' \
+      -p thresholds='{"instance_ram_usage": 0.05}' \
-      -p metrics='["memory.resident"]'
+      -p metrics='["instance_ram_usage"]'
 
 External Links
 --------------

doc/source/strategies/workload_balance.rst

@@ -21,12 +21,12 @@ Metrics
 
 The *workload_balance* strategy requires the following metrics:
 
-======================= ============ ======= =======
+======================= ============ ======= =========================
 metric                  service name plugins comment
-======================= ============ ======= =======
+======================= ============ ======= =========================
-``cpu_util``            ceilometer_  none
+``cpu``                 ceilometer_  none
 ``memory.resident``     ceilometer_  none
-======================= ============ ======= =======
+======================= ============ ======= =========================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 
@@ -64,15 +64,16 @@ Configuration
 
 Strategy parameters are:
 
-============== ====== ============= ====================================
+============== ====== ==================== ====================================
-parameter      type   default Value description
+parameter      type   default Value        description
-============== ====== ============= ====================================
+============== ====== ==================== ====================================
-``metrics``    String 'cpu_util'    Workload balance base on cpu or ram
+``metrics``    String 'instance_cpu_usage' Workload balance base on cpu or ram
-                                    utilization. choice: ['cpu_util',
+                                           utilization. Choices:
-                                    'memory.resident']
+                                           ['instance_cpu_usage',
-``threshold``  Number 25.0          Workload threshold for migration
+                                           'instance_ram_usage']
-``period``     Number 300           Aggregate time period of ceilometer
+``threshold``  Number 25.0                 Workload threshold for migration
-============== ====== ============= ====================================
+``period``     Number 300                  Aggregate time period of ceilometer
+============== ====== ==================== ====================================
 
 Efficacy Indicator
 ------------------
@@ -94,7 +95,7 @@ How to use it ?
       at1 workload_balancing --strategy workload_balance
 
     $ openstack optimize audit create -a at1 -p threshold=26.0 \
-            -p period=310 -p metrics=cpu_util
+            -p period=310 -p metrics=instance_cpu_usage
 
 External Links
 --------------

doc/source/user/event_type_audit.rst

@@ -0,0 +1,195 @@
+..
+  Licensed under the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License. You may obtain
+  a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+
+
+======================
+Audit using Aodh alarm
+======================
+
+Audit with EVENT type can be triggered by special alarm. This guide walks
+you through the steps to build an event-driven optimization solution by
+integrating Watcher with Ceilometer/Aodh.
+
+Step 1: Create an audit with EVENT type
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first step is to create an audit with EVENT type,
+you can create an audit template firstly:
+
+.. code-block:: bash
+
+  $ openstack optimize audittemplate create your_template_name <your_goal> \
+    --strategy <your_strategy>
+
+or create an audit directly with special goal and strategy:
+
+.. code-block:: bash
+
+  $ openstack optimize audit create --goal <your_goal> \
+    --strategy <your_strategy> --audit_type EVENT
+
+This is an example for creating an audit with dummy strategy:
+
+.. code-block:: bash
+
+  $ openstack optimize audit create --goal dummy \
+    --strategy dummy --audit_type EVENT
+  +---------------+--------------------------------------+
+  | Field         | Value                                |
+  +---------------+--------------------------------------+
+  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
+  | Name          | dummy-2020-01-14T03:21:19.168467     |
+  | Created At    | 2020-01-14T03:21:19.200279+00:00     |
+  | Updated At    | None                                 |
+  | Deleted At    | None                                 |
+  | State         | PENDING                              |
+  | Audit Type    | EVENT                                |
+  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
+  | Interval      | None                                 |
+  | Goal          | dummy                                |
+  | Strategy      | dummy                                |
+  | Audit Scope   | []                                   |
+  | Auto Trigger  | False                                |
+  | Next Run Time | None                                 |
+  | Hostname      | None                                 |
+  | Start Time    | None                                 |
+  | End Time      | None                                 |
+  | Force         | False                                |
+  +---------------+--------------------------------------+
+
+We need to build Aodh action url using Watcher webhook API.
+For convenience we export the url into an environment variable:
+
+.. code-block:: bash
+
+  $ export AUDIT_UUID=a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  $ export ALARM_URL="trust+http://localhost/infra-optim/v1/webhooks/$AUDIT_UUID"
+
+Step 2: Create Aodh Alarm
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Once we have the audit created, we can continue to create Aodh alarm and
+set the alarm action to Watcher webhook API. The alarm type can be event(
+i.e. ``compute.instance.create.end``) or gnocchi_resources_threshold(i.e.
+``cpu_util``), more info refer to alarm-creation_
+
+For example:
+
+.. code-block:: bash
+
+  $ openstack alarm create \
+    --type event --name instance_create \
+    --event-type "compute.instance.create.end" \
+    --enable True --repeat-actions False \
+    --alarm-action $ALARM_URL
+  +---------------------------+------------------------------------------------------------------------------------------+
+  | Field                     | Value                                                                                    |
+  +---------------------------+------------------------------------------------------------------------------------------+
+  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5'] |
+  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                     |
+  | description               | Alarm when compute.instance.create.end event occurred.                                   |
+  | enabled                   | True                                                                                     |
+  | event_type                | compute.instance.create.end                                                              |
+  | insufficient_data_actions | []                                                                                       |
+  | name                      | instance_create                                                                          |
+  | ok_actions                | []                                                                                       |
+  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                         |
+  | query                     |                                                                                          |
+  | repeat_actions            | False                                                                                    |
+  | severity                  | low                                                                                      |
+  | state                     | insufficient data                                                                        |
+  | state_reason              | Not evaluated yet                                                                        |
+  | state_timestamp           | 2020-01-14T03:56:26.894416                                                               |
+  | time_constraints          | []                                                                                       |
+  | timestamp                 | 2020-01-14T03:56:26.894416                                                               |
+  | type                      | event                                                                                    |
+  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                         |
+  +---------------------------+------------------------------------------------------------------------------------------+
+
+.. _alarm-creation: https://docs.openstack.org/aodh/latest/admin/telemetry-alarms.html#alarm-creation
+
+Step 3: Trigger the alarm
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this example, you can create a new instance to trigger the alarm.
+The alarm state will translate from ``insufficient data`` to ``alarm``.
+
+.. code-block:: bash
+
+  $ openstack alarm show b9e381fc-8e3e-4943-82ee-647e7a2ef644
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+  | Field                     | Value                                                                                                             |
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5']                          |
+  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                                              |
+  | description               | Alarm when compute.instance.create.end event occurred.                                                            |
+  | enabled                   | True                                                                                                              |
+  | event_type                | compute.instance.create.end                                                                                       |
+  | insufficient_data_actions | []                                                                                                                |
+  | name                      | instance_create                                                                                                   |
+  | ok_actions                | []                                                                                                                |
+  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                                                  |
+  | query                     |                                                                                                                   |
+  | repeat_actions            | False                                                                                                             |
+  | severity                  | low                                                                                                               |
+  | state                     | alarm                                                                                                             |
+  | state_reason              | Event <id=67dd0afa-2082-45a4-8825-9573b2cc60e5,event_type=compute.instance.create.end> hits the query <query=[]>. |
+  | state_timestamp           | 2020-01-14T03:56:26.894416                                                                                        |
+  | time_constraints          | []                                                                                                                |
+  | timestamp                 | 2020-01-14T06:17:40.350649                                                                                        |
+  | type                      | event                                                                                                             |
+  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                                                  |
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+
+Step 4: Verify the audit
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can be verified to check if the audit state was ``SUCCEEDED``:
+
+.. code-block:: bash
+
+  $ openstack optimize audit show a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  +---------------+--------------------------------------+
+  | Field         | Value                                |
+  +---------------+--------------------------------------+
+  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
+  | Name          | dummy-2020-01-14T03:21:19.168467     |
+  | Created At    | 2020-01-14T03:21:19+00:00            |
+  | Updated At    | 2020-01-14T06:26:40+00:00            |
+  | Deleted At    | None                                 |
+  | State         | SUCCEEDED                            |
+  | Audit Type    | EVENT                                |
+  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
+  | Interval      | None                                 |
+  | Goal          | dummy                                |
+  | Strategy      | dummy                                |
+  | Audit Scope   | []                                   |
+  | Auto Trigger  | False                                |
+  | Next Run Time | None                                 |
+  | Hostname      | ubuntudbs                            |
+  | Start Time    | None                                 |
+  | End Time      | None                                 |
+  | Force         | False                                |
+  +---------------+--------------------------------------+
+
+and you can use the following command to check if the action plan
+was created:
+
+.. code-block:: bash
+
+  $ openstack optimize actionplan list --audit a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
+  | UUID                                 | Audit                                | State       | Updated At | Global efficacy |
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
+  | 673b3fcb-8c16-4a41-9ee3-2956d9f6ca9e | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 | RECOMMENDED | None       |                 |
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+

doc/source/user/index.rst

@@ -1,4 +1,10 @@
+==========
+User Guide
+==========
+
 .. toctree::
    :maxdepth: 2
 
+   ways-to-install
    user-guide
+   event_type_audit

doc/source/user/user-guide.rst

@@ -4,8 +4,6 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-.. _user-guide:
-
 ==================
 Watcher User Guide
 ==================
@@ -55,13 +53,13 @@ plugin installation guide`_.
 
      $ watcher ...
 
-.. _`installation guide`: https://docs.openstack.org/python-watcherclient/latest
+.. _`installation guide`: https://docs.openstack.org/watcher/latest/install/
 .. _`Watcher Horizon plugin installation guide`: https://docs.openstack.org/watcher-dashboard/latest/install/installation.html
 .. _`OpenStack CLI`: https://docs.openstack.org/python-openstackclient/latest/cli/man/openstack.html
 .. _`Watcher CLI`: https://docs.openstack.org/python-watcherclient/latest/cli/index.html
 
-Seeing what the Watcher CLI can do ?
+Watcher CLI Command
-------------------------------------
+-------------------
 We can see all of the commands available with Watcher CLI by running the
 watcher binary without options.
 
@@ -69,8 +67,8 @@ watcher binary without options.
 
   $ openstack help optimize
 
-How do I run an audit of my cluster ?
+Running an audit of the cluster
--------------------------------------
+-------------------------------
 
 First, you need to find the :ref:`goal <goal_definition>` you want to achieve:
 

doc/source/user/ways-to-install.rst

@@ -51,7 +51,7 @@ Clone the Watcher repository:
 
 .. code-block:: bash
 
-    $ git clone https://git.openstack.org/openstack/watcher.git
+    $ git clone https://opendev.org/openstack/watcher.git
     $ cd watcher
 
 Install the Watcher modules:
@@ -92,7 +92,7 @@ these commands:
 By default, this will show logging on the console from which it was started.
 Once started, you can use the `Watcher Client`_ to play with Watcher service.
 
-.. _`Watcher Client`: https://git.openstack.org/cgit/openstack/python-watcherclient
+.. _`Watcher Client`: https://opendev.org/openstack/python-watcherclient
 
 Installing from packages: PyPI
 --------------------------------

etc/apache2/watcher

@@ -16,7 +16,7 @@ Listen 9322
 
 <VirtualHost *:9322>
     WSGIDaemonProcess watcher-api user=stack group=stack processes=2 threads=2 display-name=%{GROUP}
-    WSGIScriptAlias / /opt/stack/watcher/watcher/api/app.wsgi
+    WSGIScriptAlias / /usr/local/bin/watcher-api-wsgi
     WSGIProcessGroup watcher-api
 
     ErrorLog /var/log/httpd/watcher_error.log

lower-constraints.txt

@@ -1,165 +0,0 @@
-alabaster==0.7.10
-alembic==0.9.8
-amqp==2.2.2
-appdirs==1.4.3
-APScheduler==3.5.1
-asn1crypto==0.24.0
-automaton==1.14.0
-Babel==2.5.3
-bandit==1.4.0
-beautifulsoup4==4.6.0
-cachetools==2.0.1
-certifi==2018.1.18
-cffi==1.11.5
-chardet==3.0.4
-cliff==2.11.0
-cmd2==0.8.1
-contextlib2==0.5.5
-coverage==4.5.1
-croniter==0.3.20
-cryptography==2.1.4
-debtcollector==1.19.0
-decorator==4.2.1
-deprecation==2.0
-doc8==0.8.0
-docutils==0.14
-dogpile.cache==0.6.5
-dulwich==0.19.0
-enum34==1.1.6
-enum-compat==0.0.2
-eventlet==0.20.0
-extras==1.0.0
-fasteners==0.14.1
-fixtures==3.0.0
-flake8==2.5.5
-freezegun==0.3.10
-future==0.16.0
-futurist==1.6.0
-gitdb2==2.0.3
-GitPython==2.1.8
-gnocchiclient==7.0.1
-greenlet==0.4.13
-hacking==0.12.0
-idna==2.6
-imagesize==1.0.0
-iso8601==0.1.12
-Jinja2==2.10
-jmespath==0.9.3
-jsonpatch==1.21
-jsonpointer==2.0
-jsonschema==2.6.0
-keystoneauth1==3.4.0
-keystonemiddleware==4.21.0
-kombu==4.1.0
-linecache2==1.0.0
-logutils==0.3.5
-lxml==4.1.1
-Mako==1.0.7
-MarkupSafe==1.0
-mccabe==0.2.1
-microversion_parse==0.2.1
-mock==2.0.0
-monotonic==1.4
-mox3==0.25.0
-msgpack==0.5.6
-munch==2.2.0
-netaddr==0.7.19
-netifaces==0.10.6
-networkx==1.11
-openstackdocstheme==1.20.0
-openstacksdk==0.12.0
-os-api-ref===1.4.0
-os-client-config==1.29.0
-os-service-types==1.2.0
-os-testr==1.0.0
-osc-lib==1.10.0
-oslo.cache==1.29.0
-oslo.concurrency==3.26.0
-oslo.config==5.2.0
-oslo.context==2.20.0
-oslo.db==4.35.0
-oslo.i18n==3.20.0
-oslo.log==3.37.0
-oslo.messaging==5.36.0
-oslo.middleware==3.35.0
-oslo.policy==1.34.0
-oslo.reports==1.27.0
-oslo.serialization==2.25.0
-oslo.service==1.30.0
-oslo.upgradecheck==0.1.0
-oslo.utils==3.36.0
-oslo.versionedobjects==1.32.0
-oslotest==3.3.0
-packaging==17.1
-Paste==2.0.3
-PasteDeploy==1.5.2
-pbr==3.1.1
-pecan==1.2.1
-pep8==1.5.7
-pika==0.10.0
-pika-pool==0.1.3
-prettytable==0.7.2
-psutil==5.4.3
-pycadf==2.7.0
-pycparser==2.18
-pyflakes==0.8.1
-Pygments==2.2.0
-pyinotify==0.9.6
-pyOpenSSL==17.5.0
-pyparsing==2.2.0
-pyperclip==1.6.0
-python-ceilometerclient==2.9.0
-python-cinderclient==3.5.0
-python-dateutil==2.7.0
-python-editor==1.0.3
-python-glanceclient==2.9.1
-python-ironicclient==2.3.0
-python-keystoneclient==3.15.0
-python-mimeparse==1.6.0
-python-monascaclient==1.12.0
-python-neutronclient==6.7.0
-python-novaclient==10.1.0
-python-openstackclient==3.14.0
-python-subunit==1.2.0
-pytz==2018.3
-PyYAML==3.12
-reno==2.7.0
-repoze.lru==0.7
-requests==2.18.4
-requestsexceptions==1.4.0
-restructuredtext-lint==1.1.3
-rfc3986==1.1.0
-Routes==2.4.1
-simplegeneric==0.8.1
-simplejson==3.13.2
-six==1.11.0
-smmap2==2.0.3
-snowballstemmer==1.2.1
-Sphinx==1.6.5
-sphinxcontrib-httpdomain==1.6.1
-sphinxcontrib-pecanwsme==0.8.0
-sphinxcontrib-websupport==1.0.1
-SQLAlchemy==1.2.5
-sqlalchemy-migrate==0.11.0
-sqlparse==0.2.4
-statsd==3.2.2
-stestr==2.0.0
-stevedore==1.28.0
-taskflow==3.1.0
-Tempita==0.5.2
-tenacity==4.9.0
-testresources==2.0.1
-testscenarios==0.5.0
-testtools==2.3.0
-traceback2==1.4.0
-tzlocal==1.5.1
-ujson==1.35
-unittest2==1.1.0
-urllib3==1.22
-vine==1.1.4
-waitress==1.1.0
-warlock==1.3.0
-WebOb==1.7.4
-WebTest==2.0.29
-wrapt==1.10.11
-WSME==0.9.2

playbooks/legacy/grenade-devstack-watcher/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/grenade-devstack-watcher/run.yaml

@@ -1,60 +0,0 @@
-- hosts: all
-  name: legacy-grenade-dsvm-watcher
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-
-          export PROJECTS="openstack-dev/grenade $PROJECTS"
-          export PROJECTS="openstack/watcher $PROJECTS"
-          export PROJECTS="openstack/watcher-tempest-plugin $PROJECTS"
-          export PROJECTS="openstack/python-watcherclient $PROJECTS"
-          export DEVSTACK_PROJECT_FROM_GIT="python-watcherclient $DEVSTACK_PROJECT_FROM_GIT"
-
-          export GRENADE_PLUGINRC="enable_grenade_plugin watcher https://git.openstack.org/openstack/watcher"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"export TEMPEST_PLUGINS='/opt/stack/new/watcher-tempest-plugin'"
-
-          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
-          export DEVSTACK_GATE_GRENADE=pullup
-
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-          # Add configuration values for enabling security features in local.conf
-          function pre_test_hook {
-              if [ -f /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh ] ; then
-                  . /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh
-              fi
-          }
-          export -f pre_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

releasenotes/notes/add-force-field-to-audit-4bcaeedfe27233ad.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Add force field to Audit. User can set --force to enable the new option when
+    launching audit. If force is True, audit will be executed despite of ongoing
+    actionplan. The new audit may create a wrong actionplan if they use the same
+    data model.

releasenotes/notes/api-call-retry-fef741ac684c58dd.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    API calls while building the Compute data model will be retried upon
+    failure. The amount of failures allowed before giving up and the time before
+    reattempting are configurable. The `api_call_retries` and
+    `api_query_timeout` parameters in the `[collector]` group can be used to
+    adjust these paremeters. 10 retries with a 1 second time in between
+    reattempts is the default.

releasenotes/notes/bug-2112187-763bae283e0b736d.yaml

@@ -0,0 +1,47 @@
+---
+security:
+  - |
+    Watchers no longer forges requests on behalf of a tenant when
+    swapping volumes. Prior to this release watcher had 2 implementations
+    of moving a volume, it could use cinders volume migrate api or its own
+    internal implementation that directly calls nova volume attachment update
+    api. The former is safe and the recommend way to move volumes between
+    cinder storage backend the internal implementation was insecure, fragile
+    due to a lack of error handling and capable of deleting user data.
+
+    Insecure: the internal volume migration operation created a new keystone
+    user with a weak name and password and added it to the tenants project
+    with the admin role. It then used that user to forge request on behalf
+    of the tenant with admin right to swap the volume. if the applier was
+    restarted during the execution of this operation it would never be cleaned
+    up.
+
+    Fragile: the error handling was minimal, the swap volume api is async
+    so watcher has to poll for completion, there was no support to resume
+    that if interrupted of the time out was exceeded.
+
+    Data-loss: while the internal polling logic returned success or failure
+    watcher did not check the result, once the function returned it
+    unconditionally deleted the source volume. For larger volumes this
+    could result in irretrievable data loss.
+
+    Finally if a volume was swapped using the internal workflow it put
+    the nova instance in an out of sync state. If the VM was live migrated
+    after the swap volume completed successfully prior to a hard reboot
+    then the migration would fail or succeed and break tenant isolation.
+
+    see: https://bugs.launchpad.net/nova/+bug/2112187 for details.
+fixes:
+  - |
+    All code related to creating keystone user and granting roles has been
+    removed. The internal swap volume implementation has been removed and
+    replaced by cinders volume migrate api. Note as part of this change
+    Watcher will no longer attempt volume migrations or retypes if the
+    instance is in the `Verify Resize` task state. This resolves several
+    issues related to volume migration in the zone migration and
+    Storage capacity balance strategies. While efforts have been made
+    to maintain backward compatibility these changes are required to
+    address a security weakness in watcher's prior approach.
+
+    see: https://bugs.launchpad.net/nova/+bug/2112187 for more context.
+

releasenotes/notes/datasource-query-retry-00cba5f7e68aec39.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    All datasources can now be configured to retry retrieving a metric upon
+    encountering an error. Between each attempt will be a set amount of time
+    which can be adjusted from the configuration. These configuration
+    options can be found in the `[watcher_datasources]` group and are named
+    `query_max_retries` and `query_timeout`.
+upgrade:
+  - |
+    If Gnocchi was configured to have a custom amount of retries and or a
+    custom timeout then the configuration needs to moved into the
+    `[watcher_datasources]` group instead of the `[gnocchi_client]` group.
+deprecations:
+  - |
+    The configuration options for query retries in `[gnocchi_client]` are
+    deprecated and the option in `[watcher_datasources]` should now be used.

releasenotes/notes/deprecate-json-formatted-policy-file-3a92379e9f5dd203.yaml

@@ -0,0 +1,20 @@
+---
+upgrade:
+  - |
+    The default value of ``[oslo_policy] policy_file`` config option has
+    been changed from ``policy.json`` to ``policy.yaml``.
+    Operators who are utilizing customized or previously generated
+    static policy JSON files (which are not needed by default), should
+    generate new policy files or convert them in YAML format. Use the
+    `oslopolicy-convert-json-to-yaml
+    <https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_
+    tool to convert a JSON to YAML formatted policy file in
+    backward compatible way.
+deprecations:
+  - |
+    Use of JSON policy files was deprecated by the ``oslo.policy`` library
+    during the Victoria development cycle. As a result, this deprecation is
+    being noted in the Wallaby cycle with an anticipated future removal of support
+    by ``oslo.policy``. As such operators will need to convert to YAML policy
+    files. Please see the upgrade notes for details on migration of any
+    custom policy files.

releasenotes/notes/drop-py-2-7-54f8e806d71f19a7.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Python 2.7 support has been dropped. Last release of Watcher
+    to support py2.7 is OpenStack Train. The minimum version of Python now
+    supported by Watcher is Python 3.6.

releasenotes/notes/event-driven-optimization-based-4870f112bef8a560.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add a new webhook API and a new audit type EVENT, the microversion is 1.4.
+    Now Watcher user can create audit with EVENT type and the audit will be
+    triggered by webhook API.
+    The user guide is available online:
+    https://docs.openstack.org/watcher/latest/user/event_type_audit.html

releasenotes/notes/file-based-metric-map-c2af62b5067895df.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Allow using file to override metric map. Override the metric map of
+    each datasource as soon as it is created by the manager. This override
+    comes from a file whose path is provided by a setting in config file.
+    The setting is `watcher_decision_engine/metric_map_path`.  The file
+    contains a map per datasource whose keys are the metric names as
+    recognized by watcher and the value is the real name of the metric
+    in the datasource. This setting defaults to `/etc/watcher/metric_map.yaml`,
+    and presence of this file is optional.

releasenotes/notes/formal-datasource-interface-implementation-222769d55a127d33.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Improved interface for datasource baseclass that better defines expected
+    values and types for parameters and return types of all abstract methods.
+    This allows all strategies to work with every datasource provided the
+    metrics are configured for that given datasource.
+deprecations:
+  - |
+    The new strategy baseclass has significant changes in method parameters
+    and any out-of-tree strategies will have to be adopted.
+  - |
+    Several strategies have changed the `node` parameter to `compute_node` to
+    be better aligned with terminology. These strategies include
+    `basic_consolidation` and `workload_stabilzation`. The `node` parameter
+    will remain supported during Train release and will be removed in the
+    subsequent release.

releasenotes/notes/general-purpose-decision-engine-threadpool-0711b23abfc9d409.yaml

@@ -0,0 +1,20 @@
+---
+prelude: >
+    Many operations in the decision engine will block on I/O. Such I/O
+    operations can stall the execution of a sequential application
+    significantly. To reduce the potential bottleneck of many operations
+    the general purpose decision engine threadpool is introduced.
+features:
+  - |
+    A new threadpool for the decision engine that contributors can use to
+    improve the performance of many operations, primarily I/O bound onces.
+    The amount of workers used by the decision engine threadpool can be
+    configured to scale according to the available infrastructure using
+    the `watcher_decision_engine.max_general_workers` config option.
+    Documentation for contributors to effectively use this threadpool is
+    available online:
+    https://docs.openstack.org/watcher/latest/contributor/concurrency.html
+  - |
+    The building of the compute (Nova) data model will be done using the
+    decision engine threadpool, thereby, significantly reducing the total
+    time required to build it.

releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Watcher now supports configuring which datasource to use and in which
+    order. This configuration is done by specifying datasources in the
+    watcher_datasources section:
+
+    - ``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``
+
+    Specific strategies can override this order and use datasources which
+    are not listed in the global preference.

releasenotes/notes/grafana-datasource-b672367c23ffa0c6.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Grafana has been added as datasource that can be used for collecting
+    metrics. The configuration options allow to specify what metrics and how
+    they are stored in grafana so that no matter how Grafana is configured it
+    can still be used. The configuration can be done via the typical
+    configuration file but it is recommended to configure most options in the
+    yaml file for metrics. For a complete walkthrough on configuring Grafana
+    see: https://docs.openstack.org/watcher/latest/datasources/grafana.html

releasenotes/notes/improve-compute-data-model-b427c85e4ed2b6fb.yaml

@@ -0,0 +1,23 @@
+---
+features:
+  - |
+    Watcher can get resource information such as total, allocation ratio and
+    reserved information from Placement API.
+    Now we add some new fields to the Watcher Data Model:
+
+    * vcpu_reserved: The amount of cpu a node has reserved for its own use.
+    * vcpu_ratio: CPU allocation ratio.
+    * memory_mb_reserved: The amount of memory a node has reserved for
+      its own use.
+    * memory_ratio: Memory allocation ratio.
+    * disk_gb_reserved: The amount of disk a node has reserved for its own use.
+    * disk_ratio: Disk allocation ratio.
+
+    We also add some new propeties:
+
+    * vcpu_capacity: The amount of vcpu, take allocation ratio into account,
+      but do not include reserved.
+    * memory_mb_capacity: The amount of memory, take allocation ratio into
+      account, but do not include reserved.
+    * disk_gb_capacity: The amount of disk, take allocation ratio into
+      account, but do not include reserved.

releasenotes/notes/min-required-nova-train-71f124192d88ae52.yaml

@@ -0,0 +1,8 @@
+---
+upgrade:
+  - |
+    The minimum required version of the ``[nova_client]/api_version`` value
+    is now enforced to be ``2.56`` which is available since the Queens version
+    of the nova compute service.
+
+    A ``watcher-status upgrade check`` has been added for this.

releasenotes/notes/node-resource-consolidation-73bc0c0abfeb0b03.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Added strategy "node resource consolidation". This
+    strategy is used to centralize VMs to as few nodes
+    as possible by VM migration. User can set an input
+    parameter to decide how to select the destination node.

releasenotes/notes/show-datamodel-api-6945b744fd5d25d5.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Add show data model api for Watcher. New in version 1.3.
+    User can use 'openstack optimize datamodel list'
+    command to view the current data model information in memory.
+    User can also add '--audit <Audit_UUID>' to view specific data model
+    in memory filted by the scope in audit.
+    User can also add '--detail' to view detailed information about current data model.
+    User can also add '--type <type>' to specify the type of data model.
+    Default type is 'compute'. In the future, type 'storage' and 'baremetal'
+    will be supported.

releasenotes/notes/support-keystoneclient-option-b30d1ff45f86a2e7.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add keystone_client Group for user to configure 'interface' and 'region_name'
+    by watcher.conf. The default value of 'interface' is 'admin'.

releasenotes/notes/support-placement-api-58ce6bef1bbbe98a.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Added Placement API helper to Watcher. Now Watcher can get information
+    about resource providers, it can be used for the data model and strategies.
+    Config group placement_client with options 'api_version', 'interface' and
+    'region_name' is also added. The default values for 'api_version' and
+    'interface' are 1.29 and 'public', respectively.

releasenotes/notes/uwsgi-support-8dcea6961e56dad0.yaml

@@ -0,0 +1,12 @@
+---
+upgrade:
+  - |
+    An Watcher API WSGI application script ``watcher-api-wsgi`` is now
+    available. It is auto-generated by ``pbr`` and allows to run the API
+    service using WSGI server (for example Nginx and uWSGI).
+deprecations:
+  - |
+    Using ``watcher/api/app.wsgi`` script is deprecated and it will be removed
+    in U release.
+    Please switch to automatically generated ``watcher-api-wsgi`` script
+    instead.

releasenotes/notes/watcher-planner-selector-84d77549d46f362a.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Now Watcher strategy can select specific planner
+    beyond default. Strategy can set planner property
+    to specify its own planner.

releasenotes/source/2023.1.rst

@@ -0,0 +1,6 @@
+===========================
+2023.1 Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/2023.1

releasenotes/source/2023.2.rst

@@ -0,0 +1,6 @@
+===========================
+2023.2 Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/2023.2

releasenotes/source/conf.py

@@ -53,8 +53,7 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'watcher'
+copyright = '2016, Watcher developers'
-copyright = u'2016, Watcher developers'
 
 # Release notes are version independent
 # The short X.Y version.
@@ -91,11 +90,15 @@ exclude_patterns = ['_build']
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []
 
+# openstackdocstheme options
+openstackdocs_repo_name = 'openstack/watcher'
+openstackdocs_bug_project = 'watcher'
+openstackdocs_bug_tag = ''
 
 # -- Options for HTML output --------------------------------------------------
 
@@ -193,8 +196,8 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual])
 latex_documents = [
-  ('index', 'watcher.tex', u'Watcher Documentation',
+  ('index', 'watcher.tex', 'Watcher Documentation',
-   u'Watcher developers', 'manual'),
+   'Watcher developers', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -223,8 +226,8 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'watcher', u'Watcher Documentation',
+    ('index', 'watcher', 'Watcher Documentation',
-     [u'Watcher developers'], 1)
+     ['Watcher developers'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -237,8 +240,8 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'watcher', u'Watcher Documentation',
+  ('index', 'watcher', 'Watcher Documentation',
-   u'Watcher developers', 'watcher', 'One line description of project.',
+   'Watcher developers', 'watcher', 'One line description of project.',
    'Miscellaneous'),
 ]
 

releasenotes/source/index.rst

@@ -21,6 +21,16 @@ Contents:
    :maxdepth: 1
 
    unreleased
+   2023.2
+   2023.1
+   zed
+   yoga
+   xena
+   wallaby
+   victoria
+   ussuri
+   train
+   stein
    rocky
    queens
    pike

releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po

@@ -1,14 +1,17 @@
 # Andi Chandler <andi@gowling.com>, 2017. #zanata
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
+# Andi Chandler <andi@gowling.com>, 2020. #zanata
+# Andi Chandler <andi@gowling.com>, 2022. #zanata
+# Andi Chandler <andi@gowling.com>, 2023. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: python-watcher\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-11-08 01:22+0000\n"
+"POT-Creation-Date: 2023-08-14 03:05+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2018-11-07 06:15+0000\n"
+"PO-Revision-Date: 2023-06-21 07:54+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -54,6 +57,67 @@ msgstr "1.7.0"
 msgid "1.9.0"
 msgstr "1.9.0"
 
+msgid "2.0.0"
+msgstr "2.0.0"
+
+msgid "2023.1 Series Release Notes"
+msgstr "2023.1 Series Release Notes"
+
+msgid "3.0.0"
+msgstr "3.0.0"
+
+msgid "4.0.0"
+msgstr "4.0.0"
+
+msgid "6.0.0"
+msgstr "6.0.0"
+
+msgid "A ``watcher-status upgrade check`` has been added for this."
+msgstr "A ``watcher-status upgrade check`` has been added for this."
+
+msgid ""
+"A new threadpool for the decision engine that contributors can use to "
+"improve the performance of many operations, primarily I/O bound onces. The "
+"amount of workers used by the decision engine threadpool can be configured "
+"to scale according to the available infrastructure using the "
+"`watcher_decision_engine.max_general_workers` config option. Documentation "
+"for contributors to effectively use this threadpool is available online: "
+"https://docs.openstack.org/watcher/latest/contributor/concurrency.html"
+msgstr ""
+"A new threadpool for the decision engine that contributors can use to "
+"improve the performance of many operations, primarily I/O bound onces. The "
+"amount of workers used by the decision engine threadpool can be configured "
+"to scale according to the available infrastructure using the "
+"`watcher_decision_engine.max_general_workers` config option. Documentation "
+"for contributors to effectively use this threadpool is available online: "
+"https://docs.openstack.org/watcher/latest/contributor/concurrency.html"
+
+msgid ""
+"API calls while building the Compute data model will be retried upon "
+"failure. The amount of failures allowed before giving up and the time before "
+"reattempting are configurable. The `api_call_retries` and "
+"`api_query_timeout` parameters in the `[collector]` group can be used to "
+"adjust these paremeters. 10 retries with a 1 second time in between "
+"reattempts is the default."
+msgstr ""
+"API calls while building the Compute data model will be retried upon "
+"failure. The amount of failures allowed before giving up and the time before "
+"reattempting are configurable. The `api_call_retries` and "
+"`api_query_timeout` parameters in the `[collector]` group can be used to "
+"adjust these parameters. 10 retries with a 1 second time in between "
+"reattempts is the default."
+
+msgid ""
+"Add a new webhook API and a new audit type EVENT, the microversion is 1.4. "
+"Now Watcher user can create audit with EVENT type and the audit will be "
+"triggered by webhook API. The user guide is available online: https://docs."
+"openstack.org/watcher/latest/user/event_type_audit.html"
+msgstr ""
+"Add a new webhook API and a new audit type EVENT, the microversion is 1.4. "
+"Now Watcher user can create audit with EVENT type and the audit will be "
+"triggered by webhook API. The user guide is available online: https://docs."
+"openstack.org/watcher/latest/user/event_type_audit.html"
+
 msgid "Add a service supervisor to watch Watcher deamons."
 msgstr "Add a service supervisor to watch Watcher daemons."
 
@@ -67,6 +131,24 @@ msgstr ""
 "Add description property for dynamic action. Admin can see detail "
 "information of any specify action."
 
+msgid ""
+"Add force field to Audit. User can set --force to enable the new option when "
+"launching audit. If force is True, audit will be executed despite of ongoing "
+"actionplan. The new audit may create a wrong actionplan if they use the same "
+"data model."
+msgstr ""
+"Add force field to Audit. User can set --force to enable the new option when "
+"launching audit. If force is True, audit will be executed despite of ongoing "
+"actionplan. The new audit may create a wrong actionplan if they use the same "
+"data model."
+
+msgid ""
+"Add keystone_client Group for user to configure 'interface' and "
+"'region_name' by watcher.conf. The default value of 'interface' is 'admin'."
+msgstr ""
+"Add keystone_client Group for user to configure 'interface' and "
+"'region_name' by watcher.conf. The default value of 'interface' is 'admin'."
+
 msgid "Add notifications related to Action object."
 msgstr "Add notifications related to Action object."
 
@@ -79,6 +161,25 @@ msgstr "Add notifications related to Audit object."
 msgid "Add notifications related to Service object."
 msgstr "Add notifications related to Service object."
 
+msgid ""
+"Add show data model api for Watcher. New in version 1.3. User can use "
+"'openstack optimize datamodel list' command to view the current data model "
+"information in memory. User can also add '--audit <Audit_UUID>' to view "
+"specific data model in memory filted by the scope in audit. User can also "
+"add '--detail' to view detailed information about current data model. User "
+"can also add '--type <type>' to specify the type of data model. Default type "
+"is 'compute'. In the future, type 'storage' and 'baremetal' will be "
+"supported."
+msgstr ""
+"Add show data model API for Watcher. New in version 1.3. User can use "
+"'openstack optimize datamodel list' command to view the current data model "
+"information in memory. User can also add '--audit <Audit_UUID>' to view "
+"specific data model in memory filtered by the scope in audit. User can also "
+"add '--detail' to view detailed information about current data model. User "
+"can also add '--type <type>' to specify the type of data model. Default type "
+"is 'compute'. In the future, type 'storage' and 'baremetal' will be "
+"supported."
+
 msgid ""
 "Add start_time and end_time fields in audits table. User can set the start "
 "time and/or end time when creating CONTINUOUS audit."
@@ -93,6 +194,19 @@ msgstr ""
 "Add superseded state for an action plan if the cluster data model has "
 "changed after it has been created."
 
+msgid ""
+"Added Placement API helper to Watcher. Now Watcher can get information about "
+"resource providers, it can be used for the data model and strategies. Config "
+"group placement_client with options 'api_version', 'interface' and "
+"'region_name' is also added. The default values for 'api_version' and "
+"'interface' are 1.29 and 'public', respectively."
+msgstr ""
+"Added Placement API helper to Watcher. Now Watcher can get information about "
+"resource providers, it can be used for the data model and strategies. Config "
+"group placement_client with options 'api_version', 'interface' and "
+"'region_name' is also added. The default values for 'api_version' and "
+"'interface' are 1.29 and 'public', respectively."
+
 msgid "Added SUSPENDED audit state"
 msgstr "Added SUSPENDED audit state"
 
@@ -107,6 +221,31 @@ msgstr ""
 "scoring engine by different Strategies, which improve the code and data "
 "model re-use."
 
+msgid ""
+"Added a new config option 'action_execution_rule' which is a dict type. Its "
+"key field is strategy name and the value is 'ALWAYS' or 'ANY'. 'ALWAYS' "
+"means the callback function returns True as usual. 'ANY' means the return "
+"depends on the result of previous action execution. The callback returns "
+"True if previous action gets failed, and the engine continues to run the "
+"next action. If previous action executes success, the callback returns False "
+"then the next action will be ignored. For strategies that aren't in "
+"'action_execution_rule', the callback always returns True. Please add the "
+"next section in the watcher.conf file if your strategy needs this feature. "
+"[watcher_workflow_engines.taskflow] action_execution_rule = {'your strategy "
+"name': 'ANY'}"
+msgstr ""
+"Added a new config option 'action_execution_rule' which is a dict type. Its "
+"key field is strategy name and the value is 'ALWAYS' or 'ANY'. 'ALWAYS' "
+"means the callback function returns True as usual. 'ANY' means the return "
+"depends on the result of previous action execution. The callback returns "
+"True if previous action gets failed, and the engine continues to run the "
+"next action. If previous action executes success, the callback returns False "
+"then the next action will be ignored. For strategies that aren't in "
+"'action_execution_rule', the callback always returns True. Please add the "
+"next section in the watcher.conf file if your strategy needs this feature. "
+"[watcher_workflow_engines.taskflow] action_execution_rule = {'your strategy "
+"name': 'ANY'}"
+
 msgid ""
 "Added a new strategy based on the airflow of servers. This strategy makes "
 "decisions to migrate VMs to make the airflow uniform."
@@ -248,6 +387,15 @@ msgstr ""
 "The strategy migrates many instances and volumes efficiently with minimum "
 "downtime automatically."
 
+msgid ""
+"Added strategy \"node resource consolidation\". This strategy is used to "
+"centralize VMs to as few nodes as possible by VM migration. User can set an "
+"input parameter to decide how to select the destination node."
+msgstr ""
+"Added strategy \"node resource consolidation\". This strategy is used to "
+"centralize VMs to as few nodes as possible by VM migration. User can set an "
+"input parameter to decide how to select the destination node."
+
 msgid ""
 "Added strategy to identify and migrate a Noisy Neighbor - a low priority VM "
 "that negatively affects peformance of a high priority VM by over utilizing "
@@ -284,6 +432,19 @@ msgstr ""
 msgid "Adds baremetal data model in Watcher"
 msgstr "Adds baremetal data model in Watcher"
 
+msgid ""
+"All datasources can now be configured to retry retrieving a metric upon "
+"encountering an error. Between each attempt will be a set amount of time "
+"which can be adjusted from the configuration. These configuration options "
+"can be found in the `[watcher_datasources]` group and are named "
+"`query_max_retries` and `query_timeout`."
+msgstr ""
+"All datasources can now be configured to retry retrieving a metric upon "
+"encountering an error. Between each attempt will be a set amount of time "
+"which can be adjusted from the configuration. These configuration options "
+"can be found in the `[watcher_datasources]` group and are named "
+"`query_max_retries` and `query_timeout`."
+
 msgid ""
 "Allow decision engine to pass strategy parameters, like optimization "
 "threshold, to selected strategy, also strategy to provide parameters info to "
@@ -293,6 +454,34 @@ msgstr ""
 "threshold, to selected strategy, also strategy to provide parameters info to "
 "end user."
 
+msgid ""
+"Allow using file to override metric map. Override the metric map of each "
+"datasource as soon as it is created by the manager. This override comes from "
+"a file whose path is provided by a setting in config file. The setting is "
+"`watcher_decision_engine/metric_map_path`.  The file contains a map per "
+"datasource whose keys are the metric names as recognized by watcher and the "
+"value is the real name of the metric in the datasource. This setting "
+"defaults to `/etc/watcher/metric_map.yaml`, and presence of this file is "
+"optional."
+msgstr ""
+"Allow using file to override metric map. Override the metric map of each "
+"datasource as soon as it is created by the manager. This override comes from "
+"a file whose path is provided by a setting in config file. The setting is "
+"`watcher_decision_engine/metric_map_path`.  The file contains a map per "
+"datasource whose keys are the metric names as recognized by watcher and the "
+"value is the real name of the metric in the datasource. This setting "
+"defaults to `/etc/watcher/metric_map.yaml`, and presence of this file is "
+"optional."
+
+msgid ""
+"An Watcher API WSGI application script ``watcher-api-wsgi`` is now "
+"available. It is auto-generated by ``pbr`` and allows to run the API service "
+"using WSGI server (for example Nginx and uWSGI)."
+msgstr ""
+"An Watcher API WSGI application script ``watcher-api-wsgi`` is now "
+"available. It is auto-generated by ``pbr`` and allows to run the API service "
+"using WSGI server (for example Nginx and uWSGI)."
+
 msgid ""
 "Audits have 'name' field now, that is more friendly to end users. Audit's "
 "name can't exceed 63 characters."
@@ -300,9 +489,25 @@ msgstr ""
 "Audits have 'name' field now, that is more friendly to end users. Audit's "
 "name can't exceed 63 characters."
 
+msgid ""
+"Baremetal Model gets Audit scoper with an ability to exclude Ironic nodes."
+msgstr ""
+"Baremetal Model gets Audit scope with an ability to exclude Ironic nodes."
+
 msgid "Bug Fixes"
 msgstr "Bug Fixes"
 
+msgid ""
+"Ceilometer Datasource has been deprecated since its API has been deprecated "
+"in Ocata cycle. Watcher has supported Ceilometer for some releases after "
+"Ocata to let users migrate to Gnocchi/Monasca datasources. Since Train "
+"release, Ceilometer support will be removed."
+msgstr ""
+"Ceilometer Datasource has been deprecated since its API has been deprecated "
+"in Ocata cycle. Watcher has supported Ceilometer for some releases after "
+"Ocata to let users migrate to Gnocchi/Monasca datasources. Since Train "
+"release, Ceilometer support will be removed."
+
 msgid "Centralize all configuration options for Watcher."
 msgstr "Centralise all configuration options for Watcher."
 
@@ -360,6 +565,52 @@ msgstr ""
 "Now instances from particular project in OpenStack can be excluded from "
 "audit defining scope in audit templates."
 
+msgid ""
+"For a large cloud infrastructure, retrieving data from Nova may take a long "
+"time. To avoid getting too much data from Nova, building the compute data "
+"model according to the scope of audit."
+msgstr ""
+"For a large cloud infrastructure, retrieving data from Nova may take a long "
+"time. To avoid getting too much data from Nova, building the compute data "
+"model according to the scope of audit."
+
+msgid ""
+"Grafana has been added as datasource that can be used for collecting "
+"metrics. The configuration options allow to specify what metrics and how "
+"they are stored in grafana so that no matter how Grafana is configured it "
+"can still be used. The configuration can be done via the typical "
+"configuration file but it is recommended to configure most options in the "
+"yaml file for metrics. For a complete walkthrough on configuring Grafana "
+"see: https://docs.openstack.org/watcher/latest/datasources/grafana.html"
+msgstr ""
+"Grafana has been added as datasource that can be used for collecting "
+"metrics. The configuration options allow to specify what metrics and how "
+"they are stored in Grafana so that no matter how Grafana is configured it "
+"can still be used. The configuration can be done via the typical "
+"configuration file but it is recommended to configure most options in the "
+"yaml file for metrics. For a complete walkthrough on configuring Grafana "
+"see: https://docs.openstack.org/watcher/latest/datasources/grafana.html"
+
+msgid ""
+"If Gnocchi was configured to have a custom amount of retries and or a custom "
+"timeout then the configuration needs to moved into the "
+"`[watcher_datasources]` group instead of the `[gnocchi_client]` group."
+msgstr ""
+"If Gnocchi was configured to have a custom amount of retries and or a custom "
+"timeout then the configuration needs to moved into the "
+"`[watcher_datasources]` group instead of the `[gnocchi_client]` group."
+
+msgid ""
+"Improved interface for datasource baseclass that better defines expected "
+"values and types for parameters and return types of all abstract methods. "
+"This allows all strategies to work with every datasource provided the "
+"metrics are configured for that given datasource."
+msgstr ""
+"Improved interface for datasource baseclass that better defines expected "
+"values and types for parameters and return types of all abstract methods. "
+"This allows all strategies to work with every datasource provided the "
+"metrics are configured for that given datasource."
+
 msgid ""
 "Instance cold migration logic is now replaced with using Nova migrate "
 "Server(migrate Action) API which has host option since v2.56."
@@ -367,6 +618,17 @@ msgstr ""
 "Instance cold migration logic is now replaced with using Nova migrate "
 "Server(migrate Action) API which has host option since v2.56."
 
+msgid ""
+"Many operations in the decision engine will block on I/O. Such I/O "
+"operations can stall the execution of a sequential application "
+"significantly. To reduce the potential bottleneck of many operations the "
+"general purpose decision engine threadpool is introduced."
+msgstr ""
+"Many operations in the decision engine will block on I/O. Such I/O "
+"operations can stall the execution of a sequential application "
+"significantly. To reduce the potential bottleneck of many operations the "
+"general purpose decision engine threadpool is introduced."
+
 msgid "New Features"
 msgstr "New Features"
 
@@ -389,6 +651,13 @@ msgstr ""
 "Nova API version is now set to 2.56 by default. This needs the migrate "
 "action of migration type cold with destination_node parameter to work."
 
+msgid ""
+"Now Watcher strategy can select specific planner beyond default. Strategy "
+"can set planner property to specify its own planner."
+msgstr ""
+"Now Watcher strategy can select specific planner beyond default. Strategy "
+"can set planner property to specify its own planner."
+
 msgid "Ocata Series Release Notes"
 msgstr "Ocata Series Release Notes"
 
@@ -429,12 +698,77 @@ msgstr ""
 "resources will be called \"Audit scope\" and will be defined in each audit "
 "template (which contains the audit settings)."
 
+msgid ""
+"Python 2.7 support has been dropped. Last release of Watcher to support "
+"py2.7 is OpenStack Train. The minimum version of Python now supported by "
+"Watcher is Python 3.6."
+msgstr ""
+"Python 2.7 support has been dropped. Last release of Watcher to support "
+"py2.7 is OpenStack Train. The minimum version of Python now supported by "
+"Watcher is Python 3.6."
+
 msgid "Queens Series Release Notes"
 msgstr "Queens Series Release Notes"
 
 msgid "Rocky Series Release Notes"
 msgstr "Rocky Series Release Notes"
 
+msgid ""
+"Several strategies have changed the `node` parameter to `compute_node` to be "
+"better aligned with terminology. These strategies include "
+"`basic_consolidation` and `workload_stabilzation`. The `node` parameter will "
+"remain supported during Train release and will be removed in the subsequent "
+"release."
+msgstr ""
+"Several strategies have changed the `node` parameter to `compute_node` to be "
+"better aligned with terminology. These strategies include "
+"`basic_consolidation` and `workload_stabilzation`. The `node` parameter will "
+"remain supported during Train release and will be removed in the subsequent "
+"release."
+
+msgid ""
+"Specific strategies can override this order and use datasources which are "
+"not listed in the global preference."
+msgstr ""
+"Specific strategies can override this order and use datasources which are "
+"not listed in the global preference."
+
+msgid "Stein Series Release Notes"
+msgstr "Stein Series Release Notes"
+
+msgid ""
+"The building of the compute (Nova) data model will be done using the "
+"decision engine threadpool, thereby, significantly reducing the total time "
+"required to build it."
+msgstr ""
+"The building of the compute (Nova) data model will be done using the "
+"decision engine threadpool, thereby, significantly reducing the total time "
+"required to build it."
+
+msgid ""
+"The configuration options for query retries in `[gnocchi_client]` are "
+"deprecated and the option in `[watcher_datasources]` should now be used."
+msgstr ""
+"The configuration options for query retries in `[gnocchi_client]` are "
+"deprecated and the option in `[watcher_datasources]` should now be used."
+
+msgid ""
+"The default value of ``[oslo_policy] policy_file`` config option has been "
+"changed from ``policy.json`` to ``policy.yaml``. Operators who are utilizing "
+"customized or previously generated static policy JSON files (which are not "
+"needed by default), should generate new policy files or convert them in YAML "
+"format. Use the `oslopolicy-convert-json-to-yaml <https://docs.openstack.org/"
+"oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ tool to "
+"convert a JSON to YAML formatted policy file in backward compatible way."
+msgstr ""
+"The default value of ``[oslo_policy] policy_file`` config option has been "
+"changed from ``policy.json`` to ``policy.yaml``. Operators who are utilizing "
+"customized or previously generated static policy JSON files (which are not "
+"needed by default), should generate new policy files or convert them in YAML "
+"format. Use the `oslopolicy-convert-json-to-yaml <https://docs.openstack.org/"
+"oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ tool to "
+"convert a JSON to YAML formatted policy file in backward compatible way."
+
 msgid ""
 "The graph model describes how VMs are associated to compute hosts. This "
 "allows for seeing relationships upfront between the entities and hence can "
@@ -455,6 +789,22 @@ msgstr ""
 "was fixed. Before fixing, it booted an instance in the service project as a "
 "migrated instance."
 
+msgid ""
+"The minimum required version of the ``[nova_client]/api_version`` value is "
+"now enforced to be ``2.56`` which is available since the Queens version of "
+"the nova compute service."
+msgstr ""
+"The minimum required version of the ``[nova_client]/api_version`` value is "
+"now enforced to be ``2.56`` which is available since the Queens version of "
+"the Nova compute service."
+
+msgid ""
+"The new strategy baseclass has significant changes in method parameters and "
+"any out-of-tree strategies will have to be adopted."
+msgstr ""
+"The new strategy baseclass has significant changes in method parameters and "
+"any out-of-tree strategies will have to be adopted."
+
 msgid ""
 "There is new ability to create Watcher continuous audits with cron interval. "
 "It means you may use, for example, optional argument '--interval \"\\*/5 \\* "
@@ -468,9 +818,45 @@ msgstr ""
 "best effort basis and therefore, we recommend you to use a minimal cron "
 "interval of at least one minute."
 
+msgid "Train Series Release Notes"
+msgstr "Train Series Release Notes"
+
 msgid "Upgrade Notes"
 msgstr "Upgrade Notes"
 
+msgid ""
+"Use of JSON policy files was deprecated by the ``oslo.policy`` library "
+"during the Victoria development cycle. As a result, this deprecation is "
+"being noted in the Wallaby cycle with an anticipated future removal of "
+"support by ``oslo.policy``. As such operators will need to convert to YAML "
+"policy files. Please see the upgrade notes for details on migration of any "
+"custom policy files."
+msgstr ""
+"Use of JSON policy files was deprecated by the ``oslo.policy`` library "
+"during the Victoria development cycle. As a result, this deprecation is "
+"being noted in the Wallaby cycle with an anticipated future removal of "
+"support by ``oslo.policy``. As such operators will need to convert to YAML "
+"policy files. Please see the upgrade notes for details on migration of any "
+"custom policy files."
+
+msgid ""
+"Using ``watcher/api/app.wsgi`` script is deprecated and it will be removed "
+"in U release. Please switch to automatically generated ``watcher-api-wsgi`` "
+"script instead."
+msgstr ""
+"Using ``watcher/api/app.wsgi`` script is deprecated and it will be removed "
+"in U release. Please switch to automatically generated ``watcher-api-wsgi`` "
+"script instead."
+
+msgid "Ussuri Series Release Notes"
+msgstr "Ussuri Series Release Notes"
+
+msgid "Victoria Series Release Notes"
+msgstr "Victoria Series Release Notes"
+
+msgid "Wallaby Series Release Notes"
+msgstr "Wallaby Series Release Notes"
+
 msgid ""
 "Watcher can continuously optimize the OpenStack cloud for a specific "
 "strategy or goal by triggering an audit periodically which generates an "
@@ -480,6 +866,15 @@ msgstr ""
 "strategy or goal by triggering an audit periodically which generates an "
 "action plan and run it automatically."
 
+msgid ""
+"Watcher can get resource information such as total, allocation ratio and "
+"reserved information from Placement API. Now we add some new fields to the "
+"Watcher Data Model:"
+msgstr ""
+"Watcher can get resource information such as total, allocation ratio and "
+"reserved information from Placement API. Now we add some new fields to the "
+"Watcher Data Model:"
+
 msgid ""
 "Watcher can now run specific actions in parallel improving the performances "
 "dramatically when executing an action plan."
@@ -517,6 +912,15 @@ msgstr ""
 "includes all instances. It filters excluded instances when migration during "
 "the audit."
 
+msgid ""
+"Watcher now supports configuring which datasource to use and in which order. "
+"This configuration is done by specifying datasources in the "
+"watcher_datasources section:"
+msgstr ""
+"Watcher now supports configuring which datasource to use and in which order. "
+"This configuration is done by specifying datasources in the "
+"watcher_datasources section:"
+
 msgid ""
 "Watcher removes the support to Nova legacy notifications because of Nova "
 "will deprecate them."
@@ -557,9 +961,24 @@ msgstr ""
 "Watcher supports multiple metrics backend and relies on Ceilometer and "
 "Monasca."
 
+msgid "We also add some new propeties:"
+msgstr "We also add some new properties:"
+
 msgid "Welcome to watcher's Release Notes documentation!"
 msgstr "Welcome to watcher's Release Notes documentation!"
 
+msgid "Xena Series Release Notes"
+msgstr "Xena Series Release Notes"
+
+msgid "Yoga Series Release Notes"
+msgstr "Yoga Series Release Notes"
+
+msgid "Zed Series Release Notes"
+msgstr "Zed Series Release Notes"
+
+msgid "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
+msgstr "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
+
 msgid ""
 "all Watcher objects have been refactored to support OVO (oslo."
 "versionedobjects) which was a prerequisite step in order to implement "
@@ -569,6 +988,21 @@ msgstr ""
 "versionedobjects) which was a prerequisite step in order to implement "
 "versioned notifications."
 
+msgid ""
+"disk_gb_capacity: The amount of disk, take allocation ratio into account, "
+"but do not include reserved."
+msgstr ""
+"disk_gb_capacity: The amount of disk, take allocation ratio into account, "
+"but do not include reserved."
+
+msgid ""
+"disk_gb_reserved: The amount of disk a node has reserved for its own use."
+msgstr ""
+"disk_gb_reserved: The amount of disk a node has reserved for its own use."
+
+msgid "disk_ratio: Disk allocation ratio."
+msgstr "disk_ratio: Disk allocation ratio."
+
 msgid "instance.create.end"
 msgstr "instance.create.end"
 
@@ -635,6 +1069,21 @@ msgstr "instance.unshelve.end"
 msgid "instance.update"
 msgstr "instance.update"
 
+msgid ""
+"memory_mb_capacity: The amount of memory, take allocation ratio into "
+"account, but do not include reserved."
+msgstr ""
+"memory_mb_capacity: The amount of memory, take allocation ratio into "
+"account, but do not include reserved."
+
+msgid ""
+"memory_mb_reserved: The amount of memory a node has reserved for its own use."
+msgstr ""
+"memory_mb_reserved: The amount of memory a node has reserved for its own use."
+
+msgid "memory_ratio: Memory allocation ratio."
+msgstr "memory_ratio: Memory allocation ratio."
+
 msgid "new:"
 msgstr "new:"
 
@@ -649,3 +1098,16 @@ msgstr "service.delete"
 
 msgid "service.update"
 msgstr "service.update"
+
+msgid ""
+"vcpu_capacity: The amount of vcpu, take allocation ratio into account, but "
+"do not include reserved."
+msgstr ""
+"vcpu_capacity: The amount of vcpu, take allocation ratio into account, but "
+"do not include reserved."
+
+msgid "vcpu_ratio: CPU allocation ratio."
+msgstr "vcpu_ratio: CPU allocation ratio."
+
+msgid "vcpu_reserved: The amount of cpu a node has reserved for its own use."
+msgstr "vcpu_reserved: The amount of CPU a node has reserved for its own use."

releasenotes/source/locale/fr/LC_MESSAGES/releasenotes.po

@@ -0,0 +1,33 @@
+# Gérald LONLAS <g.lonlas@gmail.com>, 2016. #zanata
+msgid ""
+msgstr ""
+"Project-Id-Version: python-watcher\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2019-03-22 02:21+0000\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"PO-Revision-Date: 2016-10-22 06:44+0000\n"
+"Last-Translator: Gérald LONLAS <g.lonlas@gmail.com>\n"
+"Language-Team: French\n"
+"Language: fr\n"
+"X-Generator: Zanata 4.3.3\n"
+"Plural-Forms: nplurals=2; plural=(n > 1)\n"
+
+msgid "0.29.0"
+msgstr "0.29.0"
+
+msgid "Contents:"
+msgstr "Contenu :"
+
+msgid "Current Series Release Notes"
+msgstr "Note de la release actuelle"
+
+msgid "New Features"
+msgstr "Nouvelles fonctionnalités"
+
+msgid "Newton Series Release Notes"
+msgstr "Note de release pour Newton"
+
+msgid "Welcome to watcher's Release Notes documentation!"
+msgstr "Bienvenue dans la documentation de la note de Release de Watcher"

releasenotes/source/stein.rst

@@ -0,0 +1,6 @@
+===================================
+ Stein Series Release Notes
+===================================
+
+.. release-notes::
+   :branch: stable/stein

releasenotes/source/train.rst

@@ -0,0 +1,6 @@
+==========================
+Train Series Release Notes
+==========================
+
+.. release-notes::
+   :branch: stable/train

releasenotes/source/ussuri.rst

@@ -0,0 +1,6 @@
+===========================
+Ussuri Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/ussuri

releasenotes/source/victoria.rst

@@ -0,0 +1,6 @@
+=============================
+Victoria Series Release Notes
+=============================
+
+.. release-notes::
+   :branch: stable/victoria

releasenotes/source/wallaby.rst

@@ -0,0 +1,6 @@
+============================
+Wallaby Series Release Notes
+============================
+
+.. release-notes::
+   :branch: stable/wallaby

releasenotes/source/xena.rst

@@ -0,0 +1,6 @@
+=========================
+Xena Series Release Notes
+=========================
+
+.. release-notes::
+   :branch: stable/xena

releasenotes/source/yoga.rst

@@ -0,0 +1,6 @@
+=========================
+Yoga Series Release Notes
+=========================
+
+.. release-notes::
+   :branch: stable/yoga

releasenotes/source/zed.rst

@@ -0,0 +1,6 @@
+========================
+Zed Series Release Notes
+========================
+
+.. release-notes::
+   :branch: stable/zed

requirements.txt

@@ -1,49 +1,48 @@
-# The order of packages is significant, because pip processes them in the order
+#  The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
 apscheduler>=3.5.1 # MIT License
-enum34>=1.1.6;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
 jsonpatch>=1.21 # BSD
 keystoneauth1>=3.4.0 # Apache-2.0
-jsonschema<3.0.0,>=2.6.0 # MIT
+jsonschema>=3.2.0 # MIT
 keystonemiddleware>=4.21.0 # Apache-2.0
-lxml>=4.1.1 # BSD
+lxml>=4.5.1 # BSD
 croniter>=0.3.20 # MIT License
+os-resource-classes>=0.4.0
 oslo.concurrency>=3.26.0 # Apache-2.0
 oslo.cache>=1.29.0 # Apache-2.0
-oslo.config>=5.2.0 # Apache-2.0
+oslo.config>=6.8.0 # Apache-2.0
-oslo.context>=2.20.0 # Apache-2.0
+oslo.context>=2.21.0 # Apache-2.0
-oslo.db>=4.35.0 # Apache-2.0
+oslo.db>=4.44.0 # Apache-2.0
 oslo.i18n>=3.20.0 # Apache-2.0
 oslo.log>=3.37.0 # Apache-2.0
-oslo.messaging>=5.36.0 # Apache-2.0
+oslo.messaging>=14.1.0 # Apache-2.0
-oslo.policy>=1.34.0 # Apache-2.0
+oslo.policy>=3.6.0 # Apache-2.0
 oslo.reports>=1.27.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
 oslo.service>=1.30.0 # Apache-2.0
-oslo.upgradecheck>=0.1.0 # Apache-2.0
+oslo.upgradecheck>=1.3.0 # Apache-2.0
 oslo.utils>=3.36.0 # Apache-2.0
 oslo.versionedobjects>=1.32.0 # Apache-2.0
 PasteDeploy>=1.5.2 # MIT
 pbr>=3.1.1 # Apache-2.0
-pecan>=1.2.1 # BSD
+pecan>=1.3.2 # BSD
-PrettyTable<0.8,>=0.7.2 # BSD
+PrettyTable>=0.7.2 # BSD
 gnocchiclient>=7.0.1 # Apache-2.0
-python-ceilometerclient>=2.9.0 # Apache-2.0
 python-cinderclient>=3.5.0 # Apache-2.0
 python-glanceclient>=2.9.1 # Apache-2.0
 python-keystoneclient>=3.15.0 # Apache-2.0
 python-monascaclient>=1.12.0 # Apache-2.0
 python-neutronclient>=6.7.0 # Apache-2.0
-python-novaclient>=10.1.0 # Apache-2.0
+python-novaclient>=14.1.0 # Apache-2.0
 python-openstackclient>=3.14.0 # Apache-2.0
-python-ironicclient>=2.3.0 # Apache-2.0
+python-ironicclient>=2.5.0 # Apache-2.0
-six>=1.11.0 # MIT
 SQLAlchemy>=1.2.5 # MIT
 stevedore>=1.28.0 # Apache-2.0
-taskflow>=3.1.0 # Apache-2.0
+taskflow>=3.8.0 # Apache-2.0
-WebOb>=1.7.4 # MIT
+WebOb>=1.8.5 # MIT
 WSME>=0.9.2 # MIT
-networkx>=1.11 # BSD
+networkx>=2.4 # BSD
 microversion_parse>=0.2.1 # Apache-2.0
+futurist>=1.8.0 # Apache-2.0

setup.cfg

@@ -1,11 +1,12 @@
 [metadata]
 name = python-watcher
 summary = OpenStack Watcher provides a flexible and scalable resource optimization service for multi-tenant OpenStack-based clouds.
-description-file =
+description_file =
     README.rst
 author = OpenStack
-author-email = openstack-discuss@lists.openstack.org
+author_email = openstack-discuss@lists.openstack.org
-home-page = https://docs.openstack.org/watcher/latest/
+home_page = https://docs.openstack.org/watcher/latest/
+python_requires = >=3.8
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -13,10 +14,13 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
+    Programming Language :: Python :: Implementation :: CPython
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [files]
 packages =
@@ -24,10 +28,6 @@ packages =
 data_files =
     etc/ = etc/*
 
-[global]
-setup-hooks =
-    pbr.hooks.setup_hook
-
 [entry_points]
 oslo.config.opts =
     watcher = watcher.conf.opts:list_opts
@@ -46,6 +46,9 @@ console_scripts =
     watcher-sync = watcher.cmd.sync:main
     watcher-status = watcher.cmd.status:main
 
+wsgi_scripts =
+    watcher-api-wsgi = watcher.api.wsgi:initialize_wsgi_app
+
 watcher.database.migration_backend =
     sqlalchemy = watcher.db.sqlalchemy.migration
 
@@ -83,6 +86,7 @@ watcher_strategies =
     storage_capacity_balance = watcher.decision_engine.strategy.strategies.storage_capacity_balance:StorageCapacityBalance
     zone_migration = watcher.decision_engine.strategy.strategies.zone_migration:ZoneMigration
     host_maintenance = watcher.decision_engine.strategy.strategies.host_maintenance:HostMaintenance
+    node_resource_consolidation = watcher.decision_engine.strategy.strategies.node_resource_consolidation:NodeResourceConsolidation
 
 watcher_actions =
     migrate = watcher.applier.actions.migration:Migrate
@@ -99,23 +103,9 @@ watcher_workflow_engines =
 watcher_planners =
     weight = watcher.decision_engine.planner.weight:WeightPlanner
     workload_stabilization  = watcher.decision_engine.planner.workload_stabilization:WorkloadStabilizationPlanner
+    node_resource_consolidation  = watcher.decision_engine.planner.node_resource_consolidation:NodeResourceConsolidationPlanner
 
 watcher_cluster_data_model_collectors =
     compute = watcher.decision_engine.model.collector.nova:NovaClusterDataModelCollector
     storage = watcher.decision_engine.model.collector.cinder:CinderClusterDataModelCollector
     baremetal = watcher.decision_engine.model.collector.ironic:BaremetalClusterDataModelCollector
-
-
-[compile_catalog]
-directory = watcher/locale
-domain = watcher
-
-[update_catalog]
-domain = watcher
-output_dir = watcher/locale
-input_file = watcher/locale/watcher.pot
-
-[extract_messages]
-keywords = _ gettext ngettext l_ lazy_gettext _LI _LW _LE _LC
-mapping_file = babel.cfg
-output_file = watcher/locale/watcher.pot

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
     pbr=True)

test-requirements.txt

@@ -5,12 +5,11 @@
 coverage>=4.5.1 # Apache-2.0
 doc8>=0.8.0 # Apache-2.0
 freezegun>=0.3.10 # Apache-2.0
-hacking>=1.1.0,<1.2.0 # Apache-2.0
+hacking>=3.0.1,<3.1.0 # Apache-2.0
-mock>=2.0.0 # BSD
 oslotest>=3.3.0 # Apache-2.0
-os-testr>=1.0.0 # Apache-2.0
 testscenarios>=0.5.0 # Apache-2.0/BSD
 testtools>=2.3.0 # MIT
 stestr>=2.0.0 # Apache-2.0
 os-api-ref>=1.4.0 # Apache-2.0
-bandit>=1.1.0 # Apache-2.0
+bandit>=1.6.0 # Apache-2.0
+WebTest>=2.0.27 # MIT

tox.ini

@@ -1,36 +1,47 @@
 [tox]
-minversion = 2.0
+minversion = 3.18.0
-envlist = py35,py27,pep8
+envlist = py3,pep8
-skipsdist = True
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 usedevelop = True
-whitelist_externals = find
+allowlist_externals = find
                       rm
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
+install_command = pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2024.1} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/test-requirements.txt
+deps =
+    -r{toxinidir}/test-requirements.txt
+    -r{toxinidir}/requirements.txt
+    python-libmaas>=0.6.8
 commands =
   rm -f .testrepository/times.dbm
   find . -type f -name "*.py[c|o]" -delete
   stestr run {posargs}
-passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+passenv =
+  http_proxy
+  HTTP_PROXY
+  https_proxy
+  HTTPS_PROXY
+  no_proxy
+  NO_PROXY
 
 [testenv:pep8]
-basepython = python3
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    bandit -r watcher -x tests -n5 -ll -s B320
+    #bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:venv]
-basepython = python3
 setenv = PYTHONHASHSEED=0
+deps =
+    -r{toxinidir}/doc/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+    -r{toxinidir}/requirements.txt
 commands = {posargs}
 
 [testenv:cover]
-basepython = python3
 setenv =
     PYTHON=coverage run --source watcher --parallel-mode
 commands =
@@ -41,68 +52,87 @@ commands =
     coverage report
 
 [testenv:docs]
-basepython = python3
 setenv = PYTHONHASHSEED=0
-deps = -r{toxinidir}/doc/requirements.txt
+deps =
-commands = sphinx-build -W -b html doc/source doc/build/html
+  -r{toxinidir}/doc/requirements.txt
+commands =
+  rm -fr doc/build doc/source/api/ .autogenerated
+  sphinx-build -W --keep-going -b html doc/source doc/build/html
 
 [testenv:api-ref]
-basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals = bash
+allowlist_externals = bash
 commands =
   bash -c 'rm -rf api-ref/build'
-  sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
+  sphinx-build -W --keep-going -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
 
 [testenv:debug]
-basepython = python3
 commands = oslo_debug_helper -t watcher/tests {posargs}
 
 [testenv:genconfig]
-basepython = python3
 sitepackages = False
 commands =
     oslo-config-generator --config-file etc/watcher/oslo-config-generator/watcher.conf
 
 [testenv:genpolicy]
-basepython = python3
 commands =
     oslopolicy-sample-generator --config-file etc/watcher/oslo-policy-generator/watcher-policy-generator.conf
 
+[testenv:wheel]
+commands = python setup.py bdist_wheel
+
+[testenv:pdf-docs]
+envdir = {toxworkdir}/docs
+deps = {[testenv:docs]deps}
+allowlist_externals =
+  rm
+  make
+commands =
+  rm -rf doc/build/pdf
+  sphinx-build -W --keep-going -b latex doc/source doc/build/pdf
+  make -C doc/build/pdf
+
+[testenv:releasenotes]
+deps = -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
+
+[testenv:bandit]
+deps = -r{toxinidir}/test-requirements.txt
+commands = bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
+
 [flake8]
 filename = *.py,app.wsgi
 show-source=True
-ignore= H105,E123,E226,N320,H202
+# W504 line break after binary operator
+ignore= H105,E123,E226,N320,H202,W504
 builtins= _
 enable-extensions = H106,H203,H904
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*,demo/,releasenotes
 
-[testenv:wheel]
-basepython = python3
-commands = python setup.py bdist_wheel
-
 [hacking]
 import_exceptions = watcher._i18n
-local-check-factory = watcher.hacking.checks.factory
+
+[flake8:local-plugins]
+extension =
+   N319 = checks:no_translate_debug_logs
+   N321 = checks:use_jsonutils
+   N322 = checks:check_assert_called_once_with
+   N325 = checks:check_python3_xrange
+   N326 = checks:check_no_basestring
+   N327 = checks:check_python3_no_iteritems
+   N328 = checks:check_asserttrue
+   N329 = checks:check_assertfalse
+   N330 = checks:check_assertempty
+   N331 = checks:check_assertisinstance
+   N332 = checks:check_assertequal_for_httpcode
+   N333 = checks:check_log_warn_deprecated
+   N340 = checks:check_oslo_i18n_wrapper
+   N341 = checks:check_builtins_gettext
+   N342 = checks:no_redundant_import_alias
+   N366 = checks:import_stock_mock
+paths = ./watcher/hacking
 
 [doc8]
 extension=.rst
 # todo: stop ignoring doc/source/man when https://bugs.launchpad.net/doc8/+bug/1502391 is fixed
 ignore-path=doc/source/image_src,doc/source/man,doc/source/api
-
-[testenv:releasenotes]
-basepython = python3
-deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
-
-[testenv:bandit]
-basepython = python3
-deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r watcher -x tests -n5 -ll -s B320
-
-[testenv:lower-constraints]
-basepython = python3
-deps =
-  -c{toxinidir}/lower-constraints.txt
-  -r{toxinidir}/test-requirements.txt
-  -r{toxinidir}/requirements.txt