Update TOX_CONSTRAINTS_FILE for stable/2025.2

Update the URL to the upper-constraints file to point to the redirect
rule on releases.openstack.org so that anyone working on this branch
will switch to the correct upper-constraints list automatically when
the requirements repository branches.

Until the requirements repository has as stable/2025.2 branch, tests will
continue to use the upper-constraints list on master.

Change-Id: I2d56a75436026bfc745714473bba7a9a4075ef2e
Signed-off-by: OpenStack Release Bot <infra-root@openstack.org>
Generated-By: openstack/project-config:roles/copy-release-tools-scripts/files/release-tools/functions

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
+defaultbranch=stable/2025.2

.pre-commit-config.yaml

@@ -35,17 +35,17 @@ repos:
         additional_dependencies: []
         exclude: '^(doc|releasenotes|tools)/.*$'
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.6
+    rev: 1.8.3
     hooks:
       - id: bandit
         args: ['-x', 'tests', '-s', 'B101,B311,B320']
   - repo: https://github.com/hhatto/autopep8
-    rev: v2.3.1
+    rev: v2.3.2
     hooks:
       - id: autopep8
         files: '^.*\.py$'
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         args: ['--ignore-words=doc/dictionary.txt']
@@ -54,7 +54,7 @@ repos:
     hooks:
       - id: sphinx-lint
         args: [--enable=default-role]
-        files: ^doc/|releasenotes|api-guide
+        files: ^doc/|^releasenotes/|^api-guide/
         types: [rst]
   - repo: https://github.com/PyCQA/doc8
     rev: v1.1.2

.zuul.yaml

@@ -1,27 +1,3 @@
-- project:
-    queue: watcher
-    templates:
-      - check-requirements
-      - openstack-cover-jobs
-      - openstack-python3-jobs
-      - publish-openstack-docs-pti
-      - release-notes-jobs-python3
-    check:
-      jobs:
-        - watcher-tempest-functional
-        - watcher-tempest-functional-jammy
-        - watcher-grenade
-        - watcher-tempest-strategies
-        - watcher-tempest-actuator
-        - watcherclient-tempest-functional
-        - watcher-tempest-functional-ipv6-only
-        - watcher-prometheus-integration
-    gate:
-      jobs:
-        - watcher-tempest-functional
-        - watcher-tempest-functional-jammy
-        - watcher-tempest-functional-ipv6-only
-
 - job:
     name: watcher-tempest-actuator
     parent: watcher-tempest-multinode
@@ -33,7 +9,11 @@
     parent: watcher-tempest-multinode
     vars:
       tempest_concurrency: 1
-      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_strategies
+      # All tests inside watcher_tempest_plugin.tests.scenario with tag "strategy"
+      # or test_execute_strategies file
+      # excluding tests with tag "real_load"
+      tempest_test_regex: (^watcher_tempest_plugin.tests.scenario)(.*\[.*\bstrategy\b.*\].*)|(^watcher_tempest_plugin.tests.scenario.test_execute_strategies)
+      tempest_exclude_regex: .*\[.*\breal_load\b.*\].*
 
 - job:
     name: watcher-tempest-multinode
@@ -52,6 +32,11 @@
                 period: 120
               watcher_cluster_data_model_collectors.storage:
                 period: 120
+            $CINDER_CONF:
+              # enable notifications in compute node, by default they are only
+              # configured in the controller
+              oslo_messaging_notifications:
+                driver: messagingv2
         devstack_services:
           watcher-api: false
           watcher-decision-engine: true
@@ -67,6 +52,13 @@
           rabbit: false
           mysql: false
     vars:
+      devstack_localrc:
+        GNOCCHI_ARCHIVE_POLICY_TEMPEST: "ceilometer-low-rate"
+        CEILOMETER_PIPELINE_INTERVAL: 15
+      devstack_services:
+        ceilometer-acompute: false
+        ceilometer-acentral: true
+        ceilometer-anotification: true
       devstack_local_conf:
         post-config:
           $WATCHER_CONF:
@@ -76,6 +68,11 @@
               period: 120
             watcher_cluster_data_model_collectors.storage:
               period: 120
+          $CINDER_CONF:
+            # enable notifications in compute node, by default they are only
+            # configured in the controller
+            oslo_messaging_notifications:
+              driver: messagingv2
         test-config:
           $TEMPEST_CONFIG:
             compute:
@@ -86,6 +83,8 @@
               block_migration_for_live_migration: true
             placement:
               min_microversion: 1.29
+            telemetry:
+              ceilometer_polling_interval: 15
       devstack_plugins:
         ceilometer: https://opendev.org/openstack/ceilometer
 
@@ -119,17 +118,6 @@
       zuul_copy_output:
         /etc/hosts: logs
 
-# TODO(gmann): As per the 2025.1 testing runtime, we need to run at least
-# one job on jammy. This job can be removed in the next cycle(2025.2)
-- job:
-    name: watcher-tempest-functional-jammy
-    description: This is integrated job testing on Ubuntu jammy(22.04)
-    parent: watcher-tempest-functional
-    nodeset: openstack-single-node-jammy
-    vars:
-      <<: *base_vars
-      python_version: '3.9'
-
 - job:
     name: watcher-tempest-functional-ipv6-only
     parent: devstack-tempest-ipv6
@@ -158,15 +146,6 @@
       - ^tools/.*$
       - ^tox.ini$
 
-- job:
-    # This job is used in python-watcherclient repo
-    name: watcherclient-tempest-functional
-    parent: watcher-tempest-functional
-    timeout: 4200
-    vars:
-      tempest_concurrency: 1
-      tempest_test_regex: watcher_tempest_plugin.tests.client_functional
-
 - job:
     name: watcher-sg-core-tempest-base
     parent: devstack-tempest
@@ -196,6 +175,7 @@
         watcher: https://opendev.org/openstack/watcher
         devstack-plugin-prometheus: https://opendev.org/openstack/devstack-plugin-prometheus
       devstack_services:
+        ceilometer-acompute: true
         watcher-api: true
         watcher-decision-engine: true
         watcher-applier: true
@@ -213,9 +193,6 @@
         CEILOMETER_BACKENDS: "sg-core"
         CEILOMETER_PIPELINE_INTERVAL: 15
         CEILOMETER_ALARM_THRESHOLD: 6000000000
-        NODE_EXPORTER_ENABLE: false
-        PROMETHEUS_ENABLE: false
-        PROMETHEUS_SERVICE_SCRAPE_TARGETS: "sg-core,node-exporter"
         PROMETHEUS_CONFIG_FILE: "/home/zuul/prometheus.yml"
       devstack_local_conf:
         post-config:
@@ -231,6 +208,10 @@
               period: 120
             watcher_cluster_data_model_collectors.storage:
               period: 120
+            compute_model:
+              enable_extended_attributes: true
+            nova_client:
+              api_version: "2.96"
         test-config:
           $TEMPEST_CONFIG:
             compute:
@@ -250,9 +231,15 @@
               ceilometer_polling_interval: 15
             optimize:
               datasource: prometheus
+              extended_attributes_nova_microversion: "2.96"
+              data_model_collectors_period: 120
       tempest_plugins:
         - watcher-tempest-plugin
-      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_strategies
+      # All tests inside watcher_tempest_plugin.tests.scenario with tag "strategy"
+      # and test_execute_strategies, test_data_model files
+      # excluding tests with tag "real_load"
+      tempest_test_regex: (watcher_tempest_plugin.tests.scenario)(.*\[.*\bstrategy\b.*\].*)|(watcher_tempest_plugin.tests.scenario.(test_execute_strategies|test_data_model))
+      tempest_exclude_regex: .*\[.*\breal_load\b.*\].*
       tempest_concurrency: 1
       tox_envlist: all
       zuul_copy_output:
@@ -261,7 +248,6 @@
       subnode:
         devstack_plugins:
           ceilometer: https://opendev.org/openstack/ceilometer
-          sg-core: https://github.com/openstack-k8s-operators/sg-core
           devstack-plugin-prometheus: https://opendev.org/openstack/devstack-plugin-prometheus
         devstack_services:
           ceilometer-acompute: true
@@ -271,9 +257,6 @@
         devstack_localrc:
           CEILOMETER_BACKEND: "none"
           CEILOMETER_BACKENDS: "none"
-          # sg_core related var
-          NODE_EXPORTER_ENABLE: false
-          PROMETHEUS_ENABLE: false
         devstack_local_conf:
           post-config:
             $WATCHER_CONF:
@@ -287,3 +270,133 @@
 - job:
     name: watcher-prometheus-integration
     parent: watcher-sg-core-tempest-base
+    vars:
+      devstack_services:
+        ceilometer-acompute: false
+        node_exporter: false
+    group-vars:
+      subnode:
+        devstack_services:
+          ceilometer-acompute: false
+          node_exporter: false
+
+- job:
+    name: watcher-aetos-integration
+    parent: watcher-sg-core-tempest-base
+    description: |
+      This job tests Watcher with Aetos reverse-proxy for Prometheus
+      using Keystone authentication instead of direct Prometheus access.
+    required-projects:
+      - openstack/python-observabilityclient
+      - openstack/aetos
+    vars: &aetos_vars
+      devstack_services:
+        ceilometer-acompute: false
+        node_exporter: false
+      devstack_plugins:
+        ceilometer: https://opendev.org/openstack/ceilometer
+        sg-core: https://github.com/openstack-k8s-operators/sg-core
+        watcher: https://opendev.org/openstack/watcher
+        devstack-plugin-prometheus: https://opendev.org/openstack/devstack-plugin-prometheus
+        aetos: https://opendev.org/openstack/aetos
+      devstack_local_conf:
+        post-config:
+          $WATCHER_CONF:
+            watcher_datasources:
+              datasources: aetos
+            aetos_client:
+              interface: public
+              region_name: RegionOne
+              fqdn_label: fqdn
+              instance_uuid_label: resource
+        test-config:
+          $TEMPEST_CONFIG:
+            optimize:
+              datasource: prometheus
+    group-vars:
+      subnode:
+        devstack_services:
+          ceilometer-acompute: false
+          node_exporter: false
+
+- job:
+    name: watcher-prometheus-integration-realdata
+    parent: watcher-sg-core-tempest-base
+    vars: &realdata_vars
+      devstack_services:
+        ceilometer-acompute: true
+        node_exporter: true
+      devstack_localrc:
+        NODE_EXPORTER_COLLECTOR_EXCLUDE: ""
+      devstack_local_conf:
+        test-config:
+          $TEMPEST_CONFIG:
+            optimize:
+              datasource: ""
+              real_workload_period: 300
+      # All tests inside watcher_tempest_plugin.tests.scenario with tag "real_load"
+      tempest_test_regex: (^watcher_tempest_plugin.tests.scenario)(.*\[.*\breal_load\b.*\].*)
+      tempest_exclude_regex: ""
+    group-vars: &realdata_group_vars
+      subnode:
+        devstack_services:
+          ceilometer-acompute: true
+          node_exporter: true
+        devstack_localrc:
+          NODE_EXPORTER_COLLECTOR_EXCLUDE: ""
+
+- job:
+    name: watcher-prometheus-integration-threading
+    parent: watcher-prometheus-integration
+    vars:
+      devstack_localrc:
+        'SYSTEMD_ENV_VARS["watcher-decision-engine"]': OS_WATCHER_DISABLE_EVENTLET_PATCHING=true
+
+- job:
+    name: openstack-tox-py312-threading
+    parent: openstack-tox-py312
+    description: |
+      Run tox with the py3-threading environment.
+    vars:
+      tox_envlist: py3-threading
+
+- job:
+    name: watcher-aetos-integration-realdata
+    parent: watcher-aetos-integration
+    vars: *realdata_vars
+    group-vars: *realdata_group_vars
+
+- project:
+    queue: watcher
+    templates:
+      - check-requirements
+      - openstack-cover-jobs
+      - openstack-python3-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3
+    check:
+      jobs:
+        - openstack-tox-py312-threading
+        - watcher-tempest-functional
+        - watcher-grenade
+        - watcher-tempest-strategies
+        - watcher-tempest-actuator
+        - python-watcherclient-functional:
+            files:
+              - ^watcher/api/*
+        - watcher-tempest-functional-ipv6-only
+        - watcher-prometheus-integration
+        - watcher-prometheus-integration-threading
+        - watcher-aetos-integration
+    gate:
+      jobs:
+        - watcher-tempest-functional
+        - watcher-tempest-functional-ipv6-only
+    experimental:
+      jobs:
+        - watcher-prometheus-integration-realdata
+        - watcher-aetos-integration-realdata
+    periodic-weekly:
+      jobs:
+        - watcher-prometheus-integration-realdata
+        - watcher-aetos-integration-realdata

api-ref/source/parameters.yaml

@@ -189,6 +189,16 @@ action_state:
   in: body
   required: true
   type: string
+action_status_message:
+  description: |
+    Message with additional information about the Action state.
+    This field can be set when transitioning an action to SKIPPED state,
+    or updated for actions that are already in SKIPPED state to provide
+    more detailed explanations, fix typos, or expand on initial reasons.
+  in: body
+  required: false
+  type: string
+  min_version: 1.5
 action_type:
   description: |
     Action type based on specific API action. Actions in Watcher are
@@ -230,6 +240,13 @@ actionplan_state:
   in: body
   required: false
   type: string
+actionplan_status_message:
+  description: |
+    Message with additional information about the Action Plan state.
+  in: body
+  required: false
+  type: string
+  min_version: 1.5
 
 # Audit
 audit_autotrigger:
@@ -320,6 +337,13 @@ audit_state:
   in: body
   required: true
   type: string
+audit_status_message:
+  description: |
+    Message with additional information about the Audit state.
+  in: body
+  required: false
+  type: string
+  min_version: 1.5
 audit_strategy:
   description: |
     The UUID or name of the Strategy.
@@ -420,12 +444,24 @@ links:
   type: array
 
 # Data Model Node
+node_disabled_reason:
+  description: |
+    The Disabled Reason of the node.
+  in: body
+  required: true
+  type: string
 node_disk:
   description: |
     The Disk of the node(in GiB).
   in: body
   required: true
   type: integer
+node_disk_gb_reserved:
+  description: |
+    The Disk Reserved of the node (in GiB).
+  in: body
+  required: true
+  type: integer
 node_disk_ratio:
   description: |
     The Disk Ratio of the node.
@@ -444,6 +480,12 @@ node_memory:
   in: body
   required: true
   type: integer
+node_memory_mb_reserved:
+  description: |
+    The Memory Reserved of the node(in MiB).
+  in: body
+  required: true
+  type: integer
 node_memory_ratio:
   description: |
     The Memory Ratio of the node.
@@ -456,6 +498,12 @@ node_state:
   in: body
   required: true
   type: string
+node_status:
+  description: |
+    The Status of the node.
+  in: body
+  required: true
+  type: string
 node_uuid:
   description: |
     The Unique UUID of the node.
@@ -468,13 +516,18 @@ node_vcpu_ratio:
   in: body
   required: true
   type: float
+node_vcpu_reserved:
+  description: |
+    The Vcpu Reserved of the node.
+  in: body
+  required: true
+  type: integer
 node_vcpus:
   description: |
     The Vcpu of the node.
   in: body
   required: true
   type: integer
-
 # Scoring Engine
 scoring_engine_description:
   description: |
@@ -502,18 +555,50 @@ server_disk:
   in: body
   required: true
   type: integer
+server_flavor_extra_specs:
+  description: |
+    The flavor extra specs of the server.
+  in: body
+  required: true
+  type: JSON
+  min_version: 1.6
+server_locked:
+  description: |
+    Whether the server is locked.
+  in: body
+  required: true
+  type: boolean
 server_memory:
   description: |
     The Memory of server.
   in: body
   required: true
   type: integer
+server_metadata:
+  description: |
+    The metadata associated with the server.
+  in: body
+  required: true
+  type: JSON
 server_name:
   description: |
     The Name of the server.
   in: body
   required: true
   type: string
+server_pinned_az:
+  description: |
+    The pinned availability zone of the server.
+  in: body
+  required: true
+  type: string
+  min_version: 1.6
+server_project_id:
+  description: |
+    The project ID of the server.
+  in: body
+  required: true
+  type: string
 server_state:
   description: |
     The State of the server.
@@ -532,6 +617,12 @@ server_vcpus:
   in: body
   required: true
   type: integer
+server_watcher_exclude:
+  description: |
+    Whether the server is excluded from the scope.
+  in: body
+  required: true
+  type: boolean
 # Service
 service_host:
   description: |

api-ref/source/samples/action-skip-request-with-message.json

@@ -0,0 +1,12 @@
+[
+    {
+        "op": "replace",
+        "value": "SKIPPED",
+        "path": "/state"
+    },
+    {
+        "op": "replace",
+        "value": "Skipping due to maintenance window",
+        "path": "/status_message"
+    }
+]

api-ref/source/samples/action-skip-request.json

@@ -0,0 +1,7 @@
+[
+    {
+        "op": "replace",
+        "value": "SKIPPED",
+        "path": "/state"
+    }
+]

api-ref/source/samples/action-skip-response.json

@@ -0,0 +1,29 @@
+{
+    "state": "SKIPPED",
+    "description": "Migrate instance to another compute node",
+    "parents": [
+        "b4529294-1de6-4302-b57a-9b5d5dc363c6"
+    ],
+    "links": [
+        {
+            "rel": "self",
+            "href": "http://controller:9322/v1/actions/54acc7a0-91b0-46ea-a5f7-4ae2b9df0b0a"
+        },
+        {
+            "rel": "bookmark",
+            "href": "http://controller:9322/actions/54acc7a0-91b0-46ea-a5f7-4ae2b9df0b0a"
+        }
+    ],
+    "action_plan_uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
+    "uuid": "54acc7a0-91b0-46ea-a5f7-4ae2b9df0b0a",
+    "deleted_at": null,
+    "updated_at": "2018-04-10T12:15:44.026973+00:00",
+    "input_parameters": {
+        "migration_type": "live",
+        "destination_node": "compute-2",
+        "resource_id": "a1b2c3d4-e5f6-7890-1234-567890abcdef"
+    },
+    "action_type": "migrate",
+    "created_at": "2018-04-10T11:59:12.725147+00:00",
+    "status_message": "Action skipped by user. Reason:Skipping due to maintenance window"
+}

api-ref/source/samples/action-update-status-message-request.json

@@ -0,0 +1,7 @@
+[
+    {
+        "op": "replace",
+        "value": "Action skipped due to scheduled maintenance window",
+        "path": "/status_message"
+    }
+]

api-ref/source/samples/action-update-status-message-response.json

@@ -0,0 +1,29 @@
+{
+    "state": "SKIPPED",
+    "description": "Migrate instance to another compute node",
+    "parents": [
+        "b4529294-1de6-4302-b57a-9b5d5dc363c6"
+    ],
+    "links": [
+        {
+            "rel": "self",
+            "href": "http://controller:9322/v1/actions/54acc7a0-91b0-46ea-a5f7-4ae2b9df0b0a"
+        },
+        {
+            "rel": "bookmark",
+            "href": "http://controller:9322/actions/54acc7a0-91b0-46ea-a5f7-4ae2b9df0b0a"
+        }
+    ],
+    "action_plan_uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
+    "uuid": "54acc7a0-91b0-46ea-a5f7-4ae2b9df0b0a",
+    "deleted_at": null,
+    "updated_at": "2018-04-10T12:20:15.123456+00:00",
+    "input_parameters": {
+        "migration_type": "live",
+        "destination_node": "compute-2",
+        "resource_id": "a1b2c3d4-e5f6-7890-1234-567890abcdef"
+    },
+    "action_type": "migrate",
+    "created_at": "2018-04-10T11:59:12.725147+00:00",
+    "status_message": "Action skipped by user. Reason: Action skipped due to scheduled maintenance window"
+}

api-ref/source/samples/actionplan-list-detailed-response.json

@@ -21,7 +21,8 @@
             "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
             "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
             "created_at": "2018-04-10T11:59:52.640067+00:00",
-            "hostname": "controller"
+            "hostname": "controller",
+            "status_message": null
         }
     ]
 }

api-ref/source/samples/actionplan-show-response.json

@@ -17,5 +17,6 @@
     "strategy_name": "dummy_with_resize",
     "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
     "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
-    "hostname": "controller"
-}
+    "hostname": "controller",
+    "status_message": null
+}

api-ref/source/samples/actions-list-detailed-response.json

@@ -24,7 +24,8 @@
                 "duration": 3.2
             },
             "action_type": "sleep",
-            "created_at": "2018-03-26T11:56:08.235226+00:00"
+            "created_at": "2018-03-26T11:56:08.235226+00:00",
+            "status_message": null
         }
     ]
-}
+}

api-ref/source/samples/actions-show-response.json

@@ -22,5 +22,6 @@
         "message": "Welcome"
     },
     "action_type": "nop",
-    "created_at": "2018-04-10T11:59:12.725147+00:00"
-}
+    "created_at": "2018-04-10T11:59:12.725147+00:00",
+    "status_message": null
+}

api-ref/source/samples/audit-create-response.json

@@ -51,5 +51,6 @@
     "updated_at": null,
     "hostname": null,
     "start_time": null,
-    "end_time": null
+    "end_time": null,
+    "status_message": null
 }

api-ref/source/samples/audit-list-detailed-response.json

@@ -53,7 +53,8 @@
             "updated_at": "2018-04-06T09:44:01.604146+00:00",
             "hostname": "controller",
             "start_time": null,
-            "end_time": null
+            "end_time": null,
+            "status_message": null
         }
     ]
 }

api-ref/source/samples/audit-show-response.json

@@ -51,5 +51,6 @@
     "updated_at": "2018-04-06T11:54:01.266447+00:00",
     "hostname": "controller",
     "start_time": null,
-    "end_time": null
+    "end_time": null,
+    "status_message": null
 }

api-ref/source/samples/datamodel-list-response.json

@@ -1,38 +1,62 @@
 {
     "context": [
         {
-            "server_uuid": "1bf91464-9b41-428d-a11e-af691e5563bb",
+            "server_watcher_exclude": false,
             "server_name": "chenke-test1",
-            "server_vcpus": "1",
+            "server_state": "active",
             "server_memory": "512",
             "server_disk": "1",
-            "server_state": "active",
-            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112",
+            "server_vcpus": "1",
+            "server_metadata": {},
+            "server_project_id": "baea342fc74b4a1785b4a40c69a8d958",
+            "server_locked":false,
+            "server_uuid": "1bf91464-9b41-428d-a11e-af691e5563bb",
+            "server_pinned_az": "nova",
+            "server_flavor_extra_specs": {
+                "hw_rng:allowed": true
+            },
             "node_hostname": "localhost.localdomain",
-            "node_vcpus": "4",
-            "node_vcpu_ratio": "16.0",
+            "node_status": "enabled",
+            "node_disabled_reason": null,
+            "node_state": "up",
             "node_memory": "16383",
-            "node_memory_ratio": "1.5",
+            "node_memory_mb_reserved": "512",
             "node_disk": "37",
+            "node_disk_gb_reserved": "0",
+            "node_vcpus": "4",
+            "node_vcpu_reserved": "0",
+            "node_memory_ratio": "1.5",
+            "node_vcpu_ratio": "16.0",
             "node_disk_ratio": "1.0",
-            "node_state": "up"
+            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112"
         },
         {
-            "server_uuid": "e2cb5f6f-fa1d-4ba2-be1e-0bf02fa86ba4",
+            "server_watcher_exclude": false,
             "server_name": "chenke-test2",
-            "server_vcpus": "1",
+            "server_state": "active",
             "server_memory": "512",
             "server_disk": "1",
-            "server_state": "active",
-            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112",
+            "server_vcpus": "1",
+            "server_metadata": {},
+            "server_project_id": "baea342fc74b4a1785b4a40c69a8d958",
+            "server_locked": false,
+            "server_uuid": "e2cb5f6f-fa1d-4ba2-be1e-0bf02fa86ba4",
+            "server_pinned_az": "nova",
+            "server_flavor_extra_specs": {},
             "node_hostname": "localhost.localdomain",
-            "node_vcpus": "4",
-            "node_vcpu_ratio": "16.0",
+            "node_status": "enabled",
+            "node_disabled_reason": null,
+            "node_state": "up",
             "node_memory": "16383",
-            "node_memory_ratio": "1.5",
+            "node_memory_mb_reserved": "512",
             "node_disk": "37",
+            "node_disk_gb_reserved": "0",
+            "node_vcpus": "4",
+            "node_vcpu_reserved": "0",
+            "node_memory_ratio": "1.5",
+            "node_vcpu_ratio": "16.0",
             "node_disk_ratio": "1.0",
-            "node_state": "up"
+            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112"
         }
     ]
 }

api-ref/source/watcher-api-v1-actionplans.inc

@@ -139,6 +139,7 @@ Response
     - global_efficacy: actionplan_global_efficacy
     - links: links
     - hostname: actionplan_hostname
+    - status_message: actionplan_status_message
 
 **Example JSON representation of an Action Plan:**
 
@@ -177,6 +178,7 @@ Response
     - global_efficacy: actionplan_global_efficacy
     - links: links
     - hostname: actionplan_hostname
+    - status_message: actionplan_status_message
 
 **Example JSON representation of an Audit:**
 
@@ -233,6 +235,7 @@ version 1:
     - global_efficacy: actionplan_global_efficacy
     - links: links
     - hostname: actionplan_hostname
+    - status_message: actionplan_status_message
 
 **Example JSON representation of an Action Plan:**
 

api-ref/source/watcher-api-v1-actions.inc

@@ -23,6 +23,9 @@ following:
 
 -  **PENDING** : the ``Action`` has not been executed yet by the
    ``Watcher Applier``.
+-  **SKIPPED** : the ``Action`` will not be executed because a predefined
+   skipping condition is found by ``Watcher Applier`` or is explicitly
+   skipped by the ``Administrator``.
 -  **ONGOING** : the ``Action`` is currently being processed by the
    ``Watcher Applier``.
 -  **SUCCEEDED** : the ``Action`` has been executed successfully
@@ -111,6 +114,7 @@ Response
     - description: action_description
     - input_parameters: action_input_parameters
     - links: links
+    - status_message: action_status_message
 
 **Example JSON representation of an Action:**
 
@@ -148,8 +152,111 @@ Response
     - description: action_description
     - input_parameters: action_input_parameters
     - links: links
+    - status_message: action_status_message
 
 **Example JSON representation of an Action:**
 
 .. literalinclude:: samples/actions-show-response.json
+   :language: javascript
+
+Skip Action
+===========
+
+.. rest_method::  PATCH /v1/actions/{action_ident}
+
+Skips an Action resource by changing its state to SKIPPED.
+
+.. note::
+    Only Actions in PENDING state can be skipped. The Action must belong to
+    an Action Plan in RECOMMENDED or PENDING state. This operation requires
+    API microversion 1.5 or later.
+
+Normal response codes: 200
+
+Error codes: 400,404,403,409
+
+Request
+-------
+
+.. rest_parameters:: parameters.yaml
+
+   - action_ident: action_ident
+
+**Example Action skip request:**
+
+.. literalinclude:: samples/action-skip-request.json
+   :language: javascript
+
+**Example Action skip request with custom status message:**
+
+.. literalinclude:: samples/action-skip-request-with-message.json
+   :language: javascript
+
+Response
+--------
+
+.. rest_parameters:: parameters.yaml
+
+    - uuid: uuid
+    - action_type: action_type
+    - state: action_state
+    - action_plan_uuid: action_action_plan_uuid
+    - parents: action_parents
+    - description: action_description
+    - input_parameters: action_input_parameters
+    - links: links
+    - status_message: action_status_message
+
+**Example JSON representation of a skipped Action:**
+
+.. literalinclude:: samples/action-skip-response.json
+   :language: javascript
+
+Update Action Status Message
+============================
+
+.. rest_method::  PATCH /v1/actions/{action_ident}
+
+Updates the status_message of an Action that is already in SKIPPED state.
+
+.. note::
+    The status_message field can only be updated for Actions that are currently
+    in SKIPPED state. This allows administrators to fix typos, provide more
+    detailed explanations, or expand on reasons that were initially omitted.
+    This operation requires API microversion 1.5 or later.
+
+Normal response codes: 200
+
+Error codes: 400,404,403,409
+
+Request
+-------
+
+.. rest_parameters:: parameters.yaml
+
+   - action_ident: action_ident
+
+**Example status_message update request for a SKIPPED action:**
+
+.. literalinclude:: samples/action-update-status-message-request.json
+   :language: javascript
+
+Response
+--------
+
+.. rest_parameters:: parameters.yaml
+
+    - uuid: uuid
+    - action_type: action_type
+    - state: action_state
+    - action_plan_uuid: action_action_plan_uuid
+    - parents: action_parents
+    - description: action_description
+    - input_parameters: action_input_parameters
+    - links: links
+    - status_message: action_status_message
+
+**Example JSON representation of an Action with updated status_message:**
+
+.. literalinclude:: samples/action-update-status-message-response.json
    :language: javascript

api-ref/source/watcher-api-v1-audits.inc

@@ -85,6 +85,7 @@ version 1:
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
     - force: audit_force
+    - status_message: audit_status_message
 
 **Example JSON representation of an Audit:**
 
@@ -184,6 +185,7 @@ Response
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
     - force: audit_force
+    - status_message: audit_status_message
 
 **Example JSON representation of an Audit:**
 
@@ -231,6 +233,7 @@ Response
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
     - force: audit_force
+    - status_message: audit_status_message
 
 **Example JSON representation of an Audit:**
 
@@ -286,6 +289,7 @@ version 1:
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
     - force: audit_force
+    - status_message: audit_status_message
 
 **Example JSON representation of an Audit:**
 
@@ -341,6 +345,7 @@ Response
     - start_time: audit_starttime_resp
     - end_time: audit_endtime_resp
     - force: audit_force
+    - status_message: audit_status_message
 
 **Example JSON representation of an Audit:**
 

api-ref/source/watcher-api-v1-datamodel.inc

@@ -35,21 +35,32 @@ Response
 
 .. rest_parameters:: parameters.yaml
 
-    - server_uuid: server_uuid
+    - server_watcher_exclude: server_watcher_exclude
     - server_name: server_name
-    - server_vcpus: server_vcpus
+    - server_state: server_state
     - server_memory: server_memory
     - server_disk: server_disk
-    - server_state: server_state
-    - node_uuid: node_uuid
+    - server_vcpus: server_vcpus
+    - server_metadata: server_metadata
+    - server_project_id: server_project_id
+    - server_locked: server_locked
+    - server_uuid: server_uuid
+    - server_pinned_az: server_pinned_az
+    - server_flavor_extra_specs: server_flavor_extra_specs
     - node_hostname: node_hostname
-    - node_vcpus: node_vcpus
-    - node_vcpu_ratio: node_vcpu_ratio
-    - node_memory: node_memory
-    - node_memory_ratio: node_memory_ratio
-    - node_disk: node_disk
-    - node_disk_ratio: node_disk_ratio
+    - node_status: node_status
+    - node_disabled_reason: node_disabled_reason
     - node_state: node_state
+    - node_memory: node_memory
+    - node_memory_mb_reserved: node_memory_mb_reserved
+    - node_disk: node_disk
+    - node_disk_gb_reserved: node_disk_gb_reserved
+    - node_vcpus: node_vcpus
+    - node_vcpu_reserved: node_vcpu_reserved
+    - node_memory_ratio: node_memory_ratio
+    - node_vcpu_ratio: node_vcpu_ratio
+    - node_disk_ratio: node_disk_ratio
+    - node_uuid: node_uuid
 
 **Example JSON representation of a Data Model:**
 

bindep.txt

@@ -0,0 +1,23 @@
+# This is a cross-platform list tracking distribution packages needed for install and tests;
+# see https://docs.openstack.org/infra/bindep/ for additional information.
+
+mysql [platform:rpm !platform:redhat test]
+mysql-client [platform:dpkg !platform:debian test]
+mysql-devel [platform:rpm !platform:redhat test]
+mysql-server [!platform:redhat !platform:debian test]
+mariadb-devel [platform:rpm platform:redhat test]
+mariadb-server [platform:rpm platform:redhat platform:debian test]
+python3-all [platform:dpkg test]
+python3-all-dev [platform:dpkg test]
+python3 [platform:rpm test]
+python3-devel [platform:rpm test]
+sqlite-devel [platform:rpm test]
+# gettext and graphviz are needed by doc builds only.
+gettext [doc]
+graphviz [doc]
+# fonts-freefont-otf is needed for pdf docs builds with the 'xelatex' engine
+fonts-freefont-otf [pdf-docs]
+texlive [pdf-docs]
+texlive-latex-recommended [pdf-docs]
+texlive-xetex [pdf-docs]
+latexmk [pdf-docs]

devstack/files/apache-watcher-api.template

@@ -1,42 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This is an example Apache2 configuration file for using the
-# Watcher API through mod_wsgi.  This version assumes you are
-# running devstack to configure the software.
-
-Listen %WATCHER_SERVICE_PORT%
-
-<VirtualHost *:%WATCHER_SERVICE_PORT%>
-    WSGIDaemonProcess watcher-api user=%USER% processes=%APIWORKERS% threads=1 display-name=%{GROUP}
-    WSGIScriptAlias / %WATCHER_WSGI_DIR%/app.wsgi
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIProcessGroup watcher-api
-    WSGIPassAuthorization On
-
-    ErrorLogFormat "%M"
-    ErrorLog /var/log/%APACHE_NAME%/watcher-api.log
-    CustomLog /var/log/%APACHE_NAME%/watcher-api-access.log combined
-
-
-    <Directory %WATCHER_WSGI_DIR%>
-        WSGIProcessGroup watcher-api
-        WSGIApplicationGroup %{GLOBAL}
-        <IfVersion >= 2.4>
-            Require all granted
-        </IfVersion>
-        <IfVersion < 2.4>
-            Order allow,deny
-            Allow from all
-        </IfVersion>
-    </Directory>
-</VirtualHost>

devstack/lib/watcher

@@ -55,29 +55,16 @@ else
     WATCHER_BIN_DIR=$(get_python_exec_prefix)
 fi
 
-# There are 2 modes, which is "uwsgi" which runs with an apache
-# proxy uwsgi in front of it, or "mod_wsgi", which runs in
-# apache. mod_wsgi is deprecated, don't use it.
-WATCHER_USE_WSGI_MODE=${WATCHER_USE_WSGI_MODE:-$WSGI_MODE}
-WATCHER_UWSGI=$WATCHER_BIN_DIR/watcher-api-wsgi
+WATCHER_UWSGI=watcher.wsgi.api:application
 WATCHER_UWSGI_CONF=$WATCHER_CONF_DIR/watcher-uwsgi.ini
-
-if is_suse; then
-    WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/srv/www/htdocs/watcher}
-else
-    WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/var/www/watcher}
-fi
+WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/var/www/watcher}
 # Public facing bits
 WATCHER_SERVICE_HOST=${WATCHER_SERVICE_HOST:-$SERVICE_HOST}
 WATCHER_SERVICE_PORT=${WATCHER_SERVICE_PORT:-9322}
 WATCHER_SERVICE_PORT_INT=${WATCHER_SERVICE_PORT_INT:-19322}
 WATCHER_SERVICE_PROTOCOL=${WATCHER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 
-if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
-else
-    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
-fi
+WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
 
 # Entry Points
 # ------------
@@ -101,11 +88,7 @@ function _cleanup_watcher_apache_wsgi {
 # runs that a clean run would need to clean up
 function cleanup_watcher {
     sudo rm -rf $WATCHER_STATE_PATH
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
-    else
-        _cleanup_watcher_apache_wsgi
-    fi
+    remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
 }
 
 # configure_watcher() - Set config files, create data dirs, etc
@@ -154,31 +137,6 @@ function create_watcher_accounts {
         "$WATCHER_API_URL"
 }
 
-# _config_watcher_apache_wsgi() - Set WSGI config files of watcher
-function _config_watcher_apache_wsgi {
-    local watcher_apache_conf
-    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-        local service_port=$WATCHER_SERVICE_PORT
-        if is_service_enabled tls-proxy; then
-            service_port=$WATCHER_SERVICE_PORT_INT
-            service_protocol="http"
-        fi
-        sudo mkdir -p $WATCHER_WSGI_DIR
-        sudo cp $WATCHER_DIR/watcher/api/app.wsgi $WATCHER_WSGI_DIR/app.wsgi
-        watcher_apache_conf=$(apache_site_config_for watcher-api)
-        sudo cp $WATCHER_DEVSTACK_FILES_DIR/apache-watcher-api.template $watcher_apache_conf
-        sudo sed -e "
-            s|%WATCHER_SERVICE_PORT%|$service_port|g;
-            s|%WATCHER_WSGI_DIR%|$WATCHER_WSGI_DIR|g;
-            s|%USER%|$STACK_USER|g;
-            s|%APIWORKERS%|$API_WORKERS|g;
-            s|%APACHE_NAME%|$APACHE_NAME|g;
-        " -i $watcher_apache_conf
-        enable_apache_site watcher-api
-   fi
-
-}
-
 # create_watcher_conf() - Create a new watcher.conf file
 function create_watcher_conf {
     # (Re)create ``watcher.conf``
@@ -196,11 +154,6 @@ function create_watcher_conf {
         iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
         iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT_INT"
         # iniset $WATCHER_CONF api enable_ssl_api "True"
-    else
-        if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-            iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
-            iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
-        fi
     fi
 
     iniset $WATCHER_CONF oslo_policy policy_file $WATCHER_POLICY_YAML
@@ -210,7 +163,7 @@ function create_watcher_conf {
     configure_keystone_authtoken_middleware $WATCHER_CONF watcher
     configure_keystone_authtoken_middleware $WATCHER_CONF watcher "watcher_clients_auth"
 
-    if is_fedora || is_suse; then
+    if is_fedora; then
         # watcher defaults to /usr/local/bin, but fedora and suse pip like to
         # install things in /usr/bin
         iniset $WATCHER_CONF DEFAULT bindir "/usr/bin"
@@ -228,12 +181,8 @@ function create_watcher_conf {
     # Format logging
     setup_logging $WATCHER_CONF
 
-    #config apache files
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
-    else
-        _config_watcher_apache_wsgi
-    fi
+    write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim" "" "watcher-api"
+
     # Register SSL certificates if provided
     if is_ssl_enabled_service watcher; then
         ensure_certificates WATCHER
@@ -273,9 +222,6 @@ function install_watcherclient {
 function install_watcher {
     git_clone $WATCHER_REPO $WATCHER_DIR $WATCHER_BRANCH
     setup_develop $WATCHER_DIR
-    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-        install_apache_wsgi
-    fi
 }
 
 # start_watcher_api() - Start the API process ahead of other things
@@ -289,19 +235,10 @@ function start_watcher_api {
         service_port=$WATCHER_SERVICE_PORT_INT
         service_protocol="http"
     fi
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
-        watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
-    else
-        watcher_url=$service_protocol://$SERVICE_HOST:$service_port
-        enable_apache_site watcher-api
-        restart_apache_server
-        # Start proxies if enabled
-        if is_service_enabled tls-proxy; then
-            start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
-        fi
-    fi
-
+    run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
+    watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
+    # TODO(sean-k-mooney): we should probably check that we can hit
+    # the microversion endpoint and get a valid response.
     echo "Waiting for watcher-api to start..."
     if ! wait_for_service $SERVICE_TIMEOUT $watcher_url; then
         die $LINENO "watcher-api did not start"
@@ -319,17 +256,25 @@ function start_watcher {
 
 # stop_watcher() - Stop running processes (non-screen)
 function stop_watcher {
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        stop_process watcher-api
-    else
-        disable_apache_site watcher-api
-        restart_apache_server
-    fi
+    stop_process watcher-api
     for serv in watcher-decision-engine watcher-applier; do
         stop_process $serv
     done
 }
 
+# configure_tempest_for_watcher() - Configure Tempest for watcher
+function configure_tempest_for_watcher {
+    # Set default microversion for watcher-tempest-plugin
+    # Please make sure to update this when the microversion is updated, otherwise
+    # new tests may be skipped.
+    TEMPEST_WATCHER_MIN_MICROVERSION=${TEMPEST_WATCHER_MIN_MICROVERSION:-"1.0"}
+    TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.6"}
+
+    # Set microversion options in tempest.conf
+    iniset $TEMPEST_CONFIG optimize min_microversion $TEMPEST_WATCHER_MIN_MICROVERSION
+    iniset $TEMPEST_CONFIG optimize max_microversion $TEMPEST_WATCHER_MAX_MICROVERSION
+}
+
 # Restore xtrace
 $_XTRACE_WATCHER
 

devstack/local.conf.compute

@@ -26,7 +26,7 @@ GLANCE_HOSTPORT=${SERVICE_HOST}:9292
 DATABASE_TYPE=mysql
 
 # Enable services (including neutron)
-ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client
+ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client,node-exporter
 
 NOVA_VNC_ENABLED=True
 NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_auto.html"
@@ -42,6 +42,10 @@ disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
 LOGFILE=$DEST/logs/stack.sh.log
 LOGDAYS=2
 
+CEILOMETER_BACKEND="none"
+CEILOMETER_BACKENDS="none"
+enable_plugin devstack-plugin-prometheus https://opendev.org/openstack/devstack-plugin-prometheus
+
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver

devstack/local.conf.controller

@@ -18,6 +18,10 @@ NETWORK_GATEWAY=10.254.1.1 # Change this for your network
 
 MULTI_HOST=1
 
+CEILOMETER_ALARM_THRESHOLD="6000000000"
+CEILOMETER_BACKENDS="sg-core"
+CEILOMETER_PIPELINE_INTERVAL="15"
+
 
 #Set this to FALSE if do not want to run watcher-api behind mod-wsgi
 #WATCHER_USE_MOD_WSGI=TRUE
@@ -40,8 +44,10 @@ disable_service ceilometer-acompute
 # Enable the ceilometer api explicitly(bug:1667678)
 enable_service ceilometer-api
 
-# Enable the Gnocchi plugin
-enable_plugin gnocchi https://github.com/gnocchixyz/gnocchi
+enable_service prometheus
+enable_plugin aodh https://opendev.org/openstack/aodh
+enable_plugin devstack-plugin-prometheus https://opendev.org/openstack/devstack-plugin-prometheus
+enable_plugin sg-core https://github.com/openstack-k8s-operators/sg-core main
 
 LOGFILE=$DEST/logs/stack.sh.log
 LOGDAYS=2
@@ -55,3 +61,42 @@ compute_monitors=cpu.virt_driver
 # can change this to just versioned when ceilometer handles versioned
 # notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
 notification_format=both
+
+[[post-config|$WATCHER_CONF]]
+[prometheus_client]
+host = 127.0.0.1
+port = 9090
+
+[watcher_cluster_data_model_collectors.baremetal]
+period = 120
+
+[watcher_cluster_data_model_collectors.compute]
+period = 120
+
+[watcher_cluster_data_model_collectors.storage]
+period = 120
+
+[watcher_datasources]
+datasources = prometheus
+
+[[test-config|$TEMPEST_CONFIG]]
+[optimize]
+datasource = prometheus
+
+[service_available]
+sg_core = True
+
+[telemetry]
+ceilometer_polling_interval = 15
+disable_ssl_certificate_validation = True
+
+[telemetry_services]
+metric_backends = prometheus
+
+[compute]
+min_compute_nodes = 2
+min_microversion = 2.56
+
+[compute-feature-enabled]
+block_migration_for_live_migration = True
+live_migration = True

devstack/local_gnocchi.conf.compute

@@ -0,0 +1,53 @@
+# Sample ``local.conf`` for compute node for Watcher development
+# NOTE: Copy this file to the root DevStack directory for it to work properly.
+
+[[local|localrc]]
+
+ADMIN_PASSWORD=nomoresecrete
+DATABASE_PASSWORD=stackdb
+RABBIT_PASSWORD=stackqueue
+SERVICE_PASSWORD=$ADMIN_PASSWORD
+SERVICE_TOKEN=azertytoken
+
+HOST_IP=192.168.42.2 # Change this to this compute node's IP address
+#HOST_IPV6=2001:db8::7
+FLAT_INTERFACE=eth0
+
+FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is
+NETWORK_GATEWAY=10.254.1.1 # Change this for your network
+
+MULTI_HOST=1
+
+SERVICE_HOST=192.168.42.1 # Change this to the IP of your controller node
+MYSQL_HOST=$SERVICE_HOST
+RABBIT_HOST=$SERVICE_HOST
+GLANCE_HOSTPORT=${SERVICE_HOST}:9292
+
+DATABASE_TYPE=mysql
+
+# Enable services (including neutron)
+ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client
+
+NOVA_VNC_ENABLED=True
+NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_auto.html"
+VNCSERVER_LISTEN=0.0.0.0
+VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP # or HOST_IPV6
+
+NOVA_INSTANCES_PATH=/opt/stack/data/instances
+
+# Enable the Ceilometer plugin for the compute agent
+enable_plugin ceilometer https://opendev.org/openstack/ceilometer
+disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
+
+LOGFILE=$DEST/logs/stack.sh.log
+LOGDAYS=2
+
+[[post-config|$NOVA_CONF]]
+[DEFAULT]
+compute_monitors=cpu.virt_driver
+[notifications]
+# Enable both versioned and unversioned notifications. Watcher only
+# uses versioned notifications but ceilometer uses unversioned. We
+# can change this to just versioned when ceilometer handles versioned
+# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
+notification_format=both

devstack/local_gnocchi.conf.controller

@@ -0,0 +1,57 @@
+# Sample ``local.conf`` for controller node for Watcher development
+# NOTE: Copy this file to the root DevStack directory for it to work properly.
+
+[[local|localrc]]
+
+ADMIN_PASSWORD=nomoresecrete
+DATABASE_PASSWORD=stackdb
+RABBIT_PASSWORD=stackqueue
+SERVICE_PASSWORD=$ADMIN_PASSWORD
+SERVICE_TOKEN=azertytoken
+
+HOST_IP=192.168.42.1 # Change this to your controller node IP address
+#HOST_IPV6=2001:db8::7
+FLAT_INTERFACE=eth0
+
+FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is
+NETWORK_GATEWAY=10.254.1.1 # Change this for your network
+
+MULTI_HOST=1
+
+
+#Set this to FALSE if do not want to run watcher-api behind mod-wsgi
+#WATCHER_USE_MOD_WSGI=TRUE
+
+# This is the controller node, so disable nova-compute
+disable_service n-cpu
+
+# Enable the Watcher Dashboard plugin
+enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard
+
+# Enable the Watcher plugin
+enable_plugin watcher https://opendev.org/openstack/watcher
+
+# Enable the Ceilometer plugin
+enable_plugin ceilometer https://opendev.org/openstack/ceilometer
+
+# This is the controller node, so disable the ceilometer compute agent
+disable_service ceilometer-acompute
+
+# Enable the ceilometer api explicitly(bug:1667678)
+enable_service ceilometer-api
+
+# Enable the Gnocchi plugin
+enable_plugin gnocchi https://github.com/gnocchixyz/gnocchi
+
+LOGFILE=$DEST/logs/stack.sh.log
+LOGDAYS=2
+
+[[post-config|$NOVA_CONF]]
+[DEFAULT]
+compute_monitors=cpu.virt_driver
+[notifications]
+# Enable both versioned and unversioned notifications. Watcher only
+# uses versioned notifications but ceilometer uses unversioned. We
+# can change this to just versioned when ceilometer handles versioned
+# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
+notification_format=both

devstack/plugin.sh

@@ -36,6 +36,9 @@ if is_service_enabled watcher-api watcher-decision-engine watcher-applier; then
         # Start the watcher components
         echo_summary "Starting watcher"
         start_watcher
+    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
+        echo_summary "Configuring tempest for watcher"
+        configure_tempest_for_watcher
     fi
 
     if [[ "$1" == "unstack" ]]; then

devstack/prometheus.yml

@@ -0,0 +1,16 @@
+global:
+  scrape_interval: 10s
+scrape_configs:
+  - job_name: "node"
+    static_configs:
+      - targets: ["controller:3000"]
+      - targets: ["controller:9100"]
+        labels:
+          fqdn: "controller" # change the hostname here to your controller hostname
+      - targets: ["compute-1:9100"]
+        labels:
+          fqdn: "compute-1" # change the hostname here to your fist compute hostname
+      - targets: ["compute-2:9100"]
+        labels:
+          fqdn: "compute-2" # change the hostname her to your secondd compute hostname
+      # add as many blocks as compute nodes you have

doc/notification_samples/action-cancel-end.json

@@ -14,6 +14,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "CANCELLED",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -24,6 +25,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "CANCELLING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-cancel-error.json

@@ -24,6 +24,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "FAILED",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -34,6 +35,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "CANCELLING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-cancel-start.json

@@ -14,6 +14,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "CANCELLING",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -24,6 +25,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "CANCELLING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-create.json

@@ -13,6 +13,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "PENDING",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -23,6 +24,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "ONGOING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-delete.json

@@ -13,6 +13,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "DELETED",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -23,6 +24,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "ONGOING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-execution-end.json

@@ -14,6 +14,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "SUCCEEDED",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -24,6 +25,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "ONGOING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-execution-error.json

@@ -24,6 +24,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "FAILED",
+      "status_message": "Action execution failed",
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -34,6 +35,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "ONGOING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-execution-start.json

@@ -14,6 +14,7 @@
       "created_at": "2016-10-18T09:52:05Z",
       "updated_at": null,
       "state": "ONGOING",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -24,6 +25,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "ONGOING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action-update.json

@@ -18,10 +18,12 @@
         "watcher_object.name": "ActionStateUpdatePayload",
         "watcher_object.data": {
           "old_state": "PENDING",
-          "state": "ONGOING"
+          "state": "ONGOING",
+          "status_message": null
         }
       },
       "state": "ONGOING",
+      "status_message": null,
       "action_plan": {
         "watcher_object.namespace": "watcher",
         "watcher_object.version": "1.0",
@@ -32,6 +34,7 @@
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null,
           "state": "ONGOING",
+          "status_message": null,
           "audit_uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d",
           "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
           "deleted_at": null

doc/notification_samples/action_plan-cancel-end.json

@@ -21,6 +21,7 @@
           "scope": [],
           "audit_type": "ONESHOT",
           "state": "SUCCEEDED",
+          "status_message": null,
           "parameters": {},
           "interval": null,
           "updated_at": null
@@ -29,6 +30,7 @@
       "uuid": "76be87bd-3422-43f9-93a0-e85a577e3061",
       "fault": null,
       "state": "CANCELLED",
+      "status_message": null,
       "global_efficacy": [],
       "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
       "strategy": {

doc/notification_samples/action_plan-cancel-error.json

@@ -52,13 +52,15 @@
           "scope": [],
           "updated_at": null,
           "audit_type": "ONESHOT",
+          "status_message": null,
           "interval": null,
           "deleted_at": null,
           "state": "SUCCEEDED"
         }
       },
       "global_efficacy": [],
-      "state": "CANCELLING"
+      "state": "CANCELLING",
+      "status_message": null
     }
   },
   "timestamp": "2016-10-18 09:52:05.219414"

doc/notification_samples/action_plan-cancel-start.json

@@ -21,6 +21,7 @@
           "scope": [],
           "audit_type": "ONESHOT",
           "state": "SUCCEEDED",
+          "status_message": null,
           "parameters": {},
           "interval": null,
           "updated_at": null
@@ -29,6 +30,7 @@
       "uuid": "76be87bd-3422-43f9-93a0-e85a577e3061",
       "fault": null,
       "state": "CANCELLING",
+      "status_message": null,
       "global_efficacy": [],
       "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
       "strategy": {

doc/notification_samples/action_plan-create.json

@@ -33,6 +33,7 @@
           "interval": null,
           "deleted_at": null,
           "state": "PENDING",
+          "status_message": null,
           "created_at": "2016-10-18T09:52:05Z",
           "updated_at": null
         },
@@ -43,6 +44,7 @@
       "global_efficacy": {},
       "deleted_at": null,
       "state": "RECOMMENDED",
+      "status_message": null,
       "updated_at": null
     },
     "watcher_object.namespace": "watcher",

doc/notification_samples/action_plan-delete.json

@@ -18,6 +18,7 @@
           "updated_at": null,
           "deleted_at": null,
           "state": "PENDING",
+          "status_message": null,
           "created_at": "2016-10-18T09:52:05Z",
           "parameters": {}
         },
@@ -43,7 +44,8 @@
         "watcher_object.name": "StrategyPayload",
         "watcher_object.namespace": "watcher"
       },
-      "state": "DELETED"
+      "state": "DELETED",
+      "status_message": null
     },
     "watcher_object.version": "1.0",
     "watcher_object.name": "ActionPlanDeletePayload",

doc/notification_samples/action_plan-execution-end.json

@@ -22,6 +22,7 @@
           "scope": [],
           "audit_type": "ONESHOT",
           "state": "SUCCEEDED",
+          "status_message": null,
           "parameters": {},
           "interval": null,
           "updated_at": null
@@ -30,6 +31,7 @@
       "uuid": "76be87bd-3422-43f9-93a0-e85a577e3061",
       "fault": null,
       "state": "ONGOING",
+      "status_message": null,
       "global_efficacy": [],
       "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
       "strategy": {

doc/notification_samples/action_plan-execution-error.json

@@ -55,11 +55,13 @@
           "audit_type": "ONESHOT",
           "interval": null,
           "deleted_at": null,
-          "state": "PENDING"
+          "state": "PENDING",
+          "status_message": null
         }
       },
       "global_efficacy": [],
-      "state": "ONGOING"
+      "state": "ONGOING",
+      "status_message": null
     }
   },
   "timestamp": "2016-10-18 09:52:05.219414"

doc/notification_samples/action_plan-execution-start.json

@@ -22,6 +22,7 @@
           "scope": [],
           "audit_type": "ONESHOT",
           "state": "PENDING",
+          "status_message": null,
           "parameters": {},
           "interval": null,
           "updated_at": null
@@ -30,6 +31,7 @@
       "uuid": "76be87bd-3422-43f9-93a0-e85a577e3061",
       "fault": null,
       "state": "ONGOING",
+      "status_message": null,
       "global_efficacy": [],
       "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
       "strategy": {

doc/notification_samples/action_plan-update.json

@@ -16,6 +16,7 @@
           "interval": null,
           "updated_at": null,
           "state": "PENDING",
+          "status_message": null,
           "deleted_at": null,
           "parameters": {}
         },
@@ -35,6 +36,7 @@
         "watcher_object.name": "ActionPlanStateUpdatePayload"
       },
       "state": "ONGOING",
+      "status_message": null,
       "deleted_at": null,
       "strategy_uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3",
       "strategy": {

doc/notification_samples/audit-create.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "PENDING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "goal_uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",

doc/notification_samples/audit-delete.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "DELETED",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "goal_uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",

doc/notification_samples/audit-planner-end.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "ONGOING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "fault": null,

doc/notification_samples/audit-planner-error.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "ONGOING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "fault": {

doc/notification_samples/audit-planner-start.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "ONGOING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "fault": null,

doc/notification_samples/audit-strategy-end.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "ONGOING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "fault": null,

doc/notification_samples/audit-strategy-error.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "ONGOING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "fault": {

doc/notification_samples/audit-strategy-start.json

@@ -9,6 +9,7 @@
         "para1": 3.2
       },
       "state": "ONGOING",
+      "status_message": null,
       "updated_at": null,
       "deleted_at": null,
       "fault": null,

doc/notification_samples/audit-update.json

@@ -70,6 +70,7 @@
       "interval": null,
       "updated_at": null,
       "state": "ONGOING",
+      "status_message": null,
       "audit_type": "ONESHOT"
     },
     "watcher_object.namespace": "watcher",

doc/source/architecture.rst

@@ -384,7 +384,9 @@ following methods of the :ref:`Action <action_definition>` handler:
 
 -   **preconditions()**: this method will make sure that all conditions are met
     before executing the action (for example, it makes sure that an instance
-    still exists before trying to migrate it).
+    still exists before trying to migrate it). If action specific preconditions
+    are not met in this phase, the Action is set to **SKIPPED** state and will
+    not be executed.
 -   **execute()**: this method is what triggers real commands on other
     OpenStack services (such as Nova, ...) in order to change target resource
     state. If the action is successfully executed, a notification message is
@@ -479,6 +481,39 @@ change to a new value:
 .. image:: ./images/action_plan_state_machine.png
    :width: 100%
 
+.. _action_state_machine:
+
+Action State Machine
+-------------------------
+
+An :ref:`Action <action_definition>` has a life-cycle and its current state may
+be one of the following:
+
+-  **PENDING** : the :ref:`Action <action_definition>` has not been executed
+   yet by the :ref:`Watcher Applier <watcher_applier_definition>`
+-  **SKIPPED** : the :ref:`Action <action_definition>` will not be executed
+   because a predefined skipping condition is found by
+   :ref:`Watcher Applier <watcher_applier_definition>` or is explicitly
+   skipped by the :ref:`Administrator <administrator_definition>`.
+-  **ONGOING** : the :ref:`Action <action_definition>` is currently being
+   processed by the :ref:`Watcher Applier <watcher_applier_definition>`
+-  **SUCCEEDED** : the :ref:`Action <action_definition>` has been executed
+   successfully
+-  **FAILED** : an error occurred while trying to execute the
+   :ref:`Action <action_definition>`
+-  **DELETED** : the :ref:`Action <action_definition>` is still stored in the
+   :ref:`Watcher database <watcher_database_definition>` but is not returned
+   any more through the Watcher APIs.
+-  **CANCELLED** : the :ref:`Action <action_definition>` was in **PENDING** or
+   **ONGOING** state and was cancelled by the
+   :ref:`Administrator <administrator_definition>`
+
+The following diagram shows the different possible states of an
+:ref:`Action <action_definition>` and what event makes the state change
+change to a new value:
+
+.. image:: ./images/action_state_machine.png
+   :width: 100%
 
 
 .. _Watcher API: https://docs.openstack.org/api-ref/resource-optimization/

doc/source/configuration/configuring.rst

@@ -429,20 +429,38 @@ Configure Cinder Notifications
 
 Watcher can also consume notifications generated by the Cinder services, in
 order to build or update, in real time, its cluster data model related to
-storage resources. To do so, you have to update the Cinder configuration
-file on controller and volume nodes, in order to let Watcher receive Cinder
-notifications in a dedicated ``watcher_notifications`` channel.
+storage resources.
 
-  * In the file ``/etc/cinder/cinder.conf``, update the section
-    ``[oslo_messaging_notifications]``, by redefining the list of topics
-    into which Cinder services will publish events ::
+Cinder emits notifications on the ``notifications`` topic, in the openstack
+control exchange (as it can be seen in the `Cinder conf`_).
+
+  * In the file ``/etc/cinder/cinder.conf``, the value of driver in the section
+    ``[oslo_messaging_notifications]`` can't be noop.
 
       [oslo_messaging_notifications]
       driver = messagingv2
-      topics = notifications,watcher_notifications
 
-  * Restart the Cinder services.
+.. _`Cinder conf`: https://docs.openstack.org/cinder/latest/configuration/block-storage/samples/cinder.conf.html
 
+Configure Watcher listening to the Notifications
+================================================
+
+To consume either Cinder or Nova notifications, (or both), Watcher must be
+configured to listen to the notifications topics that Cinder and Nova emit.
+
+Use the `notification_topics`_ config option to indicate to Watcher that it
+should listen to the correct topics. By default, Cinder emits notifications
+on ``openstack.notifications``, while Nova emits notifications on
+``nova.versioned_notifications``. The Watcher conf should have the topics for
+the desired notifications, below is an example for both Cinder and Nova::
+
+    [watcher_decision_engine]
+
+    ...
+
+    notification_topics = nova.versioned_notifications,openstack.notifications
+
+.. _`notification_topics`: https://docs.openstack.org/watcher/latest/configuration/watcher.html#watcher_decision_engine.notification_topics
 
 Workers
 =======

doc/source/contributor/concurrency.rst

@@ -52,18 +52,43 @@ types of concurrency used in various services of Watcher.
 .. _wait_for_any: https://docs.openstack.org/futurist/latest/reference/index.html#waiters
 
 
+Concurrency modes
+#################
+
+Evenlet has been the main concurrency library within the OpenStack community
+for the last 10 years since the removal of twisted. Over the last few years,
+the maintenance of eventlet has decreased and the efforts to remove the GIL
+from Python (PEP 703), have fundamentally changed how concurrency is making
+eventlet no longer viable. While transitioning to a new native thread
+solution, Watcher services will be supporting both modes, with the usage of
+native threading mode initially classified as ``experimental``.
+
+It is possible to enable the new native threading mode by setting the following
+environment variable in the corresponding service configuration:
+
+.. code:: bash
+
+   OS_WATCHER_DISABLE_EVENTLET_PATCHING=true
+
+.. note::
+
+   The only service that supports two different concurrency modes is the
+   ``decision engine``.
+
 Decision engine concurrency
 ***************************
 
 The concurrency in the decision engine is governed by two independent
-threadpools. Both of these threadpools are GreenThreadPoolExecutor_ from the
-futurist_ library. One of these is used automatically and most contributors
+threadpools. These threadpools can be configured as GreenThreadPoolExecutor_
+or ThreadPoolExecutor_, both from the futurist_ library, depending on the
+service configuration. One of these is used automatically and most contributors
 will not interact with it while developing new features. The other threadpool
 can frequently be used while developing new features or updating existing ones.
 It is known as the DecisionEngineThreadpool and allows to achieve performance
 improvements in network or I/O bound operations.
 
-.. _GreenThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#executors
+.. _GreenThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#futurist.GreenThreadPoolExecutor
+.. _ThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#futurist.ThreadPoolExecutor
 
 AuditEndpoint
 #############

doc/source/contributor/devstack.rst

@@ -31,14 +31,45 @@ Quick Devstack Instructions with Datasources
 ============================================
 
 Watcher requires a datasource to collect metrics from compute nodes and
-instances in order to execute most strategies. To enable this a
-``[[local|localrc]]`` to setup DevStack for some of the supported datasources
-is provided. These examples specify the minimal configuration parameters to
-get both Watcher and the datasource working but can be expanded is desired.
+instances in order to execute most strategies. To enable this two possible
+examples of ``[[local|localrc]]`` to setup DevStack for some of the
+supported datasources is provided. These examples specify the minimal
+configuration parameters to get both Watcher and the datasource working
+but can be expanded is desired.
+The first example configures watcher to user prometheus as a datasource, while
+the second example show how to use gnocchi as the datasource. The procedure is
+equivalent, it just requires using the ``local.conf.controller`` and
+``local.conf.compute`` in the first example and
+``local_gnocchi.conf.controller`` and ``local_gnocchi.conf.compute`` in the
+second.
+
+Prometheus
+----------
+
+With the Prometheus datasource most of the metrics for compute nodes and
+instances will work with the provided configuration but metrics that
+require Ironic such as ``host_airflow and`` ``host_power`` will still be
+unavailable as well as ``instance_l3_cpu_cache``
+
+.. code-block:: ini
+
+   [[local|localrc]]
+
+   enable_plugin watcher https://opendev.org/openstack/watcher
+   enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard
+   enable_plugin ceilometer https://opendev.org/openstack/ceilometer.git
+   enable_plugin aodh https://opendev.org/openstack/aodh
+   enable_plugin devstack-plugin-prometheus https://opendev.org/openstack/devstack-plugin-prometheus
+   enable_plugin sg-core https://github.com/openstack-k8s-operators/sg-core main
+
+
+   CEILOMETER_BACKEND=sg-core
+   [[post-config|$NOVA_CONF]]
+   [DEFAULT]
+   compute_monitors=cpu.virt_driver
 
 Gnocchi
 -------
-
 With the Gnocchi datasource most of the metrics for compute nodes and
 instances will work with the provided configuration but metrics that
 require Ironic such as ``host_airflow and`` ``host_power`` will still be
@@ -96,7 +127,8 @@ Detailed DevStack Instructions
         cd ~
         git clone https://opendev.org/openstack/devstack.git
 
-#. For each compute node, copy the provided `local.conf.compute`_ example file
+#. For each compute node, copy the provided `local.conf.compute`_
+   (`local_gnocchi.conf.compute`_ if deploying with gnocchi) example file
    to the compute node's system at ~/devstack/local.conf. Make sure the
    HOST_IP and SERVICE_HOST values are changed appropriately - i.e., HOST_IP
    is set to the IP address of the compute node and SERVICE_HOST is set to the
@@ -112,7 +144,8 @@ Detailed DevStack Instructions
    to configure similar configuration options for the projects providing those
    metrics.
 
-#. For the controller node, copy the provided `local.conf.controller`_ example
+#. For the controller node, copy the provided `local.conf.controller`_
+   (`local_gnocchi.conf.controller`_ if deploying with gnocchi) example
    file to the controller node's system at ~/devstack/local.conf. Make sure
    the HOST_IP value is changed appropriately - i.e., HOST_IP is set to the IP
    address of the controller node.
@@ -142,6 +175,17 @@ Detailed DevStack Instructions
         to FALSE. For Production environment it is suggested to keep it at the
         default TRUE value.
 
+#. If you want to use prometheus as a datasource, you need to provide a
+   Prometheus configuration with the compute nodes set as targets, so
+   it can consume their node-exporter metrics (if you are deploying watcher
+   with gnocchi as datasource you can skip this step altogether). Copy the
+   provided `prometheus.yml`_ example file and set the appropriate hostnames
+   for all the compute nodes (the example configures 2 of them plus the
+   controller, but you should add all of them if using more than 2 compute
+   nodes). Set the value of ``PROMETHEUS_CONFIG_FILE`` to the path of the
+   file you created in the local.conf file (the sample local.conf file uses
+   ``$DEST`` as the default value for the prometheus config path).
+
 #. Start stacking from the controller node::
 
        ./devstack/stack.sh
@@ -154,6 +198,9 @@ Detailed DevStack Instructions
 
 .. _local.conf.controller: https://github.com/openstack/watcher/tree/master/devstack/local.conf.controller
 .. _local.conf.compute: https://github.com/openstack/watcher/tree/master/devstack/local.conf.compute
+.. _local_gnocchi.conf.controller: https://github.com/openstack/watcher/tree/master/devstack/local_gnocchi.conf.controller
+.. _local_gnocchi.conf.compute: https://github.com/openstack/watcher/tree/master/devstack/local_gnocchi.conf.compute
+.. _prometheus.yml: https://github.com/openstack/watcher/tree/master/devstack/prometheus.yml
 
 Multi-Node DevStack Environment
 ===============================

doc/source/contributor/index.rst

@@ -10,3 +10,4 @@ Contribution Guide
    devstack
    testing
    rally_link
+   release-guide

doc/source/contributor/release-guide.rst

@@ -0,0 +1,462 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+Chronological Release Liaison Guide
+====================================
+
+This is a reference guide that a release liaison may use as an aid, if
+they choose.
+
+Watcher uses the `Distributed Project Leadership (DPL)`__ model where
+traditional release liaison responsibilities are distributed among various
+liaisons. The release liaison is responsible for requesting releases,
+reviewing Feature Freeze Exception (FFE) requests, and coordinating
+release-related activities with the team.
+
+.. __: https://governance.openstack.org/tc/reference/distributed-project-leadership.html
+
+How to Use This Guide
+---------------------
+
+This guide is organized chronologically to follow the OpenStack release
+cycle from PTG planning through post-release activities. You can use it
+in two ways:
+
+**For New Release Liaisons**
+    Read through the entire guide to understand the full release cycle,
+    then bookmark it for reference during your term.
+
+**For Experienced Release Liaisons**
+    Jump directly to the relevant section for your current phase in the
+    release cycle. Each major section corresponds to a specific time period.
+
+**Key Navigation Tips**
+    * The :ref:`glossary` defines all acronyms and terminology used
+    * Time-sensitive activities are clearly marked by milestone phases
+    * DPL coordination notes indicate when team collaboration is required
+
+DPL Liaison Coordination
+-------------------------
+
+Under the DPL model, the release liaison coordinates with other project
+liaisons and the broader team for effective release management. The release
+liaison has authority for release-specific decisions (FFE approvals, release
+timing, etc.) while major process changes and strategic decisions require
+team consensus.
+
+This coordination approach ensures that:
+
+* Release activities are properly managed by a dedicated liaison
+* Team input is gathered for significant decisions
+* Other liaisons are informed of release-related developments that may
+  affect their areas
+* Release processes remain responsive while maintaining team alignment
+
+Project Context
+---------------
+
+* Coordinate with the watcher meeting (chair rotates each meeting, with
+  volunteers requested at the end of each meeting)
+
+  * Meeting etherpad: https://etherpad.opendev.org/p/openstack-watcher-irc-meeting
+  * IRC channel: #openstack-watcher
+
+* Get acquainted with the release schedule
+
+  * Example: https://releases.openstack.org/<current-release>/schedule.html
+
+* Familiarize with Watcher project repositories and tracking:
+
+Watcher Main Repository
+    `Primary codebase for the Watcher service <https://opendev.org/openstack/watcher>`__
+
+Watcher Dashboard
+    `Horizon plugin for Watcher UI <https://opendev.org/openstack/watcher-dashboard>`__
+
+Watcher Tempest Plugin
+    `Integration tests <https://opendev.org/openstack/watcher-tempest-plugin>`__ (follows tempest cycle)
+
+Python Watcher Client
+    `Command-line client and Python library <https://opendev.org/openstack/python-watcherclient>`__
+
+Watcher Specifications
+    `Design specifications <https://opendev.org/openstack/watcher-specs>`__ (not released)
+
+Watcher Launchpad (Main)
+    `Primary bug and feature tracking <https://launchpad.net/watcher>`__
+
+Watcher Dashboard Launchpad
+    `Dashboard-specific tracking <https://launchpad.net/watcher-dashboard/>`__
+
+Watcher Tempest Plugin Launchpad
+    `Test plugin tracking <https://launchpad.net/watcher-tempest-plugin>`__
+
+Python Watcher Client Launchpad
+    `Client library tracking <https://launchpad.net/python-watcherclient>`__
+
+Project Team Gathering
+----------------------
+
+Event Liaison Coordination
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Work with the project team to select an event liaison for PTG coordination.
+  The event liaison is responsible for:
+
+  * Reserving sufficient space at PTG for the project team's meetings
+  * Putting out an agenda for team meetings
+  * Ensuring meetings are organized and facilitated
+  * Documenting meeting results
+
+* If no event liaison is selected, these duties revert to the release liaison.
+
+* Monitor for OpenStack Events team queries on the mailing list requesting
+  event liaison volunteers - teams not responding may lose event
+  representation.
+
+PTG Planning and Execution
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Create PTG planning etherpad, retrospective etherpad and alert about it in
+  watcher meeting and dev mailing list
+
+  * Example: https://etherpad.opendev.org/p/apr2025-ptg-watcher
+
+* Run sessions at the PTG (if no event liaison is selected)
+
+* Do a retro of the previous cycle
+
+* Coordinate with team to establish agreement on the agenda for this release:
+
+Review Days Planning
+    Determine number of review days allocated for specs and implementation work
+
+Freeze Dates Coordination
+    Define Spec approval and Feature freeze dates through team collaboration
+
+Release Schedule Modifications
+    Modify the OpenStack release schedule if needed by proposing new dates
+    (Example: https://review.opendev.org/c/openstack/releases/+/877094)
+
+* Discuss the implications of the `SLURP or non-SLURP`__ current release
+
+.. __: https://governance.openstack.org/tc/resolutions/20220210-release-cadence-adjustment.html
+
+* Sign up for group photo at the PTG (if applicable)
+
+
+After PTG
+---------
+
+* Send PTG session summaries to the dev mailing list
+
+* Add `RFE bugs`__ if you have action items that are simple to do but
+  without a owner yet.
+
+* Update IRC #openstack-watcher channel topic to point to new
+  development-planning etherpad.
+
+.. __: https://bugs.launchpad.net/watcher/+bugs?field.tag=rfe
+
+A few weeks before milestone 1
+------------------------------
+
+* Plan a spec review day
+
+* Periodically check the series goals others have proposed in the “Set series
+  goals” link:
+
+  * Example: https://blueprints.launchpad.net/watcher/<current-release>/+setgoals
+
+Milestone 1
+-----------
+
+* Release watcher and python-watcherclient via the openstack/releases repo.
+  Watcher follows the `cycle-with-intermediary`__ release model:
+
+.. __: https://releases.openstack.org/reference/release_models.html#cycle-with-intermediary
+
+  * Create actual releases (not just launchpad bookkeeping) at milestone points
+  * No launchpad milestone releases are created for intermediary releases
+  * When releasing the first version of a library for the cycle,
+    bump
+    the minor version to leave room for future stable branch
+    releases
+
+* Release stable branches of watcher
+
+Stable Branch Release Process
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prepare the stable branch for evaluation:
+
+.. code-block:: bash
+
+   git checkout <stable branch>
+   git log --no-merges <last tag>..
+
+Analyze commits to determine version bump according to semantic versioning.
+
+Semantic Versioning Guidelines
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Choose version bump based on changes since last release:
+
+Major Version (X)
+    Backward-incompatible changes that break existing APIs
+
+Minor Version (Y)
+    New features that maintain backward compatibility
+
+Patch Version (Z)
+    Bug fixes that maintain backward compatibility
+
+Release Command Usage
+~~~~~~~~~~~~~~~~~~~~~
+
+Generate the release using OpenStack tooling:
+
+* Use the `new-release command
+  <https://releases.openstack.org/reference/using.html#using-new-release-command>`__
+* Propose the release with version according to chosen semver format
+  (x.y.z)
+
+Summit
+------
+
+``Responsibility Precedence for Summit Activities:``
+
+1. ``Project Update/Onboarding Liaisons`` (if appointed):
+
+   * ``Project Update Liaison``: responsible for giving the project update
+     showcasing team's achievements for the cycle to the community
+   * ``Project Onboarding Liaison``: responsible for giving/facilitating
+     onboarding sessions during events for the project's community
+
+2. ``Event Liaison`` (if no Project Update/Onboarding liaisons exist):
+
+   * Coordinates all Summit activities including project updates and onboarding
+
+3. ``Release Liaison`` (if no Event Liaison is appointed):
+
+   * Work with the team to ensure Summit activities are properly handled:
+
+     * Prepare the project update presentation
+     * Prepare the on-boarding session materials
+     * Prepare the operator meet-and-greet session
+
+.. note::
+
+   The team can choose to not have a Summit presence if desired.
+
+A few weeks before milestone 2
+------------------------------
+
+* Plan a spec review day (optional)
+
+Milestone 2
+-----------
+
+* Spec freeze (unless changed by team agreement at PTG)
+
+* Release watcher and python-watcherclient (if needed)
+
+* Stable branch releases of watcher
+
+
+Shortly after spec freeze
+-------------------------
+
+* Create a blueprint status etherpad to help track, especially non-priority
+  blueprint work, to help things get done by Feature Freeze (FF). Example:
+
+  * https://etherpad.opendev.org/p/watcher-<release>-blueprint-status
+
+* Create or review a patch to add the next release’s specs directory so people
+  can propose specs for next release after spec freeze for current release
+
+Milestone 3
+-----------
+
+* Feature freeze day
+
+* Client library freeze, release python-watcherclient
+
+* Close out all blueprints, including “catch all” blueprints like mox,
+  versioned notifications
+
+* Stable branch releases of watcher
+
+* Start writing the `cycle highlights
+  <https://docs.openstack.org/project-team-guide/release-management.html#cycle-highlights>`__
+
+Week following milestone 3
+--------------------------
+
+* If warranted, announce the FFE (feature freeze exception process) to
+  have people propose FFE requests to a special etherpad where they will
+  be reviewed.
+  FFE requests should first be discussed in the IRC meeting with the
+  requester present.
+  The release liaison has final decision on granting exceptions.
+
+  .. note::
+
+    if there is only a short time between FF and RC1 (lately it’s been 2
+    weeks), then the only likely candidates will be low-risk things that are
+    almost done. In general Feature Freeze exceptions should not be granted,
+    instead features should be deferred and reproposed for the next
+    development
+    cycle. FFE never extend beyond RC1.
+
+* Mark the max microversion for the release in the
+  :doc:`/contributor/api_microversion_history`
+
+A few weeks before RC
+---------------------
+
+* Update the release status etherpad with RC1 todos and keep track
+  of them in meetings
+
+* Go through the bug list and identify any rc-potential bugs and tag them
+
+RC
+--
+
+* Follow the standard OpenStack release checklist process
+
+* If we want to drop backward-compat RPC code, we have to do a major RPC
+  version bump and coordinate it just before the major release:
+
+  * https://wiki.openstack.org/wiki/RpcMajorVersionUpdates
+
+  * Example: https://review.opendev.org/541035
+
+* “Merge latest translations" means translation patches
+
+  * Check for translations with:
+
+    * https://review.opendev.org/#/q/status:open+project:openstack/watcher+branch:master+topic:zanata/translations
+
+* Should NOT plan to have more than one RC if possible. RC2 should only happen
+  if there was a mistake and something was missed for RC, or a new regression
+  was discovered
+
+* Write the reno prelude for the release GA
+
+  * Example: https://review.opendev.org/644412
+
+* Push the cycle-highlights in marketing-friendly sentences and propose to the
+  openstack/releases repo. Usually based on reno prelude but made more readable
+  and friendly
+
+  * Example: https://review.opendev.org/644697
+
+Immediately after RC
+--------------------
+
+* Look for bot proposed changes to reno and stable/<cycle>
+
+* Create the launchpad series for the next cycle
+
+* Set the development focus of the project to the new cycle series
+
+* Set the status of the new series to “active development”
+
+* Set the last series status to “current stable branch release”
+
+* Set the previous to last series status to “supported”
+
+* Repeat launchpad steps ^ for all watcher deliverables.
+
+* Make sure the specs directory for the next cycle gets created so people can
+  start proposing new specs
+
+* Make sure to move implemented specs from the previous release
+
+  * Move implemented specs manually (TODO: add tox command in future)
+
+  * Remove template files:
+
+    .. code-block:: bash
+
+       rm doc/source/specs/<release>/index.rst
+       rm doc/source/specs/<release>/template.rst
+
+* Ensure liaison handoff: either transition to new release liaison or confirm
+  reappointment for next cycle
+
+.. _glossary:
+
+Glossary
+--------
+
+DPL
+    Distributed Project Leadership - A governance model where traditional PTL
+    responsibilities are distributed among various specialized liaisons.
+
+FFE
+    Feature Freeze Exception - A request to add a feature after the feature
+    freeze deadline. Should be used sparingly for low-risk, nearly
+    complete features.
+
+GA
+    General Availability - The final release of a software version for
+    production use.
+
+PTG
+    Project Team Gathering - A collaborative event where OpenStack project
+    teams meet to plan and coordinate development activities.
+
+RC
+    Release Candidate - A pre-release version that is potentially the final
+    version, pending testing and bug fixes.
+
+RFE
+    Request for Enhancement - A type of bug report requesting a new feature
+    or enhancement to existing functionality.
+
+SLURP
+    Skip Level Upgrade Release Process - An extended maintenance release
+    that allows skipping intermediate versions during upgrades.
+
+Summit
+    OpenStack Summit - A conference where the OpenStack community gathers
+    for presentations, discussions, and project updates.
+
+Miscellaneous Notes
+-------------------
+
+How to track launchpad blueprint approvals
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Core team approves blueprints through team consensus. The release liaison
+ensures launchpad status is updated correctly after core team approval:
+
+* Set the approver as the core team member who approved the spec
+
+* Set the Direction => Approved and Definition => Approved and make sure the
+  Series goal is set to the current release. If code is already proposed, set
+  Implementation => Needs Code Review
+
+* Optional: add a comment to the Whiteboard explaining the approval,
+  with a date
+  (launchpad does not record approval dates). For example: “We discussed this
+  in the team meeting and agreed to approve this for <release>. -- <nick>
+  <YYYYMMDD>”
+
+How to complete a launchpad blueprint
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Set Implementation => Implemented. The completion date will be recorded by
+  launchpad

doc/source/datasources/aetos.rst

@@ -0,0 +1,157 @@
+================
+Aetos datasource
+================
+
+Synopsis
+--------
+The Aetos datasource allows Watcher to use an Aetos reverse proxy server as the
+source for collected metrics used by the Watcher decision engine. Aetos is a
+multi-tenant aware reverse proxy that sits in front of a Prometheus server and
+provides Keystone authentication and role-based access control. The Aetos
+datasource uses Keystone service discovery to locate the Aetos endpoint and
+requires authentication via Keystone tokens.
+
+Requirements
+-------------
+The Aetos datasource has the following requirements:
+
+* An Aetos reverse proxy server deployed in front of Prometheus
+* Aetos service registered in Keystone with service type 'metric-storage'
+* Valid Keystone credentials for Watcher with admin or service role
+* Prometheus metrics with appropriate labels (same as direct Prometheus access)
+
+Like the Prometheus datasource, it is required that Prometheus metrics contain
+a label to identify the hostname of the exporter from which the metric was
+collected. This is used to match against the Watcher cluster model
+``ComputeNode.hostname``. The default for this label is ``fqdn`` and in the
+prometheus scrape configs would look like:
+
+.. code-block::
+
+    scrape_configs:
+    - job_name: node
+      static_configs:
+     - targets: ['10.1.2.3:9100']
+        labels:
+          fqdn: "testbox.controlplane.domain"
+
+This default can be overridden when a deployer uses a different label to
+identify the exporter host (for example ``hostname`` or ``host``, or any other
+label, as long as it identifies the host).
+
+Internally this label is used in creating ``fqdn_instance_labels``, containing
+the list of values assigned to the label in the Prometheus targets.
+The elements of the resulting fqdn_instance_labels are expected to match the
+``ComputeNode.hostname`` used in the Watcher decision engine cluster model.
+An example ``fqdn_instance_labels`` is the following:
+
+.. code-block::
+
+    [
+     'ena.controlplane.domain',
+     'dio.controlplane.domain',
+     'tria.controlplane.domain',
+    ]
+
+For instance metrics, it is required that Prometheus contains a label
+with the uuid of the OpenStack instance in each relevant metric. By default,
+the datasource will look for the label ``resource``. The
+``instance_uuid_label`` config option in watcher.conf allows deployers to
+override this default to any other label name that stores the ``uuid``.
+
+Limitations
+-----------
+The Aetos datasource shares the same limitations as the Prometheus datasource:
+
+The current implementation doesn't support the ``statistic_series`` function of
+the Watcher ``class DataSourceBase``. It is expected that the
+``statistic_aggregation`` function (which is implemented) is sufficient in
+providing the **current** state of the managed resources in the cluster.
+The ``statistic_aggregation`` function defaults to querying back 300 seconds,
+starting from the present time (the time period is a function parameter and
+can be set to a value as required). Implementing the ``statistic_series`` can
+always be re-visited if the requisite interest and work cycles are volunteered
+by the interested parties.
+
+One further note about a limitation in the implemented
+``statistic_aggregation`` function. This function is defined with a
+``granularity`` parameter, to be used when querying whichever of the Watcher
+``DataSourceBase`` metrics providers. In the case of Aetos (like Prometheus),
+we do not fetch and then process individual metrics across the specified time
+period. Instead we use the PromQL querying operators and functions, so that the
+server itself will process the request across the specified parameters and
+then return the result. So ``granularity`` parameter is redundant and remains
+unused for the Aetos implementation of ``statistic_aggregation``. The
+granularity of the data fetched by Prometheus server is specified in
+configuration as the server ``scrape_interval`` (current default 15 seconds).
+
+Additionally, there is a slight performance impact compared to direct
+Prometheus access. Since Aetos acts as a reverse proxy in front of Prometheus,
+there is an additional step for each request, resulting in slightly longer
+delays.
+
+Configuration
+-------------
+A deployer must set the ``datasources`` parameter to include ``aetos``
+under the watcher_datasources section of watcher.conf (or add ``aetos`` in
+datasources for a specific strategy if preferred eg. under the
+``[watcher_strategies.workload_stabilization]`` section).
+
+.. note::
+   Having both Prometheus and Aetos datasources configured at the same time
+   is not supported and will result in a configuration error. Allowing this
+   can be investigated in the future if a need or a proper use case is
+   identified.
+
+The watcher.conf configuration file is also used to set the parameter values
+required by the Watcher Aetos data source. The configuration can be
+added under the ``[aetos_client]`` section and the available options are
+duplicated below from the code as they are self documenting:
+
+.. code-block::
+
+    cfg.StrOpt('interface',
+               default='public',
+               choices=['internal', 'public', 'admin'],
+               help="Type of endpoint to use in keystoneclient."),
+    cfg.StrOpt('region_name',
+               help="Region in Identity service catalog to use for "
+                    "communication with the OpenStack service."),
+    cfg.StrOpt('fqdn_label',
+               default='fqdn',
+               help="The label that Prometheus uses to store the fqdn of "
+                    "exporters. Defaults to 'fqdn'."),
+    cfg.StrOpt('instance_uuid_label',
+               default='resource',
+               help="The label that Prometheus uses to store the uuid of "
+                    "OpenStack instances. Defaults to 'resource'."),
+
+
+Authentication and Service Discovery
+------------------------------------
+Unlike the Prometheus datasource which requires explicit host and port
+configuration, the Aetos datasource uses Keystone service discovery to
+automatically locate the Aetos endpoint. The datasource:
+
+1. Uses the configured Keystone credentials to authenticate
+2. Searches the service catalog for a service with type 'metric-storage'
+3. Uses the discovered endpoint URL to connect to Aetos
+4. Attaches a Keystone token to each request for authentication
+
+If the Aetos service is not registered in Keystone, the datasource will
+fail to initialize and prevent the decision engine from starting.
+
+So a sample watcher.conf configured to use the Aetos datasource would look
+like the following:
+
+.. code-block::
+
+    [watcher_datasources]
+
+    datasources = aetos
+
+    [aetos_client]
+
+    interface = public
+    region_name = RegionOne
+    fqdn_label = fqdn

doc/source/datasources/index.rst

@@ -1,6 +1,11 @@
 Datasources
 ===========
 
+.. note::
+   The Monasca datasource is deprecated for removal and optional. To use it, install the optional extra:
+   ``pip install watcher[monasca]``. If Monasca is configured without installing the extra, Watcher will raise
+   an error guiding you to install the client.
+
 .. toctree::
    :glob:
    :maxdepth: 1

doc/source/datasources/prometheus.rst

@@ -29,25 +29,25 @@ This default can be overridden when a deployer uses a different label to
 identify the exporter host (for example ``hostname`` or ``host``, or any other
 label, as long as it identifies the host).
 
-Internally this label is used in creating a ``fqdn_instance_map``, mapping
-the fqdn with the Prometheus instance label associated with each exporter.
-The keys of the resulting fqdn_instance_map are expected to match the
+Internally this label is used in creating ``fqdn_instance_labels``, containing
+the list of values assigned to the label in the Prometheus targets.
+The elements of the resulting fqdn_instance_labels are expected to match the
 ``ComputeNode.hostname`` used in the Watcher decision engine cluster model.
-An example ``fqdn_instance_map`` is the following:
+An example ``fqdn_instance_labels`` is the following:
 
 .. code-block::
 
-    {
-     'ena.controlplane.domain': '10.1.2.1:9100',
-     'dio.controlplane.domain': '10.1.2.2:9100',
-     'tria.controlplane.domain': '10.1.2.3:9100'
-    }
+    [
+     'ena.controlplane.domain',
+     'dio.controlplane.domain',
+     'tria.controlplane.domain',
+    ]
 
 For instance metrics, it is required that Prometheus contains a label
 with the uuid of the OpenStack instance in each relevant metric. By default,
 the datasource will look for the label ``resource``. The
 ``instance_uuid_label`` config option in watcher.conf allows deployers to
-override this default to any other label name that stores the  ``uuid``.
+override this default to any other label name that stores the ``uuid``.
 
 Limitations
 -----------

doc/source/image_src/plantuml/action_state_machine.txt

@@ -0,0 +1,23 @@
+@startuml
+
+skinparam ArrowColor DarkRed
+skinparam StateBorderColor DarkRed
+skinparam StateBackgroundColor LightYellow
+skinparam Shadowing true
+
+[*] --> PENDING: The Watcher Planner\ncreates the Action
+PENDING --> SKIPPED: The Action detects skipping condition\n in pre_condition or was\n skipped by cloud Admin.
+PENDING --> FAILED: The Action fails unexpectedly\n in pre_condition.
+PENDING --> ONGOING: The Watcher Applier starts executing/n the action.
+ONGOING --> FAILED: Something failed while executing\nthe Action in the Watcher Applier
+ONGOING --> SUCCEEDED: The Watcher Applier executed\nthe Action successfully
+FAILED --> DELETED : Administrator removes\nAction Plan
+SUCCEEDED --> DELETED : Administrator removes\n theAction
+ONGOING --> CANCELLED : The Action was cancelled\n as part of an Action Plan cancellation.
+PENDING --> CANCELLED : The Action was cancelled\n as part of an Action Plan cancellation.
+CANCELLED --> DELETED
+FAILED --> DELETED
+SKIPPED --> DELETED
+DELETED --> [*]
+
+@enduml

doc/source/index.rst

@@ -42,6 +42,7 @@ specific prior release.
   user/index
   configuration/index
   contributor/plugin/index
+  integrations/index
   man/index
 
 .. toctree::

doc/source/integrations/index.rst

@@ -0,0 +1,126 @@
+============
+Integrations
+============
+
+The following table provides an Integration status with different services
+which Watcher interact with. Some integrations are marked as Supported,
+while others as Experimental due to the lack of testing and a proper
+documentations.
+
+Integration Status Matrix
+-------------------------
+
+    .. list-table::
+       :widths: 20 20 20 20
+       :header-rows: 1
+
+       * - Service Name
+         - Integration Status
+         - Documentation
+         - Testing
+       * - :ref:`Cinder <cinder_integration>`
+         - Supported
+         - Minimal
+         - Unit
+       * - :ref:`Glance <glance_integration>`
+         - Experimental
+         - Missing
+         - None
+       * - :ref:`Ironic <ironic_integration>`
+         - Experimental
+         - Minimal
+         - Unit
+       * - :ref:`Keystone <keystone_integration>`
+         - Supported
+         - Minimal
+         - Integration
+       * - :ref:`MAAS <maas_integration>`
+         - Experimental
+         - Missing
+         - Unit
+       * - :ref:`Neutron <neutron_integration>`
+         - Experimental
+         - Missing
+         - Unit
+       * - :ref:`Nova <nova_integration>`
+         - Supported
+         - Minimal
+         - Unit and Integration
+       * - :ref:`Placement <placement_integration>`
+         - Supported
+         - Minimal
+         - Unit and Integration
+
+.. note::
+   Minimal documentation covers only basic configuration and, if available,
+   how to enable notifications.
+
+.. _cinder_integration:
+
+Cinder
+^^^^^^
+The OpenStack Block Storage service integration includes a cluster data
+model collector that creates a in-memory representation of the storage
+resources, strategies that propose solutions based on storage capacity
+and Actions that perform volume migration.
+
+.. _glance_integration:
+
+Glance
+^^^^^^
+The Image service integration is consumed by Nova Helper to create instances
+from images, which was used older releases of Watcher to cold migrate
+instances. This procedure is not used by Watcher anymore and this integration
+is classified as Experimental and may be removed in future releases.
+
+.. _ironic_integration:
+
+Ironic
+^^^^^^
+The Bare Metal service integration includes a data model collector that
+creates an in-memory representation of Ironic resources and Actions that
+allows the management of the power state of nodes. This integration is
+classified as Experimental and may be removed in future releases.
+
+.. _keystone_integration:
+
+Keystone
+^^^^^^^^
+The Identity service integration includes authentication with other services
+and retrieving information about domains, projects and users.
+
+.. _maas_integration:
+
+MAAS (Metal As A Service)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+This integration allows managing bare metal servers of a MAAS service,
+which includes Actions that manage the power state of nodes. This
+integration is classified as Experimental and may be removed in future
+releases.
+
+.. _neutron_integration:
+
+Neutron
+^^^^^^^
+Neutron integration is currently consumed by Nova Helper to create instance,
+which was used by older releases of Watcher to cold migrate instances. This
+procedure is not used by Watcher anymore and this integration is classified
+as Experimental and may be removed in future releases.
+
+.. _nova_integration:
+
+Nova
+^^^^
+Nova service integration includes a cluster data model collector that creates
+an in-memory representation of the compute resources available in the cloud,
+strategies that propose solutions based on available resources and Actions
+that perform instance migrations.
+
+.. _placement_integration:
+
+Placement
+^^^^^^^^^
+Placement integration allows Watcher to track resource provider inventories
+and usages information, building a in-memory representation of those resources
+that can be used by strategies when calculating new solutions.
+

doc/source/strategies/host_maintenance.rst

@@ -11,10 +11,6 @@ Synopsis
 
     .. watcher-term:: watcher.decision_engine.strategy.strategies.host_maintenance.HostMaintenance
 
-Requirements
-------------
-
-None.
 
 Metrics
 *******
@@ -56,15 +52,29 @@ Configuration
 
 Strategy parameters are:
 
-==================== ====== ====================================
-parameter            type   default Value description
-==================== ====== ====================================
-``maintenance_node`` String The name of the compute node which
-                            need maintenance. Required.
-``backup_node``      String The name of the compute node which
-                            will backup the maintenance node.
-                            Optional.
-==================== ====== ====================================
+========================== ======== ========================== ==========
+parameter                  type     description                required
+========================== ======== ========================== ==========
+``maintenance_node``       String   The name of the            Required
+                                    compute node
+                                    which needs maintenance.
+``backup_node``            String   The name of the compute    Optional
+                                    node which will backup
+                                    the maintenance node.
+``disable_live_migration`` Boolean  False: Active instances    Optional
+                                    will be live migrated.
+                                    True: Active instances
+                                    will be cold migrated
+                                    if cold migration is
+                                    not disabled. Otherwise,
+                                    they will be stopped.
+                                    False by default.
+``disable_cold_migration`` Boolean  False: Inactive instances  Optional
+                                    will be cold migrated.
+                                    True: Inactive instances
+                                    will not be cold migrated.
+                                    False by default.
+========================== ======== ========================== ==========
 
 Efficacy Indicator
 ------------------
@@ -80,13 +90,46 @@ to: https://specs.openstack.org/openstack/watcher-specs/specs/queens/approved/cl
 How to use it ?
 ---------------
 
+Run an audit using Host Maintenance strategy.
+Executing the actions will move the servers from compute01 host
+to a host determined by the Nova scheduler service.
+
+.. code-block:: shell
+
+    $ openstack optimize audit create \
+      -g cluster_maintaining -s host_maintenance \
+      -p maintenance_node=compute01
+
+Run an audit using Host Maintenance strategy with a backup node specified.
+Executing the actions will move the servers from compute01 host
+to compute02 host.
+
 .. code-block:: shell
 
     $ openstack optimize audit create \
       -g cluster_maintaining -s host_maintenance \
       -p maintenance_node=compute01 \
-      -p backup_node=compute02 \
-      --auto-trigger
+      -p backup_node=compute02
+
+Run an audit using Host Maintenance strategy with migration disabled.
+This will only stop active instances on compute01, useful for maintenance
+scenarios where operators do not want to migrate workloads to other hosts.
+
+.. code-block:: shell
+
+    $ openstack optimize audit create \
+      -g cluster_maintaining -s host_maintenance \
+      -p maintenance_node=compute01 \
+      -p disable_live_migration=True \
+      -p disable_cold_migration=True
+
+Note that after executing this strategy, the *maintenance_node* will be
+marked as disabled, with the reason set to ``watcher_maintaining``.
+To enable the node again:
+
+.. code-block:: shell
+
+   $ openstack compute service set --enable compute01
 
 External Links
 --------------

doc/source/strategies/index.rst

@@ -6,3 +6,67 @@ Strategies
    :maxdepth: 1
 
    ./*
+
+Strategies status matrix
+------------------------
+
+    .. list-table::
+       :widths: 20 20 20 20
+       :header-rows: 1
+
+       * - Strategy Name
+         - Status
+         - Testing
+         - Can Be Triggered from Horizon (UI)
+       * - :doc:`actuation`
+         - Experimental
+         - Unit, Integration
+         - No
+       * - :doc:`basic-server-consolidation`
+         - Experimental
+         - Missing
+         - Yes, with default values
+       * - :doc:`host_maintenance`
+         - Supported
+         - Unit, Integration
+         - No (requires parameters)
+       * - :doc:`node_resource_consolidation`
+         - Supported
+         - Unit, Integration
+         - Yes, with default values
+       * - :doc:`noisy_neighbor`
+         - Deprecated
+         - Unit
+         - N/A
+       * - :doc:`outlet_temp_control`
+         - Experimental
+         - Unit
+         - Yes, with default values
+       * - :doc:`saving_energy`
+         - Experimental
+         - Unit
+         - Yes, with default values
+       * - :doc:`storage_capacity_balance`
+         - Experimental
+         - Unit
+         - Yes, with default values
+       * - :doc:`uniform_airflow`
+         - Experimental
+         - Unit
+         - Yes, with default values
+       * - :doc:`vm_workload_consolidation`
+         - Supported
+         - Unit, Integration
+         - Yes, with default values
+       * - :doc:`workload-stabilization`
+         - Experimental
+         - Missing
+         - Yes, with default values
+       * - :doc:`workload_balance`
+         - Supported
+         - Unit, Integration
+         - Yes, with default values
+       * - :doc:`zone_migration`
+         - Supported (Instance migrations), Experimental (Volume migration)
+         - Unit, Some Integration
+         - No

doc/source/strategies/strategy-template.rst

@@ -35,6 +35,11 @@ power                   ceilometer_  kwapi_  one point every 60s
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 .. _monasca: https://github.com/openstack/monasca-agent/blob/master/docs/Libvirt.md
+
+.. note::
+   The Monasca datasource is deprecated for removal and optional. If a strategy requires Monasca metrics,
+   ensure the Monasca optional extra is installed: ``pip install watcher[monasca]``.
+
 .. _kwapi: https://kwapi.readthedocs.io/en/latest/index.html
 
 

doc/source/strategies/workload-stabilization.rst

@@ -1,6 +1,6 @@
-=============================================
-Watcher Overload standard deviation algorithm
-=============================================
+===============================
+Workload Stabilization Strategy
+===============================
 
 Synopsis
 --------
@@ -19,20 +19,20 @@ Metrics
 
 The *workload_stabilization* strategy requires the following metrics:
 
-============================ ============ ======= =============================
-metric                       service name plugins comment
-============================ ============ ======= =============================
-``compute.node.cpu.percent`` ceilometer_  none    need to set the
-                                                  ``compute_monitors`` option
-                                                  to ``cpu.virt_driver`` in the
-                                                  nova.conf.
-``hardware.memory.used``     ceilometer_  SNMP_
-``cpu``                      ceilometer_  none
-``instance_ram_usage``       ceilometer_  none
-============================ ============ ======= =============================
-
-.. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
-.. _SNMP: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#snmp-based-meters
+============================ ==================================================
+metric                       description
+============================ ==================================================
+``instance_ram_usage``       ram memory usage in an instance as float in
+                             megabytes
+``instance_cpu_usage``       cpu usage in an instance as float ranging between
+                             0 and 100 representing the total cpu usage as
+                             percentage
+``host_ram_usage``           ram memory usage in a compute node as float in
+                             megabytes
+``host_cpu_usage``           cpu usage in a compute node as float ranging
+                             between 0 and 100 representing the total cpu
+                             usage as percentage
+============================ ==================================================
 
 Cluster data model
 ******************
@@ -68,23 +68,49 @@ Configuration
 
 Strategy parameters are:
 
-==================== ====== ===================== =============================
-parameter            type   default Value         description
-==================== ====== ===================== =============================
-``metrics``          array  |metrics|             Metrics used as rates of
+====================== ====== =================== =============================
+parameter              type   default Value       description
+====================== ====== =================== =============================
+``metrics``            array  |metrics|           Metrics used as rates of
                                                   cluster loads.
-``thresholds``       object |thresholds|          Dict where key is a metric
+``thresholds``         object |thresholds|        Dict where key is a metric
                                                   and value is a trigger value.
-
-``weights``          object |weights|             These weights used to
+                                                  The strategy will only will
+                                                  look for an action plan when
+                                                  the standard deviation for
+                                                  the usage of one of the
+                                                  resources included in the
+                                                  metrics, taken as a
+                                                  normalized usage between
+                                                  0 and 1 among the hosts is
+                                                  higher than the threshold.
+                                                  The value of a perfectly
+                                                  balanced cluster for the
+                                                  standard deviation would be
+                                                  0, while in a totally
+                                                  unbalanced one would be 0.5,
+                                                  which should be the maximum
+                                                  value.
+``weights``            object   |weights|         These weights are used to
                                                   calculate common standard
-                                                  deviation. Name of weight
-                                                  contains meter name and
-                                                  _weight suffix.
-``instance_metrics`` object |instance_metrics|    Mapping to get hardware
-                                                  statistics using instance
-                                                  metrics.
-``host_choice``      string retry                 Method of host's choice.
+                                                  deviation when optimizing
+                                                  the resources usage.
+                                                  Name of weight contains meter
+                                                  name and _weight suffix.
+                                                  Higher values imply the
+                                                  metric will be prioritized
+                                                  when calculating an optimal
+                                                  resulting cluster
+                                                  distribution.
+``instance_metrics``   object |instance_metrics|  This parameter represents
+                                                  the compute node metrics
+                                                  representing compute resource
+                                                  usage for the instances
+                                                  resource indicated in the
+                                                  metrics parameter.
+``host_choice``        string retry               Method of host’s choice when
+                                                  analyzing destination for
+                                                  instances.
                                                   There are cycle, retry and
                                                   fullsearch methods. Cycle
                                                   will iterate hosts in cycle.
@@ -93,32 +119,49 @@ parameter            type   default Value         description
                                                   retry_count option).
                                                   Fullsearch will return each
                                                   host from list.
-``retry_count``      number 1                     Count of random returned
+``retry_count``        number 1                   Count of random returned
                                                   hosts.
-``periods``          object |periods|             These periods are used to get
-                                                  statistic aggregation for
-                                                  instance and host metrics.
-                                                  The period is simply a
-                                                  repeating interval of time
-                                                  into which the samples are
-                                                  grouped for aggregation.
-                                                  Watcher uses only the last
-                                                  period of all received ones.
-==================== ====== ===================== =============================
+``periods``            object |periods|           Time, in seconds, to get
+                                                  statistical values for
+                                                  resources usage for instance
+                                                  and host metrics.
+                                                  Watcher will use the last
+                                                  period to calculate resource
+                                                  usage.
+``granularity``        number 300                 NOT RECOMMENDED TO MODIFY:
+                                                  The time between two measures
+                                                  in an aggregated timeseries
+                                                  of a metric.
+``aggregation_method`` object |aggn_method|       NOT RECOMMENDED TO MODIFY:
+                                                  Function used to aggregate
+                                                  multiple measures into an
+                                                  aggregated value.
+====================== ====== =================== =============================
 
 .. |metrics| replace:: ["instance_cpu_usage", "instance_ram_usage"]
 .. |thresholds| replace:: {"instance_cpu_usage": 0.2, "instance_ram_usage": 0.2}
 .. |weights| replace:: {"instance_cpu_usage_weight": 1.0, "instance_ram_usage_weight": 1.0}
-.. |instance_metrics| replace:: {"instance_cpu_usage": "compute.node.cpu.percent", "instance_ram_usage": "hardware.memory.used"}
+.. |instance_metrics| replace:: {"instance_cpu_usage": "host_cpu_usage", "instance_ram_usage": "host_ram_usage"}
 .. |periods| replace:: {"instance": 720, "node": 600}
+.. |aggn_method| replace:: {"instance": 'mean', "compute_node": 'mean'}
+
 
 Efficacy Indicator
 ------------------
 
+Global efficacy indicator:
+
 .. watcher-func::
   :format: literal_block
 
-  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+  watcher.decision_engine.goal.efficacy.specs.WorkloadBalancing.get_global_efficacy_indicator
+
+Other efficacy indicators of the goal are:
+
+- ``instance_migrations_count``: The number of VM migrations to be performed
+- ``instances_count``: The total number of audited instances in strategy
+- ``standard_deviation_after_audit``: The value of resulted standard deviation
+- ``standard_deviation_before_audit``: The value of original standard deviation
 
 Algorithm
 ---------
@@ -141,4 +184,4 @@ How to use it ?
 External Links
 --------------
 
-- `Watcher Overload standard deviation algorithm spec <https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html>`_
+None

doc/source/strategies/workload_balance.rst

@@ -11,25 +11,35 @@ Synopsis
 
     .. watcher-term:: watcher.decision_engine.strategy.strategies.workload_balance.WorkloadBalance
 
-Requirements
-------------
-
-None.
-
 Metrics
 *******
 
-The *workload_balance* strategy requires the following metrics:
+The ``workload_balance`` strategy requires the following metrics:
 
-======================= ============ ======= =========================
-metric                  service name plugins comment
-======================= ============ ======= =========================
-``cpu``                 ceilometer_  none
-``memory.resident``     ceilometer_  none
-======================= ============ ======= =========================
+======================= ============ ======= =========== ======================
+metric                  service name plugins unit        comment
+======================= ============ ======= =========== ======================
+``cpu``                 ceilometer_  none    percentage  CPU of the instance.
+                                                         Used to calculate the
+                                                         threshold
+``memory.resident``     ceilometer_  none    MB          RAM of the instance.
+                                                         Used to calculate the
+                                                         threshold
+======================= ============ ======= =========== ======================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 
+.. note::
+   * The parameters above reference the instance CPU or RAM usage, but
+     the threshold calculation is based of the CPU/RAM usage on the
+     hypervisor.
+   * The RAM usage can be calculated based on the RAM consumed by the instance,
+     and the available RAM on the hypervisor.
+   * The CPU percentage calculation relies on the CPU load, but also on the
+     number of CPUs on the hypervisor.
+   * The host memory metric is calculated by summing the RAM usage of each
+     instance on the host. This measure is close to the real usage, but is
+     not the exact usage on the host.
 
 Cluster data model
 ******************
@@ -64,16 +74,28 @@ Configuration
 
 Strategy parameters are:
 
-============== ====== ==================== ====================================
-parameter      type   default Value        description
-============== ====== ==================== ====================================
-``metrics``    String 'instance_cpu_usage' Workload balance base on cpu or ram
-                                           utilization. Choices:
-                                           ['instance_cpu_usage',
-                                           'instance_ram_usage']
-``threshold``  Number 25.0                 Workload threshold for migration
-``period``     Number 300                  Aggregate time period of ceilometer
-============== ====== ==================== ====================================
+================ ====== ==================== ==================================
+parameter        type   default value        description
+================ ====== ==================== ==================================
+``metrics``      String instance_cpu_usage   Workload balance base on cpu or
+                                             ram utilization. Choices:
+                                             ['instance_cpu_usage',
+                                             'instance_ram_usage']
+``threshold``    Number 25.0                 Workload threshold for migration.
+                                             Used for both the source and the
+                                             destination calculations.
+                                             Threshold is always a percentage.
+``period``       Number 300                  Aggregate time period of
+                                             ceilometer
+``granularity``  Number 300                  The time between two measures in
+                                             an aggregated timeseries of a
+                                             metric.
+                                             This parameter is only used
+                                             with the Gnocchi data source,
+                                             and it must match to any of the
+                                             valid archive policies for the
+                                             metric.
+================ ====== ==================== ==================================
 
 Efficacy Indicator
 ------------------
@@ -89,14 +111,36 @@ to: https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented
 How to use it ?
 ---------------
 
+Create an audit template using the Workload Balancing strategy.
+
 .. code-block:: shell
 
     $ openstack optimize audittemplate create \
       at1 workload_balancing --strategy workload_balance
 
+Run an audit using the Workload Balance strategy. The result of
+the audit should be an action plan to move VMs from any host
+where the CPU usage is over the threshold of 26%, to a host
+where the utilization of CPU is under the threshold.
+The measurements of CPU utilization are taken from the configured
+datasouce plugin with an aggregate period of 310.
+
+.. code-block:: shell
+
     $ openstack optimize audit create -a at1 -p threshold=26.0 \
             -p period=310 -p metrics=instance_cpu_usage
 
+Run an audit using the Workload Balance strategy to
+obtain a plan to balance VMs over hosts with a threshold of 20%.
+In this case, the stipulation of the CPU utilization metric
+measurement is a combination of period and granularity.
+
+.. code-block:: shell
+
+    $ openstack optimize audit create -a at1 \
+           -p granularity=30 -p threshold=20 -p period=300 \
+           -p metrics=instance_cpu_usage --auto-trigger
+
 External Links
 --------------
 

doc/source/strategies/zone_migration.rst

@@ -11,6 +11,13 @@ Synopsis
 
     .. watcher-term:: watcher.decision_engine.strategy.strategies.zone_migration.ZoneMigration
 
+.. note::
+   The term ``Zone`` in the strategy name is not a reference to
+   `Openstack availability zones <https://docs.openstack.org/nova/latest/admin/availability-zones.html>`_
+   but rather a user-defined set of Compute nodes and storage pools.
+   Currently, migrations across actual availability zones is not fully tested
+   and might not work in all cluster configurations.
+
 Requirements
 ------------
 
@@ -59,66 +66,83 @@ Configuration
 
 Strategy parameters are:
 
-======================== ======== ============= ==============================
-parameter                type     default Value description
-======================== ======== ============= ==============================
-``compute_nodes``        array    None          Compute nodes to migrate.
-``storage_pools``        array    None          Storage pools to migrate.
-``parallel_total``       integer  6             The number of actions to be
-                                                run in parallel in total.
-``parallel_per_node``    integer  2             The number of actions to be
-                                                run in parallel per compute
-                                                node.
-``parallel_per_pool``    integer  2             The number of actions to be
-                                                run in parallel per storage
-                                                pool.
-``priority``             object   None          List prioritizes instances
-                                                and volumes.
-``with_attached_volume`` boolean  False         False: Instances will migrate
-                                                after all volumes migrate.
-                                                True: An instance will migrate
-                                                after the attached volumes
-                                                migrate.
-======================== ======== ============= ==============================
+======================== ======== ======== ========= ==========================
+parameter                type     default  required  description
+======================== ======== ======== ========= ==========================
+``compute_nodes``        array    None     Optional  Compute nodes to migrate.
+``storage_pools``        array    None     Optional  Storage pools to migrate.
+``parallel_total``       integer  6        Optional  The number of actions to
+                                                     be run in parallel in
+                                                     total.
+``parallel_per_node``    integer  2        Optional  The number of actions to
+                                                     be run in parallel per
+                                                     compute node in one
+                                                     action plan.
+``parallel_per_pool``    integer  2        Optional  The number of actions to
+                                                     be run in parallel per
+                                                     storage pool.
+``priority``             object   None     Optional  List prioritizes instances
+                                                     and volumes.
+``with_attached_volume`` boolean  False    Optional  False: Instances will
+                                                     migrate after all volumes
+                                                     migrate.
+                                                     True: An instance will
+                                                     migrate after the
+                                                     attached volumes migrate.
+======================== ======== ======== ========= ==========================
+
+.. note::
+   * All parameters in the table above have defaults and therefore the
+     user can create an audit without specifying a value. However,
+     if **only** defaults parameters are used, there will be nothing
+     actionable for the audit.
+   * ``parallel_*`` parameters are not in reference to concurrency,
+     but rather on limiting the amount of actions to be added to the action
+     plan
+   * ``compute_nodes``, ``storage_pools``, and ``priority`` are optional
+     parameters, however, if they are passed they **require** the parameters
+     in the tables below:
 
 The elements of compute_nodes array are:
 
-============= ======= =============== =============================
-parameter     type    default Value   description
-============= ======= =============== =============================
-``src_node``  string    None          Compute node from which
-                                      instances migrate(mandatory).
-``dst_node``  string    None          Compute node to which
-                                      instances migrate.
-============= ======= =============== =============================
+============= ======= ======== ========= ========================
+parameter     type    default  required  description
+============= ======= ======== ========= ========================
+``src_node``  string  None     Required  Compute node from which
+                                         instances migrate.
+``dst_node``  string  None     Optional  Compute node to which
+                                         instances migrate.
+                                         If omitted, nova will
+                                         choose the destination
+                                         node automatically.
+============= ======= ======== ========= ========================
 
 The elements of storage_pools array are:
 
-============= ======= =============== ==============================
-parameter     type    default Value   description
-============= ======= =============== ==============================
-``src_pool``  string    None          Storage pool from which
-                                      volumes migrate(mandatory).
-``dst_pool``  string    None          Storage pool to which
-                                      volumes migrate.
-``src_type``  string    None          Source volume type(mandatory).
-``dst_type``  string    None          Destination volume type
-                                      (mandatory).
-============= ======= =============== ==============================
+============= ======= ======== ========= ========================
+parameter     type    default  required  description
+============= ======= ======== ========= ========================
+``src_pool``  string  None     Required  Storage pool from which
+                                         volumes migrate.
+``dst_pool``  string  None     Optional  Storage pool to which
+                                         volumes migrate.
+``src_type``  string  None     Required  Source volume type.
+``dst_type``  string  None     Required  Destination volume type
+============= ======= ======== ========= ========================
 
 The elements of priority object are:
 
-================ ======= =============== ======================
-parameter        type    default Value   description
-================ ======= =============== ======================
-``project``      array   None            Project names.
-``compute_node`` array   None            Compute node names.
-``storage_pool`` array   None            Storage pool names.
-``compute``      enum    None            Instance attributes.
-                                         |compute|
-``storage``      enum    None            Volume attributes.
-                                         |storage|
-================ ======= =============== ======================
+================ ======= ======== ========= =====================
+parameter        type    default  Required  description
+================ ======= ======== ========= =====================
+``project``      array   None     Optional  Project names.
+``compute_node`` array   None     Optional  Compute node names.
+``storage_pool`` array   None     Optional  Storage pool names.
+``compute``      enum    None     Optional  Instance attributes.
+                                            |compute|
+``storage``      enum    None     Optional  Volume attributes.
+                                            |storage|
+================ ======= ======== ========= =====================
 
 .. |compute| replace:: ["vcpu_num", "mem_size", "disk_size", "created_at"]
 .. |storage| replace:: ["size", "created_at"]
@@ -126,11 +150,26 @@ parameter        type    default Value   description
 Efficacy Indicator
 ------------------
 
+The efficacy indicators for action plans built from the command line
+are:
+
 .. watcher-func::
   :format: literal_block
 
   watcher.decision_engine.goal.efficacy.specs.HardwareMaintenance.get_global_efficacy_indicator
 
+In **Horizon**, these indictors are shown with alternative text.
+
+* ``live_migrate_instance_count`` is shown as
+  ``The number of instances actually live migrated`` in Horizon
+* ``planned_live_migrate_instance_count`` is  shown as
+  ``The number of instances planned to live migrate`` in Horizon
+* ``planned_live_migration_instance_count`` refers to the instances planned
+  to live migrate in the action plan.
+* ``live_migrate_instance_count`` tracks all the instances that could be
+  migrated according to the audit input.
+
+
 Algorithm
 ---------
 
@@ -148,6 +187,22 @@ How to use it ?
     $ openstack optimize audit create -a at1 \
       -p compute_nodes='[{"src_node": "s01", "dst_node": "d01"}]'
 
+.. note::
+   * Currently, the strategy will not generate both volume migration and
+     instance migrations in the same audit. If both are requested,
+     only volume migrations will be included in the action plan.
+   * The Cinder model collector is not enabled by default.
+     If the Cinder model collector is not enabled while deploying Watcher,
+     the model will become outdated and cause errors eventually.
+     See the `Configuration option to enable the storage collector <https://docs.openstack.org/watcher/latest/configuration/watcher.html#collector.collector_plugins>`_ documentation.
+
+Support caveats
+---------------
+
+This strategy offers the option to perform both Instance migrations and
+Volume migrations. Currently, Instance migrations are ready for production
+use while Volume migrations remain experimental.
+
 External Links
 --------------
 

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["pbr>=6.0.0", "setuptools>=64.0.0"]
+build-backend = "pbr.build"

releasenotes/notes/2025.2-prelude-a9f4c7b2e8d15692.yaml

@@ -0,0 +1,23 @@
+---
+prelude: |
+  The ``OpenStack 2025.2`` (``Watcher 15.0.0``) release delivers stronger security,
+  granular operational control, and comprehensive monitoring capabilities for cloud
+  optimization. This release strengthens the foundation for reliable, large-scale
+  cloud operations while giving administrators complete flexibility in
+  managing optimization workflows.
+
+  Cloud operators gain secure and reliable volume migration processes that eliminate
+  data loss scenarios and ensure tenant isolation. The Host Maintenance strategy
+  now provides granular control over migration behavior, including the ability to
+  disable live or cold migration and safely stop instances when migration
+  cannot proceed.
+
+  The new ``Aetos data source`` adds secure, role-based access to Prometheus metrics
+  through Keystone authentication. This enables multi-tenant monitoring while
+  maintaining access controls across your cloud infrastructure.
+
+  Administrators can now exercise precise control over optimization workflows by
+  manually skipping actions or allowing Watcher to automatically skip actions
+  based on detected conditions. Custom status messages document why administrators
+  or Watcher took specific actions, improving operational visibility and
+  troubleshooting.

releasenotes/notes/add-fail-options-to-nop-f44f506dc732f2a1.yaml

@@ -0,0 +1,14 @@
+---
+features:
+  - |
+    Three new parameters have been added to the ``nop`` action:
+
+    * ``fail_pre_condition``: When setting it to `true` the action
+      fails on the pre_condition execution.
+
+    * ``fail_execute``: When setting it to `true` the action fails
+      on the execute step.
+
+    * ``fail_post_condition``: When setting it to `true` the action
+      fails on the post_condition execution.
+

releasenotes/notes/add-wsgi-module-support-597f479e31979270.yaml

@@ -0,0 +1,30 @@
+---
+features:
+  - |
+    A new module, ``watcher.wsgi``, has been added as a place to gather WSGI
+    ``application`` objects. This is intended to ease deployment by providing
+    a consistent location for these objects. For example, if using uWSGI then
+    instead of:
+
+    .. code-block:: ini
+
+       [uwsgi]
+        wsgi-file = /bin/watcher-api-wsgi
+
+    You can now use:
+
+    .. code-block:: ini
+
+        [uwsgi]
+        module = watcher.wsgi.api:application
+
+    This also simplifies deployment with other WSGI servers that expect module
+    paths such as gunicorn.
+deprecations:
+  - |
+    The watcher-api-wsgi console script is deprecated for removal
+    in a future release. This artifact is generated using a setup-tools
+    extension that is provide by PBR which is also deprecated.
+    due to the changes in python packaging this custom extensions
+    is planned to be removed form all OpenStack projects in a future
+    PBR release in favor of module based wsgi applications entry points.

releasenotes/notes/aetos-datasource-60e50a2338c64c08.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    A new Aetos data source is added. This allows the watcher decision
+    engine to collect metrics through an Aetos reverse proxy server which
+    provides multi-tenant aware access to Prometheus with Keystone
+    authentication and role-based access control. The Aetos datasource
+    uses Keystone service discovery to automatically locate the Aetos
+    endpoint and provides enhanced security compared to direct Prometheus
+    access. For more information about the Aetos data source, including
+    configuration options see
+    https://docs.openstack.org/watcher/latest/datasources/aetos.html
+

releasenotes/notes/blueprint-add-skip-actions-4a5a997dc1133f13.yaml

@@ -0,0 +1,20 @@
+---
+features:
+  - |
+    A new state ``SKIPPED`` has been added to the Actions. Actions can reach this state
+    in two situations:
+
+    * Watcher detects a specific pre-defined condition in the `pre_condition` phase.
+
+    * An admin sets the state to SKIPPED using a call to the new Patch API `/actions/{action_id}`
+      before the action plan is started.
+
+    An action in ``SKIPPED`` state will not be executed by Watcher as part of an ActionPlan
+    run.
+
+    Additionally, a new field ``status_message`` has been added to Audits, ActionPlans and
+    Actions which will be used to provide additional details about the state of an object.
+
+    All these changes have been introduced in a new Watcher ``API microversion 1.5``.
+
+    For additional information, see the API reference.

releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml

@@ -0,0 +1,14 @@
+---
+features:
+  - |
+    The compute model was extended with additional server attributes to provide
+    more detailed information about compute instances. These additions will
+    enable strategies to make more precise decisions by considering more server
+    placement constraints. The new attributes are ``flavor extra specs`` and
+    ``pinned availability zone``. Each new attribute depends on a minimal
+    microversion to be supported in nova and configured in the watcher
+    configuration, at ``nova_client`` section. Please refer to the nova api-ref
+    documentation for more details on which microversion is required:
+    https://docs.openstack.org/api-ref/compute/
+    A new configuration option was added to allow the user to enable or disable
+    the extended attributes collection, which is disabled by default.

releasenotes/notes/bug-2103451-fixes-prometheus-queries-with-multiple-target-0e65d20711d1abe2.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    When using prometheus datasource and more that one target has the same value
+    for the ``fqdn_label``, the driver used the wrong instance label to query for host
+    metrics. The ``instance`` label is no longer used in the queries but the ``fqdn_label``
+    which identifies all the metrics for a specific compute node.
+    see Bug 2103451: https://bugs.launchpad.net/watcher/+bug/2103451 for more info.

releasenotes/notes/bug-2109494-e5bf401767fa6cd6.yaml

@@ -0,0 +1,11 @@
+---
+upgrade:
+  - |
+    The default value of ``[keystone_client] interface`` has been changed from
+    ``admin`` to ``public``.
+fixes:
+  - |
+    When trying to do volume migration using the zone migration strategy, the
+    keystone service is reached, by default through the admin endpoint.
+    The default value of ``[keystone_client] interface`` has been fixed.
+    see Bug https://bugs.launchpad.net/watcher/+bug/2109494 for more info.

releasenotes/notes/bug-2110947.yaml

@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+     Previously, when users attempted to create a new audit without providing
+     a name and a goal or an audit template, the API returned error 500 and an
+     incorrect error message was displayed.
+
+     Now, Watcher displays a helpful message and returns HTTP error 400.
+
+     For more info see: https://bugs.launchpad.net/watcher/+bug/2110947

releasenotes/notes/bug-2112100-c1e56173cd29a35e.yaml

@@ -0,0 +1,11 @@
+---
+fixes:
+  - |
+    Currently, when Watcher applies a `volume_migrate` action with value
+    `retype` for the `migratione_type`, it can wrongly report the result of
+    the action when the retype does not trigger a volume migration.
+
+    This patch fixes the logic to validate the resulting state of the action
+    and reports it correctly.
+
+    For more details: https://bugs.launchpad.net/watcher/+bug/2112100

releasenotes/notes/bug-2112187-763bae283e0b736d.yaml

@@ -0,0 +1,47 @@
+---
+security:
+  - |
+    Watchers no longer forges requests on behalf of a tenant when
+    swapping volumes. Prior to this release watcher had 2 implementations
+    of moving a volume, it could use cinders volume migrate api or its own
+    internal implementation that directly calls nova volume attachment update
+    api. The former is safe and the recommend way to move volumes between
+    cinder storage backend the internal implementation was insecure, fragile
+    due to a lack of error handling and capable of deleting user data.
+
+    Insecure: the internal volume migration operation created a new keystone
+    user with a weak name and password and added it to the tenants project
+    with the admin role. It then used that user to forge request on behalf
+    of the tenant with admin right to swap the volume. if the applier was
+    restarted during the execution of this operation it would never be cleaned
+    up.
+
+    Fragile: the error handling was minimal, the swap volume api is async
+    so watcher has to poll for completion, there was no support to resume
+    that if interrupted of the time out was exceeded.
+
+    Data-loss: while the internal polling logic returned success or failure
+    watcher did not check the result, once the function returned it
+    unconditionally deleted the source volume. For larger volumes this
+    could result in irretrievable data loss.
+
+    Finally if a volume was swapped using the internal workflow it put
+    the nova instance in an out of sync state. If the VM was live migrated
+    after the swap volume completed successfully prior to a hard reboot
+    then the migration would fail or succeed and break tenant isolation.
+
+    see: https://bugs.launchpad.net/nova/+bug/2112187 for details.
+fixes:
+  - |
+    All code related to creating keystone user and granting roles has been
+    removed. The internal swap volume implementation has been removed and
+    replaced by cinders volume migrate api. Note as part of this change
+    Watcher will no longer attempt volume migrations or retypes if the
+    instance is in the `Verify Resize` task state. This resolves several
+    issues related to volume migration in the zone migration and
+    Storage capacity balance strategies. While efforts have been made
+    to maintain backward compatibility these changes are required to
+    address a security weakness in watcher's prior approach.
+
+    see: https://bugs.launchpad.net/nova/+bug/2112187 for more context.
+

releasenotes/notes/bug-2113776-4bd314fb46623fbc.yaml

@@ -0,0 +1,14 @@
+---
+fixes:
+  - |
+    When running an audit with the `workload_stabilization` strategy with
+    `instance_ram_usage` metric in a deployment with prometheus datasource,
+    the host metric for the ram usage was wrongly reported with the incorrect
+    unit which lead to incorrect standard deviation and action plans due to the
+    application of the wrong scale factor in the algorithm.
+
+    The host ram usage metric is now properly reported in KB when using a
+    prometheus datasource and the strategy `workload_stabilization` calculates
+    the standard deviation properly.
+
+    For more details: https://launchpad.net/bugs/2113776

releasenotes/notes/bug-2117726-fix-model-list-api-ref-30cc7ed1c85c0d0e.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fix API reference documentation for ``GET /infra-optim/v1/data_model``,
+    to include all missing fields from the response body. Please see
+    `Bug 2117726 <https://bugs.launchpad.net/watcher/+bug/2117726>`_ for
+    more details.

releasenotes/notes/bug-2120586-fix-nova-microversion-check-9022a378b75d046f.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixed nova client microversion comparison in enable and disable compute
+    service methods. The code was incorrectly comparing API versions, which
+    caused failures for microversions greater than 2.99. For more details,
+    see the bug report: https://bugs.launchpad.net/watcher/+bug/2120586

releasenotes/notes/bug-2121601-allow-status-message-updates-in-skipped-state-a8b4c5d7e9f2g3h1.yaml

@@ -0,0 +1,11 @@
+---
+fixes:
+  - |
+    Fixed action status_message update restrictions to allow updates when
+    action is in SKIPPED state. Previously, users could only update the
+    status_message when initially changing the action state to SKIPPED.
+    Now users can update the status_message field at any time while the
+    action remains in SKIPPED state, enabling them to fix typos, provide
+    more detailed explanations, or expand on reasons that were initially
+    omitted. For more details, see the bug report:
+    https://bugs.launchpad.net/watcher/+bug/2121601

releasenotes/notes/decision-engine-threading-mode-26fc8066dcd499a2.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    The Decision Engine service now supports running with ``native threading``
+    mode enabled as opposed to the use of the Eventlet library.
+    Note that the use of ``native threading`` is still ``experimental``,
+    and is disabled by default. It should not be used in production. To
+    switch from Eventlet to native threading mode, the environment variable
+    ``OS_WATCHER_DISABLE_EVENTLET_PATCHING=true`` needs to be added to
+    the decision engine service configuration. For more information,
+    please check `eventlet removal
+    <https://wiki.openstack.org/wiki/Eventlet-removal>`__ documentation.

releasenotes/notes/deprecate-noisy-neighbor-strat-7da910837ae8fa80.yaml

@@ -0,0 +1,6 @@
+---
+deprecations:
+  - |
+    Noisy Neighbor strategy is deprecated and will be removed in a future release.
+    This strategy relies on Last Level Cache metrics that are not available in Nova
+    since `Victoria release <https://docs.openstack.org/releasenotes/nova/victoria.html>`_.

releasenotes/notes/donot-run-host-migration-strategy-on-disabled-hosts-24084a22d4c8f914.yaml

@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+    Host maintenance strategy should migrate servers based on backup node if specified
+    or rely on nova scheduler. It was enabling disabled hosts with watcher_disabled
+    reason and migrating servers to those nodes. It can impact customer workload. Compute
+    nodes were disabled for a reason.
+
+    Host maintenance strategy is fixed now to support migrating servers only on backup
+    node or rely on nova scheduler if no backup node is provided.

releasenotes/notes/drop-operation-not-permitted-exception-14e49b35a3ca00d1.yaml

@@ -0,0 +1,6 @@
+---
+other:
+  - |
+    Removed unused ``OperationNotPermitted`` exception that was dead code
+    since the initial import of the Watcher codebase. This exception
+    provides the appropriate 400 Bad Request response behavior.

releasenotes/notes/drop-patch-delete-post-action-api-37fe4ce5be6500db.yaml

@@ -0,0 +1,6 @@
+---
+other:
+  - |
+    The `DELETE`, `POST` and `Patch` REST methods for the `action` APIs
+    are forbidden and not implemented. They are now removed from
+    the API controller.

releasenotes/notes/drop-py39-8a9c99678b3e8eeb.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Watcher now requires python 3.10 or newer.
+    The last release of watcher to support ``3.9`` was ``2025.1``.
+    Please ensure you have a supported python version before upgrading.

releasenotes/notes/drop-python-dateutil-dependency-2118404-f5a8b2c1e9d4a6b3.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Removed the ``python-dateutil`` dependency from Watcher to reduce the
+    number of external dependencies and improve maintainability.

releasenotes/notes/experimental-integrations-490d4cc32444288d.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Glance, Ironic, MAAS, and Neutron integrations with Watcher are now marked
+    as Experimental and may be deprecated in a future release. These
+    integrations have not been tested recently and may not be fully stable.