Don't throw exception when missing metrics

When querying data from datasource, it's possible to miss some data.
In this case if we throw an exception, Audit will failed because of
the exception. We should remove the exception and give the decision
to the strategy.

Change-Id: I1b0e6b78b3bba4df9ba16e093b3910aab1de922e
Closes-Bug: #1847434
can not cherry picke from master because of code refactoring
(cherry picked from commit 306224f70c)

.gitignore

@@ -26,7 +26,6 @@ pip-log.txt
 .stestr/
 .venv
 .idea
-.testrepository/
 
 # Translations
 *.mo
@@ -75,6 +74,3 @@ releasenotes/build
 
 # Autogenerated sample config file
 etc/watcher/watcher.conf.sample
-
-# Atom
-.remote-sync.json

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
+defaultbranch=stable/rocky

.zuul.yaml

@@ -1,114 +1,76 @@
 - project:
     templates:
+      - openstack-python-jobs
+      - openstack-python35-jobs
+      - publish-openstack-sphinx-docs
       - check-requirements
-      - openstack-cover-jobs
-      - openstack-lower-constraints-jobs
-      - openstack-python3-victoria-jobs
-      - publish-openstack-docs-pti
-      - release-notes-jobs-python3
+      - release-notes-jobs
     check:
       jobs:
         - watcher-tempest-functional
-        - watcher-grenade
-        - watcher-tempest-strategies
+        - watcher-tempest-dummy_optim
         - watcher-tempest-actuator
+        - watcher-tempest-basic_optim
+        - watcher-tempest-workload_balancing
         - watcherclient-tempest-functional
-        - watcher-tls-test
-        - watcher-tempest-functional-ipv6-only
+        - watcher-tempest-zone_migration
+        - openstack-tox-lower-constraints
     gate:
       queue: watcher
       jobs:
         - watcher-tempest-functional
-        - watcher-tempest-functional-ipv6-only
+        - openstack-tox-lower-constraints
 
 - job:
     name: watcher-tempest-dummy_optim
     parent: watcher-tempest-multinode
+    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_dummy_optim
 
 - job:
     name: watcher-tempest-actuator
     parent: watcher-tempest-multinode
+    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_actuator
 
 - job:
     name: watcher-tempest-basic_optim
     parent: watcher-tempest-multinode
+    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_basic_optim
 
-- job:
-    name: watcher-tempest-vm_workload_consolidation
-    parent: watcher-tempest-multinode
-    vars:
-      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation
-      devstack_local_conf:
-        test-config:
-          $WATCHER_CONFIG:
-            watcher_strategies.vm_workload_consolidation:
-              datasource: ceilometer
-
 - job:
     name: watcher-tempest-workload_balancing
     parent: watcher-tempest-multinode
+    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_workload_balancing
 
 - job:
     name: watcher-tempest-zone_migration
     parent: watcher-tempest-multinode
+    voting: false
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_zone_migration
 
-- job:
-    name: watcher-tempest-host_maintenance
-    parent: watcher-tempest-multinode
-    vars:
-      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance
-
-- job:
-    name: watcher-tempest-storage_balance
-    parent: watcher-tempest-multinode
-    vars:
-      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_storage_balance
-      devstack_local_conf:
-        test-config:
-          $TEMPEST_CONFIG:
-            volume:
-              backend_names: ['BACKEND_1', 'BACKEND_2']
-            volume-feature-enabled:
-              multi_backend: true
-
-- job:
-    name: watcher-tempest-strategies
-    parent: watcher-tempest-multinode
-    vars:
-      tempest_concurrency: 1
-      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_strategies
-
-- job:
-    name: watcher-tls-test
-    parent: watcher-tempest-multinode
-    group-vars:
-      subnode:
-          devstack_services:
-            tls-proxy: true
-    vars:
-      devstack_services:
-        tls-proxy: true
-
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
-    nodeset: openstack-two-node-bionic
+    nodeset: openstack-two-node
+    pre-run: playbooks/pre.yaml
+    run: playbooks/orchestrate-tempest.yaml
     roles:
       - zuul: openstack/tempest
     group-vars:
       subnode:
         devstack_local_conf:
           post-config:
+            $NOVA_CONF:
+              libvirt:
+                live_migration_uri: qemu+ssh://root@%s/system
             $WATCHER_CONF:
               watcher_cluster_data_model_collectors.compute:
                 period: 120
@@ -134,6 +96,9 @@
     vars:
       devstack_local_conf:
         post-config:
+          $NOVA_CONF:
+            libvirt:
+              live_migration_uri: qemu+ssh://root@%s/system
           $WATCHER_CONF:
             watcher_cluster_data_model_collectors.compute:
               period: 120
@@ -145,12 +110,9 @@
           $TEMPEST_CONFIG:
             compute:
               min_compute_nodes: 2
-              min_microversion: 2.56
             compute-feature-enabled:
               live_migration: true
               block_migration_for_live_migration: true
-            placement:
-              min_microversion: 1.29
       devstack_plugins:
         ceilometer: https://opendev.org/openstack/ceilometer
 
@@ -158,14 +120,15 @@
     name: watcher-tempest-functional
     parent: devstack-tempest
     timeout: 7200
-    required-projects: &base_required_projects
+    required-projects:
       - openstack/ceilometer
+      - openstack/devstack-gate
       - openstack/python-openstackclient
       - openstack/python-watcherclient
       - openstack/watcher
       - openstack/watcher-tempest-plugin
       - openstack/tempest
-    vars: &base_vars
+    vars:
       devstack_plugins:
         watcher: https://opendev.org/openstack/watcher
       devstack_services:
@@ -178,46 +141,24 @@
         s-container: false
         s-object: false
         s-proxy: false
-      tempest_plugins:
-        - watcher-tempest-plugin
+      devstack_localrc:
+        TEMPEST_PLUGINS: /opt/stack/watcher-tempest-plugin
       tempest_test_regex: watcher_tempest_plugin.tests.api
       tox_envlist: all
+      tox_environment:
+        # Do we really need to set this? It's cargo culted
+        PYTHONUNBUFFERED: 'true'
       zuul_copy_output:
         /etc/hosts: logs
 
-- job:
-    name: watcher-tempest-functional-ipv6-only
-    parent: devstack-tempest-ipv6
-    description: |
-      Watcher devstack tempest tests job for IPv6-only deployment
-    required-projects: *base_required_projects
-    vars: *base_vars
-
-- job:
-    name: watcher-grenade
-    parent: grenade
-    required-projects:
-      - openstack/watcher
-      - openstack/python-watcherclient
-      - openstack/watcher-tempest-plugin
-    vars: *base_vars
-    irrelevant-files:
-      - ^(test-|)requirements.txt$
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^watcher/hacking/.*$
-      - ^watcher/tests/.*$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
-
 - job:
     # This job is used in python-watcherclient repo
     name: watcherclient-tempest-functional
     parent: watcher-tempest-functional
+    voting: false
     timeout: 4200
     vars:
       tempest_concurrency: 1
-      tempest_test_regex: watcher_tempest_plugin.tests.client_functional
+      devstack_localrc:
+        TEMPEST_PLUGINS: /opt/stack/python-watcherclient
+      tempest_test_regex: watcherclient.tests.functional

README.rst

@@ -1,6 +1,6 @@
-=======
-Watcher
-=======
+========================
+Team and repository tags
+========================
 
 .. image:: https://governance.openstack.org/tc/badges/watcher.svg
     :target: https://governance.openstack.org/tc/reference/tags/index.html
@@ -13,6 +13,10 @@ Watcher
 
           https://creativecommons.org/licenses/by/3.0/
 
+=======
+Watcher
+=======
+
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
 Watcher provides a robust framework to realize a wide range of cloud
@@ -22,8 +26,7 @@ migration, increased energy efficiency and more!
 
 * Free software: Apache license
 * Wiki: https://wiki.openstack.org/wiki/Watcher
-* Source: https://opendev.org/openstack/watcher
+* Source: https://github.com/openstack/watcher
 * Bugs: https://bugs.launchpad.net/watcher
 * Documentation: https://docs.openstack.org/watcher/latest/
 * Release notes: https://docs.openstack.org/releasenotes/watcher/
-* Design specifications: https://specs.openstack.org/openstack/watcher-specs/

api-ref/source/conf.py

@@ -22,6 +22,9 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+from watcher import version as watcher_version
+
+
 extensions = [
     'openstackdocstheme',
     'os_api_ref',
@@ -43,13 +46,21 @@ project = u'Infrastructure Optimization API Reference'
 copyright = u'2010-present, OpenStack Foundation'
 
 # openstackdocstheme options
-openstackdocs_repo_name = 'openstack/watcher'
-openstackdocs_auto_name = False
-openstackdocs_bug_project = 'watcher'
-openstackdocs_bug_tag = ''
+repository_name = 'openstack/watcher'
+bug_project = 'watcher'
+bug_tag = ''
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The full version, including alpha/beta/rc tags.
+release = watcher_version.version_info.release_string()
+# The short X.Y version.
+version = watcher_version.version_string
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'native'
+pygments_style = 'sphinx'
 
 # -- Options for HTML output --------------------------------------------------
 
@@ -64,6 +75,10 @@ html_theme_options = {
     "sidebar_mode": "toc",
 }
 
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%Y-%m-%d %H:%M'
+
 # -- Options for LaTeX output -------------------------------------------------
 
 # Grouping the document tree into LaTeX files. List of tuples

api-ref/source/index.rst

@@ -6,7 +6,6 @@ Watcher API
 
 .. rest_expand_all::
 
-.. include:: watcher-api-versions.inc
 .. include:: watcher-api-v1-audittemplates.inc
 .. include:: watcher-api-v1-audits.inc
 .. include:: watcher-api-v1-actionplans.inc
@@ -14,6 +13,4 @@ Watcher API
 .. include:: watcher-api-v1-goals.inc
 .. include:: watcher-api-v1-strategies.inc
 .. include:: watcher-api-v1-services.inc
-.. include:: watcher-api-v1-scoring_engines.inc
-.. include:: watcher-api-v1-datamodel.inc
-.. include:: watcher-api-v1-webhooks.inc
+.. include:: watcher-api-v1-scoring_engines.inc

api-ref/source/parameters.yaml

@@ -1,42 +1,3 @@
-# variables in header
-header_version:
-  description: |
-    Specific API microversion used to generate this response.
-  in: header
-  required: true
-  type: string
-openstack-api-max-version:
-  description: |
-    Maximum API microversion supported by this endpoint, eg. "1.1"
-  in: header
-  required: true
-  type: string
-openstack-api-min-version:
-  description: |
-    Minimum API microversion supported by this endpoint, eg. "1.0"
-  in: header
-  required: true
-  type: string
-openstack-api-version:
-  description: >
-    A request SHOULD include this header to indicate to the Watcher API service what
-    version the client supports. The server will transform the response object into
-    compliance with the requested version, if it is supported, or return a
-    406 Not Acceptable error.
-    If this header is not supplied, the server will response with server minimum
-    supported version.
-  in: header
-  required: true
-  type: string
-openstack-request-id:
-  description: >
-    An unique ID for tracking the request. The request ID associated with the request
-    appears in the log lines for that request. By default, the middleware configuration
-    ensures that the request ID appears in the log files.
-  in: header
-  required: false
-  type: string
-
 # Path
 action_ident:
   description: |
@@ -129,14 +90,6 @@ r_strategy:
   in: query
   required: false
   type: string
-r_type:
-  description: |
-    Type of data model user want to list. Default type is compute.
-    Supported values: compute.
-    Future support values: storage, baremetal.
-  in: query
-  required: false
-  type: string
 sort_dir:
   description: |
     Sorts the response by the requested sort direction.
@@ -217,12 +170,6 @@ actionplan_global_efficacy:
   in: body
   required: false
   type: array
-actionplan_hostname:
-  description: |
-    Hostname the actionplan is running on
-  in: body
-  required: false
-  type: string
 actionplan_state:
   description: |
     State of this action plan. To get more information about states and
@@ -234,44 +181,16 @@ actionplan_state:
 # Audit
 audit_autotrigger:
   description: |
-    Auto execute action plan once audit is succeeded.
+    Autoexecute action plan once audit is succeeded.
   in: body
   required: false
   type: boolean
-audit_endtime_req:
-  description: |
-    The local time after which audit can't be executed.
-    It will be converted to UTC time by Watcher.
-  in: body
-  required: false
-  type: string
-  min_version: 1.1
-audit_endtime_resp:
-  description: |
-    The UTC time after which audit can't be executed.
-  in: body
-  required: false
-  type: string
-  min_version: 1.1
-audit_force:
-  description: |
-    Launch audit even if action plan is ongoing.
-  in: body
-  required: false
-  type: boolean
-  min_version: 1.2
 audit_goal:
   description: |
     The UUID or name of the Goal.
   in: body
   required: false
   type: string
-audit_hostname:
-  description: |
-    Hostname the audit is running on
-  in: body
-  required: false
-  type: string
 audit_interval:
   description: |
     Time interval between audit's execution.
@@ -298,21 +217,6 @@ audit_parameters:
   in: body
   required: false
   type: JSON
-audit_starttime_req:
-  description: |
-    The local time after which audit can be executed in accordance
-    with interval. It will be converted to UTC time by Watcher.
-  in: body
-  required: false
-  type: string
-  min_version: 1.1
-audit_starttime_resp:
-  description: |
-    The UTC time after which audit can be executed in accordance with interval.
-  in: body
-  required: false
-  type: string
-  min_version: 1.1
 audit_state:
   description: |
     State of this audit. To get more information about states and
@@ -328,7 +232,7 @@ audit_strategy:
   type: string
 audit_type:
   description: |
-    Type of this audit. Can only be either ONESHOT or CONTINUOUS.
+    Type of this audit. Can be either ONESHOT or CONTINUOUS.
   in: body
   required: true
   type: string
@@ -364,12 +268,6 @@ audittemplate_strategy:
   in: body
   required: false
   type: string
-audittemplate_uuid:
-  description: |
-    The UUID of the Audit template.
-  in: body
-  required: true
-  type: string
 
 created_at:
   description: |
@@ -419,62 +317,6 @@ links:
   required: true
   type: array
 
-# Data Model Node
-node_disk:
-  description: |
-    The Disk of the node(in GiB).
-  in: body
-  required: true
-  type: integer
-node_disk_ratio:
-  description: |
-    The Disk Ratio of the node.
-  in: body
-  required: true
-  type: float
-node_hostname:
-  description: |
-    The Host Name of the node.
-  in: body
-  required: true
-  type: string
-node_memory:
-  description: |
-    The Memory of the node(in MiB).
-  in: body
-  required: true
-  type: integer
-node_memory_ratio:
-  description: |
-    The Memory Ratio of the node.
-  in: body
-  required: true
-  type: float
-node_state:
-  description: |
-    The State of the node. The value is up or down.
-  in: body
-  required: true
-  type: string
-node_uuid:
-  description: |
-    The Unique UUID of the node.
-  in: body
-  required: true
-  type: string
-node_vcpu_ratio:
-  description: |
-    The Vcpu ratio of the node.
-  in: body
-  required: true
-  type: float
-node_vcpus:
-  description: |
-    The Vcpu of the node.
-  in: body
-  required: true
-  type: integer
-
 # Scoring Engine
 scoring_engine_description:
   description: |
@@ -494,72 +336,34 @@ scoring_engine_name:
   in: body
   required: true
   type: string
-
-# Data Model Server
-server_disk:
-  description: |
-    The Disk of the server.
-  in: body
-  required: true
-  type: integer
-server_memory:
-  description: |
-    The Memory of server.
-  in: body
-  required: true
-  type: integer
-server_name:
-  description: |
-    The Name of the server.
-  in: body
-  required: true
-  type: string
-server_state:
-  description: |
-    The State of the server.
-  in: body
-  required: true
-  type: string
-server_uuid:
-  description: |
-    The Unique UUID of the server.
-  in: body
-  required: true
-  type: string
-server_vcpus:
-  description: |
-    The Vcpu of the server.
-  in: body
-  required: true
-  type: integer
 # Service
 service_host:
   description: |
-    The Name of host where service is placed on.
+    Name of host where service is placed on.
   in: body
   required: true
   type: string
 service_id:
   description: |
-    The ID of service.
+    ID of service.
   in: body
   required: true
   type: integer
 service_last_seen_up:
   description: |
-    The Time when Watcher service sent latest heartbeat.
+    Time when Watcher service sent latest heartbeat.
   in: body
   required: true
   type: string
 service_name:
   description: |
-    The Name of service like ``watcher-applier``.
+    Name of service like ``watcher-applier``.
   in: body
   required: true
   type: string
 service_status:
   description: |
-    The State of service. It can be either in ACTIVE or FAILED state.
+    State of service. It can be either in ACTIVE or FAILED state.
   in: body
   required: true
   type: string
@@ -627,29 +431,3 @@ uuid:
   in: body
   required: true
   type: string
-
-# Version
-version:
-  description: |
-    Versioning of this API response, eg. "1.1".
-  in: body
-  required: true
-  type: string
-version_description:
-  description: |
-    Descriptive text about the Watcher service.
-  in: body
-  required: true
-  type: string
-version_id:
-  description: |
-    Major API version, eg, "v1"
-  in: body
-  required: true
-  type: string
-versions:
-  description: |
-    Array of information about currently supported versions.
-  in: body
-  required: true
-  type: array

api-ref/source/samples/actionplan-list-detailed-response.json

@@ -20,8 +20,7 @@
             "deleted_at": null,
             "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
             "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
-            "created_at": "2018-04-10T11:59:52.640067+00:00",
-            "hostname": "controller"
+            "created_at": "2018-04-10T11:59:52.640067+00:00"
         }
     ]
 }

api-ref/source/samples/actionplan-show-response.json

@@ -16,6 +16,5 @@
     "updated_at": "2018-04-10T11:59:52.640067+00:00",
     "strategy_name": "dummy_with_resize",
     "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
-    "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
-    "hostname": "controller"
+    "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a"
 }

api-ref/source/samples/actionplan-start-response.json

@@ -18,6 +18,5 @@
     "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
     "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
     "created_at": "2018-04-10T11:59:12.592729+00:00",
-    "deleted_at": null,
-    "hostname": null
+    "deleted_at": null
 }

api-ref/source/samples/api-root-response.json

@@ -1,30 +0,0 @@
-{
-  "default_version": {
-    "id": "v1",
-    "links": [
-      {
-        "href": "http://controller:9322/v1/",
-        "rel": "self"
-      }
-    ],
-    "min_version": "1.0",
-    "status": "CURRENT",
-    "max_version": "1.1"
-  },
-  "description": "Watcher is an OpenStack project which aims to improve physical resources usage through better VM placement.",
-  "name": "OpenStack Watcher API",
-  "versions": [
-    {
-      "id": "v1",
-      "links": [
-        {
-          "href": "http://controller:9322/v1/",
-          "rel": "self"
-        }
-      ],
-      "min_version": "1.0",
-      "status": "CURRENT",
-      "max_version": "1.1"
-    }
-  ]
-}

api-ref/source/samples/api-v1-root-response.json

@@ -1,80 +0,0 @@
-{
-  "scoring_engines": [
-    {
-      "href": "http://controller:9322/v1/scoring_engines/",
-      "rel": "self"
-    },
-    {
-      "href": "http://controller:9322/scoring_engines/",
-      "rel": "bookmark"
-    }
-  ],
-  "media_types": [
-    {
-      "base": "application/json",
-      "type": "application/vnd.openstack.watcher.v1+json"
-    }
-  ],
-  "links": [
-    {
-      "href": "http://controller:9322/v1/",
-      "rel": "self"
-    },
-    {
-      "href": "http://docs.openstack.org/developer/watcher/dev/api-spec-v1.html",
-      "type": "text/html",
-      "rel": "describedby"
-    }
-  ],
-  "actions": [
-    {
-      "href": "http://controller:9322/v1/actions/",
-      "rel": "self"
-    },
-    {
-      "href": "http://controller:9322/actions/",
-      "rel": "bookmark"
-    }
-  ],
-  "audit_templates": [
-    {
-      "href": "http://controller:9322/v1/audit_templates/",
-      "rel": "self"
-    },
-    {
-      "href": "http://controller:9322/audit_templates/",
-      "rel": "bookmark"
-    }
-  ],
-  "action_plans": [
-    {
-      "href": "http://controller:9322/v1/action_plans/",
-      "rel": "self"
-    },
-    {
-      "href": "http://controller:9322/action_plans/",
-      "rel": "bookmark"
-    }
-  ],
-  "services": [
-    {
-      "href": "http://controller:9322/v1/services/",
-      "rel": "self"
-    },
-    {
-      "href": "http://controller:9322/services/",
-      "rel": "bookmark"
-    }
-  ],
-  "audits": [
-    {
-      "href": "http://controller:9322/v1/audits/",
-      "rel": "self"
-    },
-    {
-      "href": "http://controller:9322/audits/",
-      "rel": "bookmark"
-    }
-  ],
-  "id": "v1"
-}

api-ref/source/samples/audit-cancel-response.json

@@ -28,7 +28,6 @@
         }
     },
     "auto_trigger": false,
-    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],
@@ -48,8 +47,5 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": "2018-04-06T11:56:00",
-    "updated_at": "2018-04-06T11:54:01.266447+00:00",
-    "hostname": "controller",
-    "start_time": null,
-    "end_time": null
-}
+    "updated_at": "2018-04-06T11:54:01.266447+00:00"
+}

api-ref/source/samples/audit-create-request-continuous.json

@@ -1,6 +1,5 @@
 {
     "auto_trigger": false,
-    "force": false,
     "audit_template_uuid": "76fddfee-a9c4-40b0-8da0-c19ad6904f09",
     "name": "test_audit",
     "parameters": {
@@ -9,7 +8,5 @@
         ]
     },
     "audit_type": "CONTINUOUS",
-    "interval": "*/2 * * * *",
-    "start_time":"2018-04-02 20:30:00",
-    "end_time": "2018-04-04 20:30:00"
-}
+    "interval": "*/2 * * * *"
+}

api-ref/source/samples/audit-create-request-oneshot.json

@@ -1,6 +1,5 @@
 {
     "audit_type": "ONESHOT",
     "auto_trigger": false,
-    "force": true,
     "audit_template_uuid": "5e70a156-ced7-4012-b1c6-88fcb02ee0c1"
-}
+}

api-ref/source/samples/audit-create-response.json

@@ -28,7 +28,6 @@
         }
     },
     "auto_trigger": false,
-    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],
@@ -48,8 +47,5 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": null,
-    "updated_at": null,
-    "hostname": null,
-    "start_time": null,
-    "end_time": null
-}
+    "updated_at": null
+}

api-ref/source/samples/audit-list-detailed-response.json

@@ -30,7 +30,6 @@
                 }
             },
             "auto_trigger": false,
-	    "force": false,
             "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
             "goal_name": "workload_balancing",
             "scope": [],
@@ -50,10 +49,7 @@
             ],
             "strategy_name": "workload_stabilization",
             "next_run_time": "2018-04-06T09:46:00",
-            "updated_at": "2018-04-06T09:44:01.604146+00:00",
-            "hostname": "controller",
-            "start_time": null,
-            "end_time": null
+            "updated_at": "2018-04-06T09:44:01.604146+00:00"
         }
     ]
-}
+}

api-ref/source/samples/audit-show-response.json

@@ -28,7 +28,6 @@
         }
     },
     "auto_trigger": false,
-    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],
@@ -48,8 +47,5 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": "2018-04-06T11:56:00",
-    "updated_at": "2018-04-06T11:54:01.266447+00:00",
-    "hostname": "controller",
-    "start_time": null,
-    "end_time": null
-}
+    "updated_at": "2018-04-06T11:54:01.266447+00:00"
+}

api-ref/source/samples/audit-update-response.json

@@ -28,7 +28,6 @@
         }
     },
     "auto_trigger": false,
-    "force": false,
     "uuid": "65a5da84-5819-4aea-8278-a28d2b489028",
     "goal_name": "workload_balancing",
     "scope": [],
@@ -48,8 +47,5 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": "2018-04-06T11:56:00",
-    "updated_at": "2018-04-06T11:54:01.266447+00:00",
-    "hostname": "controller",
-    "start_time": null,
-    "end_time": null
-}
+    "updated_at": "2018-04-06T11:54:01.266447+00:00"
+}

api-ref/source/samples/datamodel-list-response.json

@@ -1,38 +0,0 @@
-{
-    "context": [
-        {
-            "server_uuid": "1bf91464-9b41-428d-a11e-af691e5563bb",
-            "server_name": "chenke-test1",
-            "server_vcpus": "1",
-            "server_memory": "512",
-            "server_disk": "1",
-            "server_state": "active",
-            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112",
-            "node_hostname": "localhost.localdomain",
-            "node_vcpus": "4",
-            "node_vcpu_ratio": "16.0",
-            "node_memory": "16383",
-            "node_memory_ratio": "1.5",
-            "node_disk": "37"
-            "node_disk_ratio": "1.0",
-            "node_state": "up",
-        },
-        {
-            "server_uuid": "e2cb5f6f-fa1d-4ba2-be1e-0bf02fa86ba4",
-            "server_name": "chenke-test2",
-            "server_vcpus": "1",
-            "server_memory": "512",
-            "server_disk": "1",
-            "server_state": "active",
-            "node_uuid": "253e5dd0-9384-41ab-af13-4f2c2ce26112",
-            "node_hostname": "localhost.localdomain",
-            "node_vcpus": "4",
-            "node_vcpu_ratio": "16.0",
-            "node_memory": "16383",
-            "node_memory_ratio": "1.5",
-            "node_disk": "37"
-            "node_disk_ratio": "1.0",
-            "node_state": "up",
-        }
-    ]
-}

api-ref/source/watcher-api-v1-actionplans.inc

@@ -50,7 +50,6 @@ version 1:
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
-    - hostname: actionplan_hostname
 
 **Example JSON representation of an Action Plan:**
 
@@ -138,7 +137,6 @@ Response
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
-    - hostname: actionplan_hostname
 
 **Example JSON representation of an Action Plan:**
 
@@ -176,7 +174,6 @@ Response
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
-    - hostname: actionplan_hostname
 
 **Example JSON representation of an Audit:**
 
@@ -232,7 +229,6 @@ version 1:
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
-    - hostname: actionplan_hostname
 
 **Example JSON representation of an Action Plan:**
 

api-ref/source/watcher-api-v1-audits.inc

@@ -28,6 +28,10 @@ itself. In the first case, there also should be supplied
 ``audit_template_uuid``. If ``Audit`` is created without ``Audit Template``,
 ``goal`` should be provided.
 
+.. warning::
+    **Only ``audit_template_uuid`` can be used to create audit so far.**
+    It should be fixed during the ``Rocky`` cycle.
+
 Normal response codes: 201
 
 Error codes: 400,404
@@ -37,17 +41,15 @@ Request
 
 .. rest_parameters:: parameters.yaml
 
-   - audit_template_uuid: audittemplate_uuid
+   - audit_template_uuid: audittemplate_name
    - audit_type: audit_type
    - name: audit_name
    - goal: audit_goal
    - strategy: audit_strategy
    - parameters: audit_parameters
    - interval: audit_interval
+   - scope: audittemplate_scope
    - auto_trigger: audit_autotrigger
-   - start_time: audit_starttime_req
-   - end_time: audit_endtime_req
-   - force: audit_force
 
 **Example ONESHOT Audit creation request:**
 
@@ -81,10 +83,6 @@ version 1:
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
-    - hostname: audit_hostname
-    - start_time: audit_starttime_resp
-    - end_time: audit_endtime_resp
-    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -180,10 +178,6 @@ Response
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
-    - hostname: audit_hostname
-    - start_time: audit_starttime_resp
-    - end_time: audit_endtime_resp
-    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -227,10 +221,6 @@ Response
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
-    - hostname: audit_hostname
-    - start_time: audit_starttime_resp
-    - end_time: audit_endtime_resp
-    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -282,10 +272,6 @@ version 1:
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
-    - hostname: audit_hostname
-    - start_time: audit_starttime_resp
-    - end_time: audit_endtime_resp
-    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -337,10 +323,6 @@ Response
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
-    - hostname: audit_hostname
-    - start_time: audit_starttime_resp
-    - end_time: audit_endtime_resp
-    - force: audit_force
 
 **Example JSON representation of an Audit:**
 
@@ -364,4 +346,4 @@ Request
 
 .. rest_parameters:: parameters.yaml
 
-   - audit_ident: audit_ident
+   - audit_ident: audit_ident

api-ref/source/watcher-api-v1-datamodel.inc

@@ -1,57 +0,0 @@
-.. -*- rst -*-
-
-==========
-Data Model
-==========
-
-.. versionadded:: 1.3
-
-``Data Model`` is very important for Watcher to generate resource
-optimization solutions. Users can easily view the data model by the
-API.
-
-
-List Data Model
-===============
-
-.. rest_method::  GET /v1/data_model
-
-Returns the information about Data Model.
-
-Normal response codes: 200
-
-Error codes: 400,401,406
-
-Request
--------
-
-.. rest_parameters:: parameters.yaml
-
-   - audit: r_audit
-   - type: r_type
-
-Response
---------
-
-.. rest_parameters:: parameters.yaml
-
-    - server_uuid: server_uuid
-    - server_name: server_name
-    - server_vcpus: server_vcpus
-    - server_memory: server_memory
-    - server_disk: server_disk
-    - server_state: server_state
-    - node_uuid: node_uuid
-    - node_hostname: node_hostname
-    - node_vcpus: node_vcpus
-    - node_vcpu_ratio: node_vcpu_ratio
-    - node_memory: node_memory
-    - node_memory_ratio: node_memory_ratio
-    - node_disk: node_disk
-    - node_disk_ratio: node_disk_ratio
-    - node_state: node_state
-
-**Example JSON representation of a Data Model:**
-
-.. literalinclude:: samples/datamodel-list-response.json
-   :language: javascript

api-ref/source/watcher-api-v1-webhooks.inc

@@ -1,26 +0,0 @@
-.. -*- rst -*-
-
-========
-Webhooks
-========
-
-.. versionadded:: 1.4
-
-Triggers an event based Audit.
-
-
-Trigger EVENT Audit
-===================
-
-.. rest_method::  POST /v1/webhooks/{audit_ident}
-
-Normal response codes: 202
-
-Error codes: 400,404
-
-Request
--------
-
-.. rest_parameters:: parameters.yaml
-
-   - audit_ident: audit_ident

api-ref/source/watcher-api-versions.inc

@@ -1,81 +0,0 @@
-.. -*- rst -*-
-
-============
-API versions
-============
-
-In order to bring new features to users over time, the Watcher API
-supports versioning. There are two kinds of versions in Watcher.
-
-- ''major versions'', which have dedicated URLs.
-- ''microversions'', which can be requested using the
-  ``OpenStack-API-Version`` header.
-
-.. note:: The maximum microversion depends on release.
-  Please reference:
-  `API Microversion History
-  <https://docs.openstack.org/watcher/latest/contributor/api_microversion_history.html>`__
-  for API microversion history details.
-
-The Version API resource works differently from other API resources as they *do not*
-require authentication.
-
-If Watcher receives a request with unsupported version, it responds with a 406 Not Acceptable,
-along with the -Min- and -Max- headers that it can support.
-
-List API versions
-=================
-
-.. rest_method::  GET /
-
-This fetches all the information about all known major API versions in the
-deployment. Links to more specific information will be provided for each major
-API version, as well as information about supported min and max microversions.
-
-Normal response codes: 200
-
-Request
--------
-
-Response Example
-----------------
-
-.. rest_parameters::  parameters.yaml
-
-    - description: version_description
-    - versions: versions
-    - version: version
-    - id: version_id
-    - links: links
-    - min_version: openstack-api-min-version
-    - max_version: openstack-api-max-version
-
-.. literalinclude:: samples/api-root-response.json
-   :language: javascript
-
-
-Show v1 API
-===========
-
-.. rest_method::  GET /v1/
-
-Show all the resources within the Watcher v1 API.
-
-Normal response codes: 200
-
-Request
--------
-
-Response Example
-----------------
-
-.. rest_parameters::  parameters.yaml
-
-    - id: version_id
-    - links: links
-    - OpenStack-API-Version: header_version
-    - OpenStack-API-Minimum-Version: openstack-api-min-version
-    - OpenStack-API-Maximum-Version: openstack-api-max-version
-
-.. literalinclude:: samples/api-v1-root-response.json
-   :language: javascript

babel.cfg

@@ -0,0 +1,2 @@
+[python: **.py]
+

devstack/lib/watcher

@@ -47,23 +47,14 @@ WATCHER_POLICY_YAML=$WATCHER_CONF_DIR/policy.yaml.sample
 WATCHER_DEVSTACK_DIR=$WATCHER_DIR/devstack
 WATCHER_DEVSTACK_FILES_DIR=$WATCHER_DEVSTACK_DIR/files
 
+NOVA_CONF_DIR=/etc/nova
+NOVA_CONF=$NOVA_CONF_DIR/nova.conf
+
 if is_ssl_enabled_service "watcher" || is_service_enabled tls-proxy; then
     WATCHER_SERVICE_PROTOCOL="https"
 fi
 
-# Support entry points installation of console scripts
-if [[ -d $WATCHER_DIR/bin ]]; then
-    WATCHER_BIN_DIR=$WATCHER_DIR/bin
-else
-    WATCHER_BIN_DIR=$(get_python_exec_prefix)
-fi
-
-# There are 2 modes, which is "uwsgi" which runs with an apache
-# proxy uwsgi in front of it, or "mod_wsgi", which runs in
-# apache. mod_wsgi is deprecated, don't use it.
-WATCHER_USE_WSGI_MODE=${WATCHER_USE_WSGI_MODE:-$WSGI_MODE}
-WATCHER_UWSGI=$WATCHER_BIN_DIR/watcher-api-wsgi
-WATCHER_UWSGI_CONF=$WATCHER_CONF_DIR/watcher-uwsgi.ini
+WATCHER_USE_MOD_WSGI=$(trueorfalse True WATCHER_USE_MOD_WSGI)
 
 if is_suse; then
     WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/srv/www/htdocs/watcher}
@@ -71,15 +62,16 @@ else
     WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/var/www/watcher}
 fi
 # Public facing bits
-WATCHER_SERVICE_HOST=${WATCHER_SERVICE_HOST:-$SERVICE_HOST}
+WATCHER_SERVICE_HOST=${WATCHER_SERVICE_HOST:-$HOST_IP}
 WATCHER_SERVICE_PORT=${WATCHER_SERVICE_PORT:-9322}
 WATCHER_SERVICE_PORT_INT=${WATCHER_SERVICE_PORT_INT:-19322}
 WATCHER_SERVICE_PROTOCOL=${WATCHER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 
-if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
+# Support entry points installation of console scripts
+if [[ -d $WATCHER_DIR/bin ]]; then
+    WATCHER_BIN_DIR=$WATCHER_DIR/bin
 else
-    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
+    WATCHER_BIN_DIR=$(get_python_exec_prefix)
 fi
 
 # Entry Points
@@ -104,9 +96,7 @@ function _cleanup_watcher_apache_wsgi {
 # runs that a clean run would need to clean up
 function cleanup_watcher {
     sudo rm -rf $WATCHER_STATE_PATH $WATCHER_AUTH_CACHE_DIR
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
-    else
+    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         _cleanup_watcher_apache_wsgi
     fi
 }
@@ -152,32 +142,29 @@ function create_watcher_accounts {
         "infra-optim" "Watcher Infrastructure Optimization Service")
     get_or_create_endpoint $watcher_service \
         "$REGION_NAME" \
-        "$WATCHER_API_URL"\
-        "$WATCHER_API_URL"\
-        "$WATCHER_API_URL"
+        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" \
+        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" \
+        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
 }
 
 # _config_watcher_apache_wsgi() - Set WSGI config files of watcher
 function _config_watcher_apache_wsgi {
     local watcher_apache_conf
-    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-        local service_port=$WATCHER_SERVICE_PORT
-        if is_service_enabled tls-proxy; then
-            service_port=$WATCHER_SERVICE_PORT_INT
-            service_protocol="http"
-        fi
+    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         sudo mkdir -p $WATCHER_WSGI_DIR
         sudo cp $WATCHER_DIR/watcher/api/app.wsgi $WATCHER_WSGI_DIR/app.wsgi
         watcher_apache_conf=$(apache_site_config_for watcher-api)
         sudo cp $WATCHER_DEVSTACK_FILES_DIR/apache-watcher-api.template $watcher_apache_conf
         sudo sed -e "
-            s|%WATCHER_SERVICE_PORT%|$service_port|g;
+            s|%WATCHER_SERVICE_PORT%|$WATCHER_SERVICE_PORT|g;
             s|%WATCHER_WSGI_DIR%|$WATCHER_WSGI_DIR|g;
             s|%USER%|$STACK_USER|g;
             s|%APIWORKERS%|$API_WORKERS|g;
             s|%APACHE_NAME%|$APACHE_NAME|g;
         " -i $watcher_apache_conf
         enable_apache_site watcher-api
+        tail_log watcher-access /var/log/$APACHE_NAME/watcher-api-access.log
+        tail_log watcher-api /var/log/$APACHE_NAME/watcher-api.log
    fi
 
 }
@@ -193,23 +180,22 @@ function create_watcher_conf {
     iniset_rpc_backend watcher $WATCHER_CONF
 
     iniset $WATCHER_CONF database connection $(database_connection_url watcher)
-    iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
+    iniset $WATCHER_CONF api host "$WATCHER_SERVICE_HOST"
 
     if is_service_enabled tls-proxy; then
-        iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
         iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT_INT"
         # iniset $WATCHER_CONF api enable_ssl_api "True"
     else
-        if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-            iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
-            iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
-        fi
+        iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
     fi
 
     iniset $WATCHER_CONF oslo_policy policy_file $WATCHER_POLICY_YAML
 
     iniset $WATCHER_CONF oslo_messaging_notifications driver "messagingv2"
 
+    iniset $NOVA_CONF oslo_messaging_notifications topics "notifications,watcher_notifications"
+    iniset $NOVA_CONF notifications notify_on_state_change "vm_and_task_state"
+
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR "watcher_clients_auth"
 
@@ -229,12 +215,15 @@ function create_watcher_conf {
     fi
 
     # Format logging
-    setup_logging $WATCHER_CONF
+    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
+        setup_colorized_logging $WATCHER_CONF DEFAULT
+    else
+        # Show user_name and project_name instead of user_id and project_id
+        iniset $WATCHER_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(project_domain)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
+    fi
 
     #config apache files
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
-    else
+    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         _config_watcher_apache_wsgi
     fi
     # Register SSL certificates if provided
@@ -246,6 +235,10 @@ function create_watcher_conf {
 
         iniset $WATCHER_CONF DEFAULT enabled_ssl_apis "$WATCHER_ENABLED_APIS"
     fi
+
+    if is_service_enabled ceilometer; then
+        iniset $WATCHER_CONF watcher_messaging notifier_driver "messaging"
+    fi
 }
 
 # create_watcher_cache_dir() - Part of the init_watcher() process
@@ -281,7 +274,7 @@ function install_watcherclient {
 function install_watcher {
     git_clone $WATCHER_REPO $WATCHER_DIR $WATCHER_BRANCH
     setup_develop $WATCHER_DIR
-    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
+    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         install_apache_wsgi
     fi
 }
@@ -292,29 +285,24 @@ function start_watcher_api {
 
     local service_port=$WATCHER_SERVICE_PORT
     local service_protocol=$WATCHER_SERVICE_PROTOCOL
-    local watcher_url
     if is_service_enabled tls-proxy; then
         service_port=$WATCHER_SERVICE_PORT_INT
         service_protocol="http"
     fi
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
-        watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
-    else
-        watcher_url=$service_protocol://$SERVICE_HOST:$service_port
-        enable_apache_site watcher-api
+    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         restart_apache_server
-        # Start proxies if enabled
-        if is_service_enabled tls-proxy; then
-            start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
-        fi
+    else
+        run_process watcher-api "$WATCHER_BIN_DIR/watcher-api --config-file $WATCHER_CONF"
     fi
-
     echo "Waiting for watcher-api to start..."
-    if ! wait_for_service $SERVICE_TIMEOUT $watcher_url; then
+    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$WATCHER_SERVICE_HOST:$service_port; then
         die $LINENO "watcher-api did not start"
     fi
 
+    # Start proxies if enabled
+    if is_service_enabled tls-proxy; then
+        start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
+    fi
 }
 
 # start_watcher() - Start running processes, including screen
@@ -327,11 +315,11 @@ function start_watcher {
 
 # stop_watcher() - Stop running processes (non-screen)
 function stop_watcher {
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        stop_process watcher-api
-    else
+    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         disable_apache_site watcher-api
-        restart_apache_server
+	restart_apache_server
+    else
+        stop_process watcher-api
     fi
     for serv in watcher-decision-engine watcher-applier; do
         stop_process $serv

devstack/local.conf.compute

@@ -10,7 +10,6 @@ SERVICE_PASSWORD=$ADMIN_PASSWORD
 SERVICE_TOKEN=azertytoken
 
 HOST_IP=192.168.42.2 # Change this to this compute node's IP address
-#HOST_IPV6=2001:db8::7
 FLAT_INTERFACE=eth0
 
 FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is
@@ -31,12 +30,12 @@ ENABLED_SERVICES=n-cpu,n-api-meta,c-vol,q-agt,placement-client
 NOVA_VNC_ENABLED=True
 NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_auto.html"
 VNCSERVER_LISTEN=0.0.0.0
-VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP # or HOST_IPV6
+VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP
 
 NOVA_INSTANCES_PATH=/opt/stack/data/instances
 
 # Enable the Ceilometer plugin for the compute agent
-enable_plugin ceilometer https://opendev.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
 
 LOGFILE=$DEST/logs/stack.sh.log
@@ -45,9 +44,6 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
+notify_on_state_change = vm_and_task_state
 [notifications]
-# Enable both versioned and unversioned notifications. Watcher only
-# uses versioned notifications but ceilometer uses unversioned. We
-# can change this to just versioned when ceilometer handles versioned
-# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
-notification_format=both
+notify_on_state_change = vm_and_task_state

devstack/local.conf.controller

@@ -10,7 +10,6 @@ SERVICE_PASSWORD=$ADMIN_PASSWORD
 SERVICE_TOKEN=azertytoken
 
 HOST_IP=192.168.42.1 # Change this to your controller node IP address
-#HOST_IPV6=2001:db8::7
 FLAT_INTERFACE=eth0
 
 FIXED_RANGE=10.254.1.0/24 # Change this to whatever your network is
@@ -26,13 +25,13 @@ MULTI_HOST=1
 disable_service n-cpu
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard https://opendev.org/openstack/watcher-dashboard
+enable_plugin watcher-dashboard https://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
-enable_plugin watcher https://opendev.org/openstack/watcher
+enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 # Enable the Ceilometer plugin
-enable_plugin ceilometer https://opendev.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 
 # This is the controller node, so disable the ceilometer compute agent
 disable_service ceilometer-acompute
@@ -49,9 +48,6 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
+notify_on_state_change = vm_and_task_state
 [notifications]
-# Enable both versioned and unversioned notifications. Watcher only
-# uses versioned notifications but ceilometer uses unversioned. We
-# can change this to just versioned when ceilometer handles versioned
-# notifications from nova: https://bugs.launchpad.net/ceilometer/+bug/1665449
-notification_format=both
+notify_on_state_change = vm_and_task_state

devstack/override-defaults

@@ -1,9 +0,0 @@
-# Plug-in overrides
-# https://docs.openstack.org/devstack/latest/plugins.html#plugin-interface
-
-# Enable both versioned and unversioned notifications. Watcher only
-# uses versioned notifications but ceilometer uses unversioned. We
-# can change this to just versioned when ceilometer handles
-# versioned notifications from nova:
-# https://bugs.launchpad.net/ceilometer/+bug/1665449
-NOVA_NOTIFICATION_FORMAT=both

devstack/upgrade/from_rocky/upgrade-watcher

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-# ``upgrade-watcher``
-
-function configure_watcher_upgrade {
-    XTRACE=$(set +o | grep xtrace)
-    set -o xtrace
-
-    # Copy release-specific files
-    sudo cp $TARGET_RELEASE_DIR/watcher/etc/watcher/watcher.conf $WATCHER_CONF_DIR/watcher.conf
-    sudo cp $TARGET_RELEASE_DIR/watcher/etc/watcher/policy.yaml.sample $WATCHER_CONF_DIR/policy.yaml.sample
-
-    # reset to previous state
-    $XTRACE
-}

devstack/upgrade/resources.sh

@@ -1,126 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-
-source $GRENADE_DIR/grenaderc
-source $GRENADE_DIR/functions
-
-source $TOP_DIR/openrc admin demo
-
-set -o xtrace
-
-function _wait_for_status {
-     while :
-     do
-         state=$("${@:2}" -f value -c State)
-         [[ $state == "SUCCEEDED" ]] && break
-         if [ $state == "ERROR" ]; then
-             die $LINENO "ERROR creating audit"
-         fi
-         sleep 10
-     done
- }
-
-function create_audit_template {
-    at_id=$(openstack optimize audittemplate create d1 dummy -s dummy -f value -c UUID)
-    resource_save watcher at_id $at_id
- }
-
-function create_audit {
-    audit_id=$(openstack optimize audit create -s dummy -g dummy -f value -c UUID)
-    resource_save watcher audit_id $audit_id
-}
-
-function create_audit_with_autotrigger {
-    audit_at_id=$(openstack optimize audit create -s dummy -g dummy -f value -c UUID --auto-trigger)
-    resource_save watcher audit_at_id $audit_at_id
-}
-
-function verify_audit_template {
-    local at_id=$(resource_get watcher at_id)
-    openstack optimize audittemplate show $at_id
-}
-
-function verify_audit_with_autotrigger {
-    local audit_at_id=$(resource_get watcher audit_at_id)
-    _wait_for_status "SUCCEEDED" openstack optimize audit show $audit_at_id
-    local actionplan_at_id=$(openstack optimize actionplan list --audit $audit_at_id -c UUID -f value)
-    resource_save watcher actionplan_at $actionplan_at_id
-    actionplan_at_state=$(openstack optimize actionplan show $actionplan_at_id -c State -f value)
-    if [ $actionplan_at_state != "SUCCEEDED" ]; then
-        die $LINENO "ERROR executing actionplan"
-    fi
-}
-
-function verify_audit {
-    local audit_id=$(resource_get watcher audit_id)
-    _wait_for_status "SUCCEEDED" openstack optimize audit show $audit_id
-    local actionplan_id=$(openstack optimize actionplan list --audit $audit_id -c UUID -f value)
-    resource_save watcher actionplan $actionplan_id
-    actionplan_state=$(openstack optimize actionplan show $actionplan_id -c State -f value)
-    if [ $actionplan_state != "RECOMMENDED" ]; then
-        die $LINENO "ERROR creating actionplan"
-    fi
-}
-
-function verify_noapi {
-    # currently no good way
-    :
-}
-
-function delete_audit {
-  local audit_id=$(resource_get watcher audit_id)
-  local actionplan_id=$(resource_get watcher actionplan)
-  watcher actionplan delete $actionplan_id
-  openstack optimize audit delete $audit_id
-}
-
-function delete_audit_with_autotrigger {
-  local audit_at_id=$(resource_get watcher audit_at_id)
-  local actionplan_id=$(resource_get watcher actionplan_at)
-  watcher actionplan delete $actionplan_id
-  openstack optimize audit delete $audit_at_id
-}
-
-function delete_audit_template {
-  local at_id=$(resource_get watcher at_id)
-  openstack optimize audittemplate delete $at_id
-}
-
-function create {
-  create_audit_template
-  create_audit
-  create_audit_with_autotrigger
-}
-
-function verify {
-  verify_audit_template
-  verify_audit
-  verify_audit_with_autotrigger
-}
-
-function destroy {
-  delete_audit_template
-  delete_audit
-  delete_audit_with_autotrigger
-}
-
-# Dispatcher
-case $1 in
-    "create")
-        create
-        ;;
-    "verify_noapi")
-        verify_noapi
-        ;;
-    "verify")
-        verify
-        ;;
-    "destroy")
-        destroy
-        ;;
-    "force_destroy")
-        set +o errexit
-        destroy
-        ;;
-esac

devstack/upgrade/settings

@@ -1,18 +0,0 @@
-register_project_for_upgrade watcher
-register_db_to_save watcher
-
-devstack_localrc base enable_plugin watcher https://opendev.org/openstack/watcher $BASE_DEVSTACK_BRANCH
-devstack_localrc target enable_plugin watcher https://opendev.org/openstack/watcher
-
-devstack_localrc base enable_service watcher-api watcher-decision-engine watcher-applier
-devstack_localrc target enable_service watcher-api watcher-decision-engine watcher-applier
-
-BASE_RUN_SMOKE=False
-TARGET_RUN_SMOKE=False
-
-# Enable both versioned and unversioned notifications. Watcher only
-# uses versioned notifications but ceilometer uses unversioned. We
-# can change this to just versioned when ceilometer handles
-# versioned notifications from nova:
-# https://bugs.launchpad.net/ceilometer/+bug/1665449
-devstack_localrc base NOVA_NOTIFICATION_FORMAT=both

devstack/upgrade/shutdown.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-
-source $GRENADE_DIR/grenaderc
-source $GRENADE_DIR/functions
-
-# We need base DevStack functions for this
-source $BASE_DEVSTACK_DIR/functions
-source $BASE_DEVSTACK_DIR/stackrc # needed for status directory
-source $BASE_DEVSTACK_DIR/lib/tls
-source $BASE_DEVSTACK_DIR/lib/apache
-
-WATCHER_DEVSTACK_DIR=$(dirname $(dirname $0))
-source $WATCHER_DEVSTACK_DIR/settings
-source $WATCHER_DEVSTACK_DIR/plugin.sh
-source $WATCHER_DEVSTACK_DIR/lib/watcher
-
-set -o xtrace
-
-stop_watcher
-
-# sanity check that service is actually down
-ensure_services_stopped watcher-api watcher-decision-engine watcher-applier

devstack/upgrade/upgrade.sh

@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-
-# ``upgrade-watcher``
-
-echo "*********************************************************************"
-echo "Begin $0"
-echo "*********************************************************************"
-
-# Clean up any resources that may be in use
-cleanup() {
-    set +o errexit
-
-    echo "********************************************************************"
-    echo "ERROR: Abort $0"
-    echo "********************************************************************"
-
-    # Kill ourselves to signal any calling process
-    trap 2; kill -2 $$
-}
-
-trap cleanup SIGHUP SIGINT SIGTERM
-
-# Keep track of the grenade directory
-RUN_DIR=$(cd $(dirname "$0") && pwd)
-
-# Source params
-source $GRENADE_DIR/grenaderc
-
-# Import common functions
-source $GRENADE_DIR/functions
-
-# This script exits on an error so that errors don't compound and you see
-# only the first error that occurred.
-set -o errexit
-
-# Upgrade watcher
-# ============
-
-# Get functions from current DevStack
-source $TARGET_DEVSTACK_DIR/stackrc
-source $TARGET_DEVSTACK_DIR/lib/apache
-source $TARGET_DEVSTACK_DIR/lib/tls
-source $TARGET_DEVSTACK_DIR/lib/keystone
-
-source $TOP_DIR/openrc admin admin
-
-source $(dirname $(dirname $BASH_SOURCE))/settings
-source $(dirname $(dirname $BASH_SOURCE))/plugin.sh
-
-# Print the commands being run so that we can see the command that triggers
-# an error.  It is also useful for following allowing as the install occurs.
-set -o xtrace
-
-# Save current config files for posterity
-[[ -d $SAVE_DIR/etc.watcher ]] || cp -pr $WATCHER_CONF_DIR $SAVE_DIR/etc.watcher
-
-# Install the target watcher
-install_watcher
-
-# calls upgrade-watcher for specific release
-upgrade_project watcher $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
-
-if [[ ! -f "$WATCHER_UWSGI_CONF" ]] && [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]
-then write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
-     endpoints=$(openstack endpoint list --service watcher -c ID -f value)
-     for id in $endpoints; do
-         openstack endpoint delete $id
-     done
-     create_watcher_accounts
-fi
-
-# Migrate the database
-watcher-db-manage upgrade || die $LINO "DB migration error"
-
-start_watcher
-
-# Don't succeed unless the services come up
-ensure_services_started watcher-api watcher-decision-engine watcher-applier
-
-set +o xtrace
-echo "*********************************************************************"
-echo "SUCCESS: End $0"
-echo "*********************************************************************"

doc/ext/term.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import unicode_literals
+
 import importlib
 import inspect
 

doc/requirements.txt

@@ -1,10 +0,0 @@
-# The order of packages is significant, because pip processes them in the order
-# of appearance. Changing the order has an impact on the overall integration
-# process, which may cause wedges in the gate later.
-openstackdocstheme>=2.2.1 # Apache-2.0
-sphinx>=2.0.0,!=2.1.0 # BSD
-sphinxcontrib-pecanwsme>=0.8.0 # Apache-2.0
-sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD
-reno>=3.1.0 # Apache-2.0
-sphinxcontrib-apidoc>=0.2.0  # BSD
-os-api-ref>=1.4.0 # Apache-2.0

doc/source/admin/apache-mod-wsgi.rst

@@ -44,6 +44,6 @@ Installing API behind mod_wsgi
     Fedora/RHEL7/CentOS7:
       sudo systemctl reload httpd
 
-    Debian/Ubuntu:
-      sudo a2ensite watcher
-      sudo service apache2 reload
+   Debian/Ubuntu:
+    sudo a2ensite watcher
+    sudo service apache2 reload

doc/source/admin/index.rst

@@ -8,7 +8,5 @@ Administrator Guide
    apache-mod-wsgi
    gmr
    policy
+   ways-to-install
    ../strategies/index
-   ../datasources/index
-   ../contributor/notifications
-   ../contributor/concurrency

doc/source/admin/ways-to-install.rst

@@ -51,7 +51,7 @@ Clone the Watcher repository:
 
 .. code-block:: bash
 
-    $ git clone https://opendev.org/openstack/watcher.git
+    $ git clone https://git.openstack.org/openstack/watcher.git
     $ cd watcher
 
 Install the Watcher modules:
@@ -92,7 +92,7 @@ these commands:
 By default, this will show logging on the console from which it was started.
 Once started, you can use the `Watcher Client`_ to play with Watcher service.
 
-.. _`Watcher Client`: https://opendev.org/openstack/python-watcherclient
+.. _`Watcher Client`: https://git.openstack.org/cgit/openstack/python-watcherclient
 
 Installing from packages: PyPI
 --------------------------------

doc/source/api/index.rst

@@ -0,0 +1,8 @@
+==================================================
+OpenStack Infrastructure Optimization Service APIs
+==================================================
+
+.. toctree::
+   :maxdepth: 1
+
+   v1

doc/source/api/v1.rst

@@ -0,0 +1,100 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+====================
+RESTful Web API (v1)
+====================
+
+Goals
+=====
+
+.. rest-controller:: watcher.api.controllers.v1.goal:GoalsController
+   :webprefix: /v1/goal
+
+.. autotype:: watcher.api.controllers.v1.goal.GoalCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.goal.Goal
+   :members:
+
+Strategies
+==========
+
+.. rest-controller:: watcher.api.controllers.v1.strategy:StrategiesController
+   :webprefix: /v1/strategies
+
+.. autotype:: watcher.api.controllers.v1.strategy.StrategyCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.strategy.Strategy
+   :members:
+
+Audit Templates
+===============
+
+.. rest-controller:: watcher.api.controllers.v1.audit_template:AuditTemplatesController
+   :webprefix: /v1/audit_templates
+
+.. autotype:: watcher.api.controllers.v1.audit_template.AuditTemplateCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.audit_template.AuditTemplate
+   :members:
+
+Audits
+======
+
+.. rest-controller:: watcher.api.controllers.v1.audit:AuditsController
+   :webprefix: /v1/audits
+
+.. autotype:: watcher.api.controllers.v1.audit.AuditCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.audit.Audit
+   :members:
+
+Links
+=====
+
+.. autotype:: watcher.api.controllers.link.Link
+   :members:
+
+Action Plans
+============
+
+.. rest-controller:: watcher.api.controllers.v1.action_plan:ActionPlansController
+   :webprefix: /v1/action_plans
+
+.. autotype:: watcher.api.controllers.v1.action_plan.ActionPlanCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.action_plan.ActionPlan
+   :members:
+
+
+Actions
+=======
+
+.. rest-controller:: watcher.api.controllers.v1.action:ActionsController
+   :webprefix: /v1/actions
+
+.. autotype:: watcher.api.controllers.v1.action.ActionCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.action.Action
+   :members:
+
+Scoring Engine
+==============
+
+.. rest-controller:: watcher.api.controllers.v1.scoring_engine:ScoringEngineController
+   :webprefix: /v1/scoring_engine
+
+.. autotype:: watcher.api.controllers.v1.scoring_engine.ScoringEngineCollection
+   :members:
+
+.. autotype:: watcher.api.controllers.v1.scoring_engine.ScoringEngine
+   :members:

doc/source/architecture.rst

@@ -76,7 +76,6 @@ Watcher Applier
 This component is in charge of executing the
 :ref:`Action Plan <action_plan_definition>` built by the
 :ref:`Watcher Decision Engine <watcher_decision_engine_definition>`.
-Taskflow is the default workflow engine for Watcher.
 
 It connects to the :ref:`message bus <amqp_bus_definition>` and launches the
 :ref:`Action Plan <action_plan_definition>` whenever a triggering message is
@@ -111,23 +110,6 @@ If the :ref:`Action <action_definition>` fails, the
 previous state of the :ref:`Managed resource <managed_resource_definition>`
 (i.e. before the command was sent to the underlying OpenStack service).
 
-In Stein, added a new config option 'action_execution_rule' which is a
-dict type. Its key field is strategy name and the value is 'ALWAYS' or 'ANY'.
-'ALWAYS' means the callback function returns True as usual.
-'ANY' means the return depends on the result of previous action execution.
-The callback returns True if previous action gets failed, and the engine
-continues to run the next action. If previous action executes success,
-the callback returns False then the next action will be ignored.
-For strategies that aren't in 'action_execution_rule', the callback always
-returns True.
-Please add the next section in the watcher.conf file
-if your strategy needs this feature.
-
-::
-
-    [watcher_workflow_engines.taskflow]
-    action_execution_rule = {'your strategy name': 'ANY'}
-
 .. _archi_watcher_cli_definition:
 
 Watcher CLI
@@ -281,13 +263,11 @@ previously created :ref:`Audit template <audit_template_definition>`:
    :width: 100%
 
 The :ref:`Administrator <administrator_definition>` also can specify type of
-Audit and interval (in case of CONTINUOUS type). There is three types of Audit:
-ONESHOT, CONTINUOUS and EVENT. ONESHOT Audit is launched once and if it
-succeeded executed new action plan list will be provided; CONTINUOUS Audit
-creates action plans with specified interval (in seconds or cron format, cron
-inteval can be used like: `*/5 * * * *`), if action plan
-has been created, all previous action plans get CANCELLED state;
-EVENT audit is launched when receiving webhooks API.
+Audit and interval (in case of CONTINUOUS type). There is two types of Audit:
+ONESHOT and CONTINUOUS. Oneshot Audit is launched once and if it succeeded
+executed new action plan list will be provided. Continuous Audit creates
+action plans with specified interval (in seconds); if action plan
+has been created, all previous action plans get CANCELLED state.
 
 A message is sent on the :ref:`AMQP bus <amqp_bus_definition>` which triggers
 the Audit in the
@@ -481,4 +461,4 @@ change to a new value:
 
 
 
-.. _Watcher API: https://docs.openstack.org/api-ref/resource-optimization/
+.. _Watcher API: webapi/v1.html

doc/source/conf.py

@@ -14,6 +14,7 @@
 import os
 import sys
 
+from watcher import version as watcher_version
 from watcher import objects
 
 objects.register_all()
@@ -31,16 +32,17 @@ sys.path.insert(0, os.path.abspath('./'))
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'oslo_config.sphinxext',
+    'sphinx.ext.autodoc',
     'sphinx.ext.viewcode',
     'sphinxcontrib.httpdomain',
     'sphinxcontrib.pecanwsme.rest',
     'stevedore.sphinxext',
+    'wsmeext.sphinxext',
     'ext.term',
     'ext.versioned_notifications',
     'oslo_config.sphinxconfiggen',
     'openstackdocstheme',
     'sphinx.ext.napoleon',
-    'sphinxcontrib.rsvgconverter',
 ]
 
 wsme_protocols = ['restjson']
@@ -49,6 +51,10 @@ config_generator_config_file = [(
     '_static/watcher')]
 sample_config_basename = 'watcher'
 
+# autodoc generation is a bit aggressive and a nuisance when doing heavy
+# text edit cycles.
+# execute "export SPHINX_DEBUG=1" in your terminal to disable
+
 # The suffix of source filenames.
 source_suffix = '.rst'
 
@@ -59,6 +65,16 @@ master_doc = 'index'
 project = u'Watcher'
 copyright = u'OpenStack Foundation'
 
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+# The full version, including alpha/beta/rc tags.
+release = watcher_version.version_info.release_string()
+# The short X.Y version.
+version = watcher_version.version_string
+
 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['watcher.']
 
@@ -83,7 +99,7 @@ add_module_names = True
 suppress_warnings = ['app.add_directive']
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'native'
+pygments_style = 'sphinx'
 
 # -- Options for man page output --------------------------------------------
 
@@ -114,34 +130,22 @@ html_theme = 'openstackdocs'
 # Output file base name for HTML help builder.
 htmlhelp_basename = '%sdoc' % project
 
+html_last_updated_fmt = '%Y-%m-%d %H:%M'
 
 #openstackdocstheme options
-openstackdocs_repo_name = 'openstack/watcher'
-openstackdocs_pdf_link = True
-openstackdocs_auto_name = False
-openstackdocs_bug_project = 'watcher'
-openstackdocs_bug_tag = ''
+repository_name = 'openstack/watcher'
+bug_project = 'watcher'
+bug_tag = ''
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass
 # [howto/manual]).
 latex_documents = [
     ('index',
-     'doc-watcher.tex',
-     u'Watcher Documentation',
+     '%s.tex' % project,
+     u'%s Documentation' % project,
      u'OpenStack Foundation', 'manual'),
 ]
 
-# If false, no module index is generated.
-latex_domain_indices = False
-
-latex_elements = {
-    'makeindex': '',
-    'printindex': '',
-    'preamble': r'\setcounter{tocdepth}{3}',
-}
-
-# Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664
-latex_use_xindy = False
 # Example configuration for intersphinx: refer to the Python standard library.
 # intersphinx_mapping = {'http://docs.python.org/': None}

doc/source/configuration/configuring.rst

@@ -159,7 +159,7 @@ MySQL database that is used by other OpenStack services.
    ``watcher`` user. Replace WATCHER_DBPASSWORD
    with the actual password::
 
-    # mysql
+    $ mysql -u root -p
 
     mysql> CREATE DATABASE watcher CHARACTER SET utf8;
     mysql> GRANT ALL PRIVILEGES ON watcher.* TO 'watcher'@'localhost' \
@@ -178,7 +178,7 @@ You can easily generate and update a sample configuration file
 named :ref:`watcher.conf.sample <watcher_sample_configuration_files>` by using
 these following commands::
 
-    $ git clone https://opendev.org/openstack/watcher.git
+    $ git clone https://git.openstack.org/openstack/watcher
     $ cd watcher/
     $ tox -e genconfig
     $ vi etc/watcher/watcher.conf.sample
@@ -403,22 +403,26 @@ Watcher can consume notifications generated by the Nova services, in order to
 build or update, in real time, its cluster data model related to computing
 resources.
 
-Nova emits unversioned(legacy) and versioned notifications on different
-topics. Because legacy notifications will be deprecated, Watcher consumes
-Nova versioned notifications.
+Nova publishes, by default, notifications on ``notifications`` AMQP queue
+(configurable) and ``versioned_notifications`` AMQP queue (not
+configurable). ``notifications`` queue is mainly used by ceilometer, so we can
+not use it. And some events, related to nova-compute service state, are only
+sent into the ``versioned_notifications`` queue.
 
-  * In the file ``/etc/nova/nova.conf``, the value of driver in the section
-    ``[oslo_messaging_notifications]`` can't be noop, and the value of
-    notification_format in the section ``[notifications]``
-    should be both or versioned ::
+By default, Watcher listens to AMQP queues named ``watcher_notifications``
+and ``versioned_notifications``. So you have to update the Nova
+configuration file on controller and compute nodes, in order
+to Watcher receives Nova notifications in ``watcher_notifications`` as well.
+
+  * In the file ``/etc/nova/nova.conf``, update the section
+    ``[oslo_messaging_notifications]``, by redefining the list of topics
+    into which Nova services will publish events ::
 
       [oslo_messaging_notifications]
       driver = messagingv2
+      topics = notifications,watcher_notifications
 
-      ...
-
-      [notifications]
-      notification_format = both
+  * Restart the Nova services.
 
 
 Configure Cinder Notifications

doc/source/contributor/api_microversion_history.rst

@@ -1 +0,0 @@
-.. include:: ../../../watcher/api/controllers/rest_api_version_history.rst

doc/source/contributor/concurrency.rst

@@ -1,248 +0,0 @@
-===========
-Concurrency
-===========
-
-Introduction
-************
-
-Modern processors typically contain multiple cores all capable of executing
-instructions in parallel. Ensuring applications can fully utilize modern
-underlying hardware requires developing with these concepts in mind. The
-OpenStack foundation maintains a number of libraries to facilitate this
-utilization, combined with constructs like CPython's GIL_ the proper use of
-these concepts becomes more straightforward compared to other programming
-languages.
-
-The primary libraries maintained by OpenStack to facilitate concurrency are
-futurist_ and taskflow_. Here futurist is a more straightforward and
-lightweight library while taskflow is more advanced supporting features like
-rollback mechanisms. Within Watcher both libraries are used to facilitate
-concurrency.
-
-.. _GIL: https://wiki.python.org/moin/GlobalInterpreterLock
-.. _futurist: https://docs.openstack.org/futurist/latest/
-.. _taskflow: https://docs.openstack.org/taskflow/latest/
-
-Threadpool
-**********
-
-A threadpool is a collection of one or more threads typically called *workers*
-to which tasks can be submitted. These submitted tasks will be scheduled by a
-threadpool and subsequently executed. In the case of Python tasks typically are
-bounded or unbounded methods while other programming languages like Java
-require implementing an interface.
-
-The order and amount of concurrency with which these tasks are executed is up
-to the threadpool to decide. Some libraries like taskflow allow for either
-strong or loose ordering of tasks while others like futurist might only support
-loose ordering. Taskflow supports building tree-based hierarchies of dependent
-tasks for example.
-
-Upon submission of a task to a threadpool a so called future_ is returned.
-These objects allow to determine information about the task such as if it is
-currently being executed or if it has finished execution. When the task has
-finished execution the future can also be used to retrieve what was returned by
-the method.
-
-Some libraries like futurist provide synchronization primitives for collections
-of futures such as wait_for_any_. The following sections will cover different
-types of concurrency used in various services of Watcher.
-
-.. _future: https://docs.python.org/3/library/concurrent.futures.html
-.. _wait_for_any: https://docs.openstack.org/futurist/latest/reference/index.html#waiters
-
-
-Decision engine concurrency
-***************************
-
-The concurrency in the decision engine is governed by two independent
-threadpools. Both of these threadpools are GreenThreadPoolExecutor_ from the
-futurist_ library. One of these is used automatically and most contributors
-will not interact with it while developing new features. The other threadpool
-can frequently be used while developing new features or updating existing ones.
-It is known as the DecisionEngineThreadpool and allows to achieve performance
-improvements in network or I/O bound operations.
-
-.. _GreenThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#executors
-
-AuditEndpoint
-#############
-
-The first threadpool is used to allow multiple audits to be run in parallel.
-In practice, however, only one audit can be run in parallel. This is due to
-the data model used by audits being a singleton. To prevent audits destroying
-each others data model one must wait for the other to complete before being
-allowed to access this data model. A performance improvement could be achieved
-by being more intelligent in the use, caching and construction of these
-data models.
-
-DecisionEngineThreadPool
-########################
-
-The second threadpool is used for generic tasks, typically networking and I/O
-could benefit the most of this threadpool. Upon execution of an audit this
-threadpool can be utilized to retrieve information from the Nova compute
-service for instance. This second threadpool is a singleton and is shared
-amongst concurrently running audits as a result the amount of workers is static
-and independent from the amount of workers in the first threadpool. The use of
-the :class:`~.DecisionEngineThreadpool` while building the Nova compute data
-model is demonstrated to show how it can effectively be used.
-
-In the following example a reference to the
-:class:`~.DecisionEngineThreadpool` is stored in ``self.executor``. Here two
-tasks are submitted one with function ``self._collect_aggregates`` and the
-other function ``self._collect_zones``. With both ``self.executor.submit``
-calls subsequent arguments are passed to the function. All subsequent arguments
-are passed to the function being submitted as task following the common
-``(fn, *args, **kwargs)`` signature. One of the original signatures would be
-``def _collect_aggregates(host_aggregates, compute_nodes)`` for example.
-
-.. code-block:: python
-
-    zone_aggregate_futures = {
-        self.executor.submit(
-            self._collect_aggregates, host_aggregates, compute_nodes),
-        self.executor.submit(
-            self._collect_zones, availability_zones, compute_nodes)
-    }
-    waiters.wait_for_all(zone_aggregate_futures)
-
-The last statement of the example above waits on all futures to complete.
-Similarly, ``waiters.wait_for_any`` will wait for any future of the specified
-collection to complete. To simplify the usage of ``wait_for_any`` the
-:class:`~.DecisiongEngineThreadpool` defines a ``do_while_futures`` method.
-This method will iterate in a do_while loop over a collection of futures until
-all of them have completed. The advantage of ``do_while_futures`` is that it
-allows to immediately call a method as soon as a future finishes. The arguments
-for this callback method can be supplied when calling ``do_while_futures``,
-however, the first argument to the callback is always the future itself! If
-the collection of futures can safely be modified ``do_while_futures_modify``
-can be used and should have slightly better performance. The following example
-will show how ``do_while_futures`` is used in the decision engine.
-
-.. code-block:: python
-
-    # For every compute node from compute_nodes submit a task to gather the node it's information.
-    # List comprehension is used to store all the futures of the submitted tasks in node_futures.
-    node_futures = [self.executor.submit(
-        self.nova_helper.get_compute_node_by_name,
-        node, servers=True, detailed=True)
-        for node in compute_nodes]
-    LOG.debug("submitted {0} jobs".format(len(compute_nodes)))
-
-    future_instances = []
-    # do_while iterate over node_futures and upon completion of a future call
-    # self._compute_node_future with the future and future_instances as arguments.
-    self.executor.do_while_futures_modify(
-        node_futures, self._compute_node_future, future_instances)
-
-    # Wait for all instance jobs to finish
-    waiters.wait_for_all(future_instances)
-
-Finally, let's demonstrate how powerful this ``do_while_futures`` can be by
-showing what the ``compute_node_future`` callback does. First, it retrieves the
-result from the future and adds the compute node to the data model. Afterwards,
-it checks if the compute node has any associated instances and if so it submits
-an additional task to the :class:`~.DecisionEngineThreadpool`. The future is
-appended to the ``future_instances`` so ``waiters.wait_for_all`` can be called
-on this list. This is important as otherwise the building of the data model
-might return before all tasks for instances have finished.
-
-.. code-block:: python
-
-    # Get the result from the future.
-    node_info = future.result()[0]
-
-    # Filter out baremetal nodes.
-    if node_info.hypervisor_type == 'ironic':
-        LOG.debug("filtering out baremetal node: %s", node_info)
-        return
-
-    # Add the compute node to the data model.
-    self.add_compute_node(node_info)
-    # Get the instances from the compute node.
-    instances = getattr(node_info, "servers", None)
-    # Do not submit job if there are no instances on compute node.
-    if instances is None:
-        LOG.info("No instances on compute_node: {0}".format(node_info))
-        return
-    # Submit a job to retrieve detailed information about the instances.
-    future_instances.append(
-        self.executor.submit(
-            self.add_instance_node, node_info, instances)
-    )
-
-Without ``do_while_futures`` an additional ``waiters.wait_for_all`` would be
-required in between the compute node tasks and the instance tasks. This would
-cause the progress of the decision engine to stall as less and less tasks
-remain active before the instance tasks could be submitted. This demonstrates
-how ``do_while_futures`` can be used to achieve more constant utilization of
-the underlying hardware.
-
-Applier concurrency
-*******************
-
-The applier does not use the futurist_ GreenThreadPoolExecutor_ directly but
-instead uses taskflow_. However, taskflow still utilizes a greenthreadpool.
-This threadpool is initialized in the workflow engine called
-:class:`~.DefaultWorkFlowEngine`. Currently Watcher supports one workflow
-engine but the base class allows contributors to develop other workflow engines
-as well. In taskflow tasks are created using different types of flows such as a
-linear, unordered or a graph flow. The linear and graph flow allow for strong
-ordering between individual tasks and it is for this reason that the workflow
-engine utilizes a graph flow. The creation of tasks, subsequently linking them
-into a graph like structure and submitting them is shown below.
-
-.. code-block:: python
-
-    self.execution_rule = self.get_execution_rule(actions)
-    flow = gf.Flow("watcher_flow")
-    actions_uuid = {}
-    for a in actions:
-        task = TaskFlowActionContainer(a, self)
-        flow.add(task)
-        actions_uuid[a.uuid] = task
-
-    for a in actions:
-        for parent_id in a.parents:
-            flow.link(actions_uuid[parent_id], actions_uuid[a.uuid],
-                      decider=self.decider)
-
-    e = engines.load(
-        flow, executor='greenthreaded', engine='parallel',
-        max_workers=self.config.max_workers)
-    e.run()
-
-    return flow
-
-In the applier tasks are contained in a :class:`~.TaskFlowActionContainer`
-which allows them to trigger events in the workflow engine. This way the
-workflow engine can halt or take other actions while the action plan is being
-executed based on the success or failure of individual actions. However, the
-base workflow engine simply uses these notifies to store the result of
-individual actions in the database. Additionally, since taskflow uses a graph
-flow if any of the tasks would fail all childs of this tasks not be executed
-while ``do_revert`` will be triggered for all parents.
-
-.. code-block:: python
-
-    class TaskFlowActionContainer(...):
-        ...
-        def do_execute(self, *args, **kwargs):
-            ...
-            result = self.action.execute()
-            if result is True:
-                return self.engine.notify(self._db_action,
-                                          objects.action.State.SUCCEEDED)
-            else:
-                self.engine.notify(self._db_action,
-                                   objects.action.State.FAILED)
-
-    class BaseWorkFlowEngine(...):
-        ...
-        def notify(self, action, state):
-            db_action = objects.Action.get_by_uuid(self.context, action.uuid,
-                                                   eager=True)
-            db_action.state = state
-            db_action.save()
-            return db_action

doc/source/contributor/contributing.rst

@@ -1,111 +1,72 @@
-============================
-So You Want to Contribute...
-============================
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
 
-For general information on contributing to OpenStack, please check out the
-`contributor guide <https://docs.openstack.org/contributors/>`_ to get started.
-It covers all the basics that are common to all OpenStack projects:
-the accounts you need, the basics of interacting with our Gerrit review system,
-how we communicate as a community, etc.
+          https://creativecommons.org/licenses/by/3.0/
 
-Below will cover the more project specific information you need to get started
-with Watcher.
+.. _contributing:
 
-Communication
-~~~~~~~~~~~~~~
-.. This would be a good place to put the channel you chat in as a project; when/
-   where your meeting is, the tags you prepend to your ML threads, etc.
+=======================
+Contributing to Watcher
+=======================
+
+If you're interested in contributing to the Watcher project,
+the following will help get you started.
+
+Contributor License Agreement
+-----------------------------
+
+.. index::
+   single: license; agreement
+
+In order to contribute to the Watcher project, you need to have
+signed OpenStack's contributor's agreement.
+
+.. seealso::
+
+   * https://docs.openstack.org/infra/manual/developers.html
+   * https://wiki.openstack.org/CLA
+
+LaunchPad Project
+-----------------
+
+Most of the tools used for OpenStack depend on a launchpad.net ID for
+authentication. After signing up for a launchpad account, join the
+"openstack" team to have access to the mailing list and receive
+notifications of important events.
+
+.. seealso::
+
+   * https://launchpad.net
+   * https://launchpad.net/watcher
+   * https://launchpad.net/openstack
+
+
+Project Hosting Details
+-----------------------
+
+Bug tracker
+    https://launchpad.net/watcher
+
+Mailing list (prefix subjects with ``[watcher]`` for faster responses)
+    http://lists.openstack.org/pipermail/openstack-dev/
+
+Wiki
+    https://wiki.openstack.org/Watcher
+
+Code Hosting
+    https://git.openstack.org/cgit/openstack/watcher
+
+Code Review
+    https://review.openstack.org/#/q/status:open+project:openstack/watcher,n,z
 
 IRC Channel
     ``#openstack-watcher`` (changelog_)
 
-Mailing list(prefix subjects with ``[watcher]``)
-    http://lists.openstack.org/pipermail/openstack-discuss/
-
 Weekly Meetings
-    Bi-weekly, on Wednesdays at 08:00 UTC on odd weeks in the
-    ``#openstack-meeting-alt`` IRC channel (`meetings logs`_)
-
-Meeting Agenda
-    https://wiki.openstack.org/wiki/Watcher_Meeting_Agenda
+    On Wednesdays at 14:00 UTC on even weeks in the ``#openstack-meeting-4``
+    IRC channel, 08:00 UTC on odd weeks in the ``#openstack-meeting-alt``
+    IRC channel (`meetings logs`_)
 
 .. _changelog: http://eavesdrop.openstack.org/irclogs/%23openstack-watcher/
 .. _meetings logs:  http://eavesdrop.openstack.org/meetings/watcher/
-
-Contacting the Core Team
-~~~~~~~~~~~~~~~~~~~~~~~~~
-.. This section should list the core team, their irc nicks, emails, timezones etc.
-   If all this info is maintained elsewhere (i.e. a wiki), you can link to that
-   instead of enumerating everyone here.
-
-+--------------------+---------------+------------------------------------+
-| Name               | IRC           | Email                              |
-+====================+===============+====================================+
-| `Li Canwei`_       | licanwei      | li.canwei2@zte.com.cn              |
-+--------------------+---------------+------------------------------------+
-| `chen ke`_         | chenke        | chen.ke14@zte.com.cn               |
-+--------------------+---------------+------------------------------------+
-| `Corne Lukken`_    | dantalion     | info@dantalion.nl                  |
-+--------------------+---------------+------------------------------------+
-| `su zhengwei`_     | suzhengwei    | sugar-2008@163.com                 |
-+--------------------+---------------+------------------------------------+
-| `Yumeng Bao`_      | Yumeng        | yumeng_bao@yahoo.com               |
-+--------------------+---------------+------------------------------------+
-
-.. _Corne Lukken: https://launchpad.net/~dantalion
-.. _Li Canwei: https://launchpad.net/~li-canwei2
-.. _su zhengwei: https://launchpad.net/~sue.sam
-.. _Yumeng Bao: https://launchpad.net/~yumeng-bao
-.. _chen ke: https://launchpad.net/~chenker
-
-New Feature Planning
-~~~~~~~~~~~~~~~~~~~~
-.. This section is for talking about the process to get a new feature in. Some
-   projects use blueprints, some want specs, some want both! Some projects
-   stick to a strict schedule when selecting what new features will be reviewed
-   for a release.
-
-New feature will be discussed via IRC or ML (with [Watcher] prefix).
-Watcher team uses blueprints in `Launchpad`_ to manage the new features.
-
-.. _Launchpad: https://launchpad.net/watcher
-
-Task Tracking
-~~~~~~~~~~~~~~
-.. This section is about where you track tasks- launchpad? storyboard?
-   is there more than one launchpad project? what's the name of the project
-   group in storyboard?
-
-We track our tasks in Launchpad.
-If you're looking for some smaller, easier work item to pick up and get started
-on, search for the 'low-hanging-fruit' tag.
-
-.. NOTE: If your tag is not 'low-hanging-fruit' please change the text above.
-
-Reporting a Bug
-~~~~~~~~~~~~~~~
-.. Pretty self explanatory section, link directly to where people should report bugs for
-   your project.
-
-You found an issue and want to make sure we are aware of it? You can do so
-`HERE`_.
-
-.. _HERE: https://bugs.launchpad.net/watcher
-
-Getting Your Patch Merged
-~~~~~~~~~~~~~~~~~~~~~~~~~
-.. This section should have info about what it takes to get something merged.
-   Do you require one or two +2's before +W? Do some of your repos require
-   unit test changes with all patches? etc.
-
-Due to the small number of core reviewers of the Watcher project,
-we only need one +2 before +W (merge). All patches excepting for documentation
-or typos fixes must have unit test.
-
-Project Team Lead Duties
-------------------------
-.. this section is where you can put PTL specific duties not already listed in
-   the common PTL guide (linked below)  or if you already have them written
-   up elsewhere, you can link to that doc here.
-
-All common PTL duties are enumerated here in the `PTL guide <https://docs.openstack.org/project-team-guide/ptl.html>`_.

doc/source/contributor/devstack.rst

@@ -19,7 +19,7 @@ model. To enable the Watcher plugin with DevStack, add the following to the
 `[[local|localrc]]` section of your controller's `local.conf` to enable the
 Watcher plugin::
 
-    enable_plugin watcher https://opendev.org/openstack/watcher
+    enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 For more detailed instructions, see `Detailed DevStack Instructions`_. Check
 out the `DevStack documentation`_ for more information regarding DevStack.
@@ -27,36 +27,6 @@ out the `DevStack documentation`_ for more information regarding DevStack.
 .. _PluginModelDocs: https://docs.openstack.org/devstack/latest/plugins.html
 .. _DevStack documentation: https://docs.openstack.org/devstack/latest
 
-Quick Devstack Instructions with Datasources
-============================================
-
-Watcher requires a datasource to collect metrics from compute nodes and
-instances in order to execute most strategies. To enable this a
-`[[local|localrc]]` to setup DevStack for some of the supported datasources
-is provided. These examples specify the minimal configuration parameters to
-get both Watcher and the datasource working but can be expanded is desired.
-
-Gnocchi
--------
-
-With the Gnocchi datasource most of the metrics for compute nodes and
-instances will work with the provided configuration but metrics that
-require Ironic such as `host_airflow and` `host_power` will still be
-unavailable as well as `instance_l3_cpu_cache`::
-
-        [[local|localrc]]
-        enable_plugin watcher https://opendev.org/openstack/watcher
-
-        enable_plugin ceilometer https://opendev.org/openstack/ceilometer.git
-        CEILOMETER_BACKEND=gnocchi
-
-        enable_plugin aodh https://opendev.org/openstack/aodh
-        enable_plugin panko https://opendev.org/openstack/panko
-
-        [[post-config|$NOVA_CONF]]
-        [DEFAULT]
-        compute_monitors=cpu.virt_driver
-
 Detailed DevStack Instructions
 ==============================
 
@@ -74,7 +44,7 @@ Detailed DevStack Instructions
 
        sudo apt-get update
        sudo apt-get install git
-       git clone https://opendev.org/openstack/devstack.git
+       git clone https://git.openstack.org/openstack-dev/devstack
        sudo ./devstack/tools/create-stack-user.sh
 
    Now you have a stack user that is used to run the DevStack processes. You
@@ -86,7 +56,7 @@ Detailed DevStack Instructions
 
        sudo su stack
        cd ~
-       git clone https://opendev.org/openstack/devstack.git
+       git clone https://git.openstack.org/openstack-dev/devstack
 
 #. For each compute node, copy the provided `local.conf.compute`_ example file
    to the compute node's system at ~/devstack/local.conf. Make sure the

doc/source/contributor/environment.rst

@@ -16,7 +16,7 @@ for development purposes.
 To install Watcher from packaging, refer instead to Watcher `User
 Documentation`_.
 
-.. _`Git Repository`: https://opendev.org/openstack/watcher
+.. _`Git Repository`: https://git.openstack.org/cgit/openstack/watcher
 .. _`User Documentation`: https://docs.openstack.org/watcher/latest/
 
 Prerequisites
@@ -47,7 +47,7 @@ Make a clone of the code from our `Git repository`:
 
 .. code-block:: bash
 
-    $ git clone https://opendev.org/openstack/watcher.git
+    $ git clone https://git.openstack.org/openstack/watcher.git
 
 When that is complete, you can:
 
@@ -99,7 +99,7 @@ useful to keep a clean environment for working on Watcher.
 .. code-block:: bash
 
     $ mkvirtualenv watcher
-    $ git clone https://opendev.org/openstack/watcher.git
+    $ git clone https://git.openstack.org/openstack/watcher
 
     # Use 'python setup.py' to link Watcher into Python's site-packages
     $ cd watcher && python setup.py install

doc/source/contributor/index.rst

@@ -1,12 +1,8 @@
-==================
-Contribution Guide
-==================
-
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
-   contributing
    environment
    devstack
+   notifications
    testing
    rally_link

doc/source/contributor/plugin/index.rst

@@ -1,7 +1,3 @@
-============
-Plugin Guide
-============
-
 .. toctree::
    :maxdepth: 1
 

doc/source/contributor/plugin/strategy-plugin.rst

@@ -245,16 +245,15 @@ Querying metrics
 
 A large set of metrics, generated by OpenStack modules, can be used in your
 strategy implementation. To collect these metrics, Watcher provides a
-`DataSourceManager`_ for two data sources which are `Ceilometer`_
-(with `Gnocchi`_ as API) and `Monasca`_. If you wish to query metrics from a
-different data source, you can implement your own and use it via
-DataSourceManager from within your new strategy. Indeed, strategies in Watcher
-have the cluster data models decoupled from the data sources which means that
-you may keep the former while changing the latter. The recommended way for you
-to support a new data source is to implement a new helper that would
-encapsulate within separate methods the queries you need to perform. To then
-use it, you would just have to add it to appropriate watcher_strategies.*
-section in config file.
+`Helper`_ for two data sources which are `Ceilometer`_ and `Monasca`_. If you
+wish to query metrics from a different data source, you can implement your own
+and directly use it from within your new strategy. Indeed, strategies in
+Watcher have the cluster data models decoupled from the data sources which
+means that you may keep the former while changing the latter.
+The recommended way for you to support a new data source is to implement a new
+helper that would encapsulate within separate methods the queries you need to
+perform. To then use it, you would just have to instantiate it within your
+strategy.
 
 If you want to use Ceilometer but with your own metrics database backend,
 please refer to the `Ceilometer developer guide`_. The list of the available
@@ -264,45 +263,52 @@ requires new metrics not covered by Ceilometer, you can add them through a
 `Ceilometer plugin`_.
 
 
-.. _`DataSourceManager`: https://github.com/openstack/watcher/blob/master/watcher/datasource/manager.py
+.. _`Helper`: https://github.com/openstack/watcher/blob/master/watcher/datasource/ceilometer.py
 .. _`Ceilometer developer guide`: https://docs.openstack.org/ceilometer/latest/contributor/architecture.html#storing-accessing-the-data
 .. _`Ceilometer`: https://docs.openstack.org/ceilometer/latest
 .. _`Monasca`: https://github.com/openstack/monasca-api/blob/master/docs/monasca-api-spec.md
 .. _`here`: https://docs.openstack.org/ceilometer/latest/contributor/install/dbreco.html#choosing-a-database-backend
 .. _`Ceilometer plugin`: https://docs.openstack.org/ceilometer/latest/contributor/plugins.html
 .. _`Ceilosca`: https://github.com/openstack/monasca-ceilometer/blob/master/ceilosca/ceilometer/storage/impl_monasca.py
-.. _`Gnocchi`: https://gnocchi.xyz/
 
 Read usage metrics using the Watcher Datasource Helper
 ------------------------------------------------------
 
-The following code snippet shows how datasource_backend is defined:
+The following code snippet shows how to invoke a Datasource Helper class:
 
 .. code-block:: py
 
-    from watcher.datasource import manager as ds_manager
+    from watcher.datasource import ceilometer as ceil
+    from watcher.datasource import monasca as mon
 
     @property
-    def datasource_backend(self):
-        if not self._datasource_backend:
+    def ceilometer(self):
+        if self._ceilometer is None:
+            self._ceilometer = ceil.CeilometerHelper(osc=self.osc)
+        return self._ceilometer
 
-            # Load the global preferred datasources order but override it
-            # if the strategy has a specific datasources config
-            datasources = CONF.watcher_datasources
-            if self.config.datasources:
-                datasources = self.config
-
-            self._datasource_backend = ds_manager.DataSourceManager(
-                config=datasources,
-                osc=self.osc
-            ).get_backend(self.DATASOURCE_METRICS)
-        return self._datasource_backend
+    @property
+    def monasca(self):
+        if self._monasca is None:
+            self._monasca = mon.MonascaHelper(osc=self.osc)
+        return self._monasca
 
 Using that you can now query the values for that specific metric:
 
 .. code-block:: py
 
-    avg_meter = self.datasource_backend.statistic_aggregation(
-        instance.uuid, 'cpu_util', self.periods['instance'],
-        self.granularity,
-        aggregation=self.aggregation_method['instance'])
+        if self.config.datasource == "ceilometer":
+            resource_id = "%s_%s" % (node.uuid, node.hostname)
+            return self.ceilometer.statistic_aggregation(
+                resource_id=resource_id,
+                meter_name='compute.node.cpu.percent',
+                period="7200",
+                aggregate='avg',
+            )
+        elif self.config.datasource == "monasca":
+            statistics = self.monasca.statistic_aggregation(
+                meter_name='compute.node.cpu.percent',
+                dimensions=dict(hostname=node.uuid),
+                period=7200,
+                aggregate='avg'
+            )

doc/source/contributor/testing.rst

@@ -4,41 +4,48 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-=================
-Developer Testing
-=================
+=======
+Testing
+=======
 
 .. _unit_tests:
 
 Unit tests
 ==========
 
-All unit tests should be run using `tox`_. Before running the unit tests, you
-should download the latest `watcher`_ from the github. To run the same unit
-tests that are executing onto `Gerrit`_ which includes ``py36``, ``py37`` and
-``pep8``, you can issue the following command::
+All unit tests should be run using `tox`_. To run the same unit tests that are
+executing onto `Gerrit`_ which includes ``py35``, ``py27`` and ``pep8``, you
+can issue the following command::
 
-    $ git clone https://opendev.org/openstack/watcher
-    $ cd watcher
-    $ pip install tox
-    $ tox
+    $ workon watcher
+    (watcher) $ pip install tox
+    (watcher) $ cd watcher
+    (watcher) $ tox
 
-If you only want to run one of the aforementioned, you can then issue one of
+If you want to only run one of the aforementioned, you can then issue one of
 the following::
 
-    $ tox -e py36
-    $ tox -e py37
-    $ tox -e pep8
+    $ workon watcher
+    (watcher) $ tox -e py35
+    (watcher) $ tox -e py27
+    (watcher) $ tox -e pep8
 
 .. _tox: https://tox.readthedocs.org/
-.. _watcher: https://opendev.org/openstack/watcher
-.. _Gerrit: https://review.opendev.org/
+.. _Gerrit: https://review.openstack.org/
 
-If you only want to run specific unit test code and don't like to waste time
-waiting for all unit tests to execute, you can add parameters ``--`` followed
-by a regex string::
+You may pass options to the test programs using positional arguments. To run a
+specific unit test, you can pass extra options to `os-testr`_ after putting
+the ``--`` separator. So using the ``-r`` option followed by a regex string,
+you can run the desired test::
 
-    $ tox -e py37 -- watcher.tests.api
+    $ workon watcher
+    (watcher) $ tox -e py27 -- -r watcher.tests.api
+
+.. _os-testr: https://docs.openstack.org/os-testr/latest
+
+When you're done, deactivate the virtualenv::
+
+    $ deactivate
 
 .. _tempest_tests:
 
@@ -48,4 +55,4 @@ Tempest tests
 Tempest tests for Watcher has been migrated to the external repo
 `watcher-tempest-plugin`_.
 
-.. _watcher-tempest-plugin: https://opendev.org/openstack/watcher-tempest-plugin
+.. _watcher-tempest-plugin: https://github.com/openstack/watcher-tempest-plugin

doc/source/datasources/grafana.rst

@@ -1,426 +0,0 @@
-==================
-Grafana datasource
-==================
-
-Synopsis
---------
-
-Grafana can interface with many different types of storage backends that
-Grafana calls datasources_. Since the term datasources causes significant
-confusion by overlapping definitions used in Watcher these **datasources are
-called projects instead**. Some examples of supported projects are InfluxDB
-or Elasticsearch while others might be more familiar such as Monasca or
-Gnocchi. The Grafana datasource provides the functionality to retrieve metrics
-from Grafana for different projects. This functionality is achieved by using
-the proxy interface exposed in Grafana to communicate with Grafana projects
-directly.
-
-Background
-**********
-
-Since queries to retrieve metrics from Grafana are proxied to the project the
-format of these queries will change significantly depending on the type of
-project. The structure of the projects themselves will also change
-significantly as they are structured by users and administrators. For instance,
-some developers might decide to store metrics about compute_nodes in MySQL and
-use the UUID as primary key while others use InfluxDB and use the hostname as
-primary key. Furthermore, datasources in Watcher should return metrics in
-specific units strictly defined in the baseclass_ depending on how the units
-are stored in the projects they might require conversion before being returned.
-The flexible configuration parameters of the Grafana datasource allow to
-specify exactly how the deployment is configured and this will enable to
-correct retrieval of metrics and with the correct units.
-
-.. _datasources: https://grafana.com/plugins?type=datasource
-.. _baseclass: https://github.com/openstack/watcher/blob/584eeefdc8/watcher/datasources/base.py
-
-Requirements
-------------
-
-The use of the Grafana datasource requires a reachable Grafana endpoint and an
-authentication token for access to the desired projects. The projects behind
-Grafana will need to contain the metrics for compute_nodes_ or instances_ and
-these need to be identifiable by an attribute of the Watcher datamodel_ for
-instance hostname or UUID.
-
-.. _compute_nodes: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/element/node.py
-.. _instances: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/element/instance.py
-.. _datamodel: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/element
-
-Limitations
-***********
-
-* Only the InfluxDB project is currently supported [#f1]_.
-* All metrics must be retrieved from the same Grafana endpoint (same URL).
-* All metrics must be retrieved with the same authentication token.
-
-.. [#f1] A base class for projects is available_ and easily extensible.
-.. _available: https://review.opendev.org/#/c/649341/24/watcher/datasources/grafana_translator/base.py
-
-Configuration
--------------
-
-Several steps are required in order to use the Grafana datasource, Most steps
-are related configuring Watcher to match the deployed Grafana setup such as
-queries proxied to the project or the type of project for any given metric.
-Most of the configuration can either be supplied via the traditional
-configuration file or in a `special yaml`_ file.
-
-.. _special yaml: https://specs.openstack.org/openstack/watcher-specs/specs/train/approved/file-based-metricmap.html
-
-token
-*****
-
-First step is to generate an access token with access to the required projects.
-This can be done from the api_ or from the web interface_. Tokens generated
-from the web interface will have the same access to projects as the user that
-created them while using the cli allows to generate a key for a specific
-role.The token will only be displayed once so store it well. This token will go
-into the configuration file later and this parameter can not be placed in the
-yaml.
-
-.. _api: https://grafana.com/docs/http_api/auth/#create-api-key
-.. _interface: https://grafana.com/docs/http_api/auth/#create-api-token
-
-base_url
-********
-
-Next step is supplying the base url of the Grafana endpoint. The base url
-parameter will need to specify the type of http protocol and the use of
-plain text http is strongly discouraged due to the transmission of the access
-token. Additionally the path to the proxy interface needs to be supplied as
-well in case Grafana is placed in a sub directory of the web server. An example
-would be: `https://mygrafana.org/api/datasource/proxy/` were
-`/api/datasource/proxy` is the default path without any subdirectories.
-Likewise, this parameter can not be placed in the yaml.
-
-To prevent many errors from occurring and potentially filing the logs files it
-is advised to specify the desired datasource in the configuration as it would
-prevent the datasource manager from having to iterate and try possible
-datasources with the launch of each audit. To do this specify `datasources` in
-the `[watcher_datasources]` group.
-
-The current configuration that is required to be placed in the traditional
-configuration file would look like the following:
-
-.. code-block:: shell
-
-    [grafana_client]
-    token = 0JLbF0oB4R3Q2Fl337Gh4Df5VN12D3adBE3f==
-    base_url = https://mygranfa.org/api/datasource/proxy
-
-    [watcher_datasources]
-    datasources = grafana
-
-metric parameters
-*****************
-
-The last five remaining configuration parameters can all be placed both in the
-traditional configuration file or in the yaml, however, it is not advised to
-mix and match but in the case it does occur the yaml would override the
-settings from the traditional configuration file. All five of these parameters
-are dictionaries mapping specific metrics to a configuration parameter. For
-instance the `project_id_map` will specify the specific project id in Grafana
-to be used. The parameters are named as follow:
-
-* project_id_map
-* database_map
-* translator_map
-* attribute_map
-* query_map
-
-These five parameters are named differently if configured using the yaml
-configuration file. The parameters are named as follows and are in
-identical order as to the list of the traditional configuration file:
-
-* project
-* db
-* translator
-* attribute
-* query
-
-When specified in the yaml the parameters are no longer dictionaries instead
-each parameter needs to be defined per metric as sub-parameters. Examples of
-these parameters configured for both the yaml and traditional configuration
-are described at the end of this document.
-
-project_id
-**********
-
-The project id's can only be determined by someone with the admin role in
-Grafana as that role is required to open the list of projects. The list of
-projects can be found on `/datasources` in the web interface but
-unfortunately it does not immediately display the project id. To display
-the id one can best hover the mouse over the projects and the url will show the
-project id's for example `/datasources/edit/7563`. Alternatively the entire
-list of projects can be retrieved using the `REST api`_. To easily make
-requests to the REST api a tool such as Postman can be used.
-
-.. _REST api: https://grafana.com/docs/http_api/data_source/#get-all-datasources
-
-database
-********
-
-The database is the parameter for the schema / database that is actually
-defined in the project. For instance, if the project would be based on MySQL
-this is were the name of schema used within the MySQL server would be
-specified. For many different projects it is possible to list all the databases
-currently available. Tools like Postman can be used to list all the available
-databases per project. For InfluxDB based projects this would be with the
-following path and query, however be sure to construct these request in Postman
-as the header needs to contain the authorization token:
-
-.. code-block:: shell
-
-    https://URL.DOMAIN/api/datasources/proxy/PROJECT_ID/query?q=SHOW%20DATABASES
-
-translator
-**********
-
-Each translator is for a specific type of project will have a uniquely
-identifiable name and the baseclass allows to easily support new types of
-projects such as elasticsearch or prometheus. Currently only InfluxDB based
-projects are supported as a result the only valid value for this parameter is `
-influxdb`.
-
-attribute
-*********
-
-The attribute parameter specifies which attribute to use from Watcher's
-data model in order to construct the query. The available attributes differ
-per type of object in the data model but the following table shows the
-attributes for ComputeNodes, Instances and IronicNodes.
-
-+-----------------+-----------------+--------------------+
-| ComputeNode     | Instance        | IronicNode         |
-+=================+=================+====================+
-| uuid            | uuid            | uuid               |
-+-----------------+-----------------+--------------------+
-| id              | name            | human_id           |
-+-----------------+-----------------+--------------------+
-| hostname        | project_id      | power_state        |
-+-----------------+-----------------+--------------------+
-| status          | watcher_exclude | maintenance        |
-+-----------------+-----------------+--------------------+
-| disabled_reason | locked          | maintenance_reason |
-+-----------------+-----------------+--------------------+
-| state           | metadata        | extra              |
-+-----------------+-----------------+--------------------+
-| memory          | state           |                    |
-+-----------------+-----------------+--------------------+
-| disk            | memory          |                    |
-+-----------------+-----------------+--------------------+
-| disk_capacity   | disk            |                    |
-+-----------------+-----------------+--------------------+
-| vcpus           | disk_capacity   |                    |
-+-----------------+-----------------+--------------------+
-|                 | vcpus           |                    |
-+-----------------+-----------------+--------------------+
-
-Many if not all of these attributes map to attributes of the objects that are
-fetched from clients such as Nova. To see how these attributes are put into the
-data model the following source files can be analyzed for Nova_ and Ironic_.
-
-.. _Nova: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/collector/nova.py#L304
-.. _Ironic: https://opendev.org/openstack/watcher/src/branch/master/watcher/decision_engine/model/collector/ironic.py#L85
-
-query
-*****
-
-The query is the single most important parameter it will be passed to the
-project and should return the desired metric for the specific host and return
-the value in the correct unit. The units for all available metrics are
-documented in the `datasource baseclass`_. This might mean the query specified
-in this parameter is responsible for converting the unit. The following query
-demonstrates how such a conversion could be achieved and demonstrates the
-conversion from bytes to megabytes.
-
-.. code-block:: shell
-
-    SELECT value/1000000 FROM memory...
-
-Queries will be formatted using the .format string method within Python. This
-format will currently have give attributes exposed to it labeled `{0}` to
-`{4}`. Every occurrence of these characters within the string will be replaced
-with the specific attribute.
-
-- {0} is the aggregate typically `mean`, `min`, `max` but `count` is also
-  supported.
-- {1} is the attribute as specified in the attribute parameter.
-- {2} is the period of time to aggregate data over in seconds.
-- {3} is the granularity or the interval between data points in seconds.
-- {4} is translator specific and in the case of InfluxDB it will be used for
-  retention_periods.
-
-**InfluxDB**
-
-Constructing the queries or rather anticipating how the results should look to
-be correctly interpreted by Watcher can be a challenge. The following json
-example demonstrates how what the result should look like and the query used to
-get this result.
-
-.. code-block:: json
-
-    {
-    "results": [
-        {
-            "statement_id": 0,
-            "series": [
-                {
-                    "name": "vmstats",
-                    "tags": {
-                        "host": "autoserver01"
-                    },
-                    "columns": [
-                        "time",
-                        "mean"
-                    ],
-                    "values": [
-                        [
-                            1560848284284,
-                            7680000
-                        ]
-                    ]
-                }
-            ]
-        }
-    ]
-    }
-
-.. code-block:: shell
-
-    SELECT {0}("{0}_value") FROM "vmstats" WHERE host =~ /^{1}$/ AND
-    "type_instance" =~ /^mem$/ AND time >= now() - {2}s GROUP BY host
-
-.. _datasource baseclass: https://opendev.org/openstack/watcher/src/branch/master/watcher/datasources/base.py
-
-Example configuration
----------------------
-
-The example configurations will show both how to achieve the entire
-configuration in the config file or use a combination of the regular file and
-yaml. Using yaml to define all the parameters for each metric is recommended
-since it has better human readability and supports mutli-line option
-definitions.
-
-Configuration file
-******************
-
-**It is important to note that the line breaks shown in between assignments of
-parameters can not be used in the actual configuration and these are simply here
-for readability reasons.**
-
-.. code-block:: shell
-
-    [grafana_client]
-    # Authentication token to gain access (string value)
-    # Note: This option can be changed without restarting.
-    token = eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk==
-
-    # first part of the url (including https:// or http://) up until project id
-    # part. Example: https://secure.org/api/datasource/proxy/ (string value)
-    # Note: This option can be changed without restarting.
-    base_url = https://monitoring-grafana.com/api/datasources/proxy/
-
-    # Project id as in url (integer value)
-    # Note: This option can be changed without restarting.
-    project_id_map = host_cpu_usage:1337,host_ram_usage:6969,
-    instance_cpu_usage:1337,instance_ram_usage:9696
-
-    # Mapping of grafana databases to datasource metrics. (dict value)
-    # Note: This option can be changed without restarting.
-    database_map = host_cpu_usage:monit_production,
-    host_ram_usage:monit_production,instance_cpu_usage:prod_cloud,
-    instance_ram_usage:prod_cloud
-
-    translator_map = host_cpu_usage:influxdb,host_ram_usage:influxdb,
-    instance_cpu_usage:influxdb,instance_ram_usage:influxdb
-
-    attribute_map = host_cpu_usage:hostname,host_ram_usage:hostname,
-    instance_cpu_usage:name,instance_ram_usage:name
-
-    query_map = host_cpu_usage:SELECT 100-{0}("{0}_value") FROM {4}.cpu WHERE
-    ("host" =~ /^{1}$/ AND "type_instance" =~/^idle$/ AND time > now()-{2}s),
-    host_ram_usage:SELECT {0}("{0}_value")/1000000 FROM {4}.memory WHERE
-     ("host" =~ /^{1}$/) AND "type_instance" =~ /^used$/ AND time >= now()-{2}s
-     GROUP BY "type_instance",instance_cpu_usage:SELECT {0}("{0}_value") FROM
-     "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^cpu$/ AND time >=
-     now() - {2}s GROUP BY host,instance_ram_usage:SELECT {0}("{0}_value") FROM
-     "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^mem$/ AND time >=
-     now() - {2}s GROUP BY host
-
-    [grafana_translators]
-
-    retention_periods = one_week:10080,one_month:302400,five_years:525600
-
-    [watcher_datasources]
-    datasources = grafana
-
-yaml
-****
-
-When using the yaml configuration file some parameters still need to be defined
-using the regular configuration such as the path for the yaml file these
-parameters are detailed below:
-
-.. code-block:: shell
-
-    [grafana_client]
-    token = eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk==
-
-    base_url = https://monitoring-grafana.com/api/datasources/proxy/
-
-    [watcher_datasources]
-    datasources = grafana
-
-    [watcher_decision_engine]
-    metric_map_path = /etc/watcher/metric_map.yaml
-
-Using the yaml allows to more effectively define the parameters per metric with
-greater human readability due to the availability of multi line options. These
-multi line options are demonstrated in the query parameters.
-
-.. code-block:: yaml
-
-  grafana:
-    host_cpu_usage:
-      project: 1337
-      db: monit_production
-      translator: influxdb
-      attribute: hostname
-      query: >
-          SELECT 100-{0}("{0}_value") FROM {4}.cpu
-          WHERE ("host" =~ /^{1}$/ AND "type_instance" =~/^idle$/ AND
-          time > now()-{2}s)
-    host_ram_usage:
-      project: 6969
-      db: monit_production
-      translator: influxdb
-      attribute: hostname
-      query: >
-          SELECT {0}("{0}_value")/1000000 FROM {4}.memory WHERE
-           ("host" =~ /^{1}$/) AND "type_instance" =~ /^used$/ AND time >=
-           now()-{2}s GROUP BY "type_instance"
-    instance_cpu_usage:
-      project: 1337
-      db: prod_cloud
-      translator: influxdb
-      attribute: name
-      query: >
-          SELECT {0}("{0}_value") FROM
-           "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^cpu$/ AND
-           time >= now() - {2}s GROUP BY host
-    instance_ram_usage:
-      project: 9696
-      db: prod_cloud
-      translator: influxdb
-      attribute: name
-      query: >
-          SELECT {0}("{0}_value") FROM
-           "vmstats" WHERE host =~ /^{1}$/ AND "type_instance" =~ /^mem$/ AND
-           time >= now() - {2}s GROUP BY host
-
-External Links
---------------
-
-- `List of Grafana datasources <https://grafana.com/plugins?type=datasource>`_

doc/source/datasources/index.rst

@@ -1,8 +0,0 @@
-Datasources
-===========
-
-.. toctree::
-   :glob:
-   :maxdepth: 1
-
-   ./*

doc/source/glossary.rst

@@ -8,7 +8,7 @@
 Glossary
 ========
 
-.. _glossary:
+.. glossary::
    :sorted:
 
 This page explains the different terms used in the Watcher system.

doc/source/image_src/plantuml/audit_state_machine.txt

@@ -2,7 +2,7 @@
 
 [*] --> PENDING: Audit requested by Administrator
 PENDING --> ONGOING: Audit request is received\nby the Watcher Decision Engine
-ONGOING --> FAILED: Audit fails\n(Exception occurred)
+ONGOING --> FAILED: Audit fails\n(no solution found, technical error, ...)
 ONGOING --> SUCCEEDED: The Watcher Decision Engine\ncould find at least one Solution
 ONGOING --> SUSPENDED: Administrator wants to\nsuspend the Audit
 SUSPENDED --> ONGOING: Administrator wants to\nresume the Audit

doc/source/index.rst

@@ -28,25 +28,101 @@ The documentation provided here is continually kept up-to-date based
 on the latest code, and may not represent the state of the project at any
 specific prior release.
 
-.. _watcher: https://opendev.org/openstack/watcher/
-.. _python-watcherclient: https://opendev.org/openstack/python-watcherclient/
-.. _watcher-dashboard: https://opendev.org/openstack/watcher-dashboard/
+.. _watcher: https://git.openstack.org/cgit/openstack/watcher/
+.. _python-watcherclient: https://git.openstack.org/cgit/openstack/python-watcherclient/
+.. _watcher-dashboard: https://git.openstack.org/cgit/openstack/watcher-dashboard/
 
-.. toctree::
-  :maxdepth: 2
+Developer Guide
+===============
 
-  architecture
-  contributor/index
-  install/index
-  admin/index
-  user/index
-  configuration/index
-  contributor/plugin/index
-  man/index
+Introduction
+------------
 
 .. toctree::
   :maxdepth: 1
 
-  API Reference <https://docs.openstack.org/api-ref/resource-optimization/>
-  Watcher API Microversion History </contributor/api_microversion_history>
   glossary
+  architecture
+  contributor/contributing
+
+
+Getting Started
+---------------
+
+.. toctree::
+  :maxdepth: 1
+
+  contributor/index
+
+Installation
+============
+.. toctree::
+  :maxdepth: 2
+
+  install/index
+
+Admin Guide
+===========
+
+.. toctree::
+  :maxdepth: 2
+
+  admin/index
+
+User Guide
+==========
+
+.. toctree::
+  :maxdepth: 2
+
+  user/index
+
+API References
+--------------
+
+.. toctree::
+  :maxdepth: 1
+
+  api/index
+
+Plugins
+-------
+
+.. toctree::
+  :maxdepth: 1
+
+  contributor/plugin/index
+
+Watcher Configuration Options
+=============================
+
+.. toctree::
+  :maxdepth: 2
+
+  configuration/index
+
+Watcher Manual Pages
+====================
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   man/index
+
+.. # NOTE(mriedem): This is the section where we hide things that we don't
+   # actually want in the table of contents but sphinx build would fail if
+   # they aren't in the toctree somewhere. For example, we hide api/autoindex
+   # since that's already covered with modindex below.
+.. toctree::
+   :hidden:
+
+   api/autoindex
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

doc/source/install/common_prerequisites.rst

@@ -11,7 +11,7 @@ you must create a database, service credentials, and API endpoints.
 
      .. code-block:: console
 
-        # mysql
+        $ mysql -u root -p
 
    * Create the ``watcher`` database:
 

doc/source/install/index.rst

@@ -1,6 +1,6 @@
-=============
-Install Guide
-=============
+===================================
+Infrastructure Optimization service
+===================================
 
 .. toctree::
    :maxdepth: 2

doc/source/install/install-ubuntu.rst

@@ -4,7 +4,7 @@ Install and configure for Ubuntu
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 This section describes how to install and configure the Infrastructure
-Optimization service for Ubuntu 16.04 (LTS).
+Optimization service for Ubuntu 14.04 (LTS).
 
 .. include:: common_prerequisites.rst
 
@@ -25,15 +25,10 @@ Install and configure components
 Finalize installation
 ---------------------
 
-Start the Infrastructure Optimization services and configure them to start when
-the system boots:
+Restart the Infrastructure Optimization services:
 
 .. code-block:: console
 
-   # systemctl enable watcher-api.service \
-     watcher-decision-engine.service \
-     watcher-applier.service
-
-   # systemctl start watcher-api.service \
-     watcher-decision-engine.service \
-     watcher-applier.service
+   # service watcher-api restart
+   # service watcher-decision-engine restart
+   # service watcher-applier restart

doc/source/man/index.rst

@@ -1,7 +1,3 @@
-====================
-Watcher Manual Pages
-====================
-
 .. toctree::
    :glob:
    :maxdepth: 1
@@ -9,5 +5,4 @@ Watcher Manual Pages
    watcher-api
    watcher-applier
    watcher-db-manage
-   watcher-decision-engine
-   watcher-status
+   watcher-decision-engine

doc/source/man/watcher-status.rst

@@ -1,87 +0,0 @@
-==============
-watcher-status
-==============
-
------------------------------------------
-CLI interface for Watcher status commands
------------------------------------------
-
-Synopsis
-========
-
-::
-
-  watcher-status <category> <command> [<args>]
-
-Description
-===========
-
-:program:`watcher-status` is a tool that provides routines for checking the
-status of a Watcher deployment.
-
-Options
-=======
-
-The standard pattern for executing a :program:`watcher-status` command is::
-
-    watcher-status <category> <command> [<args>]
-
-Run without arguments to see a list of available command categories::
-
-    watcher-status
-
-Categories are:
-
-* ``upgrade``
-
-Detailed descriptions are below:
-
-You can also run with a category argument such as ``upgrade`` to see a list of
-all commands in that category::
-
-    watcher-status upgrade
-
-These sections describe the available categories and arguments for
-:program:`Watcher-status`.
-
-Upgrade
-~~~~~~~
-
-.. _watcher-status-checks:
-
-``watcher-status upgrade check``
-  Performs a release-specific readiness check before restarting services with
-  new code. For example, missing or changed configuration options,
-  incompatible object states, or other conditions that could lead to
-  failures while upgrading.
-
-  **Return Codes**
-
-  .. list-table::
-     :widths: 20 80
-     :header-rows: 1
-
-     * - Return code
-       - Description
-     * - 0
-       - All upgrade readiness checks passed successfully and there is nothing
-         to do.
-     * - 1
-       - At least one check encountered an issue and requires further
-         investigation. This is considered a warning but the upgrade may be OK.
-     * - 2
-       - There was an upgrade status check failure that needs to be
-         investigated. This should be considered something that stops an
-         upgrade.
-     * - 255
-       - An unexpected error occurred.
-
-  **History of Checks**
-
-  **2.0.0 (Stein)**
-
-  * Sample check to be filled in with checks as they are added in Stein.
-
-  **3.0.0 (Train)**
-
-  * A check was added to enforce the minimum required version of nova API used.

doc/source/strategies/basic-server-consolidation.rst

@@ -19,16 +19,12 @@ Metrics
 
 The *basic* strategy requires the following metrics:
 
-============================ ============ ======= ===========================
+============================ ============ ======= =======
 metric                       service name plugins comment
-============================ ============ ======= ===========================
-``compute.node.cpu.percent`` ceilometer_  none    need to set the
-                                                  ``compute_monitors`` option
-                                                  to ``cpu.virt_driver`` in
-                                                  the nova.conf.
-``cpu_util``                 ceilometer_  none    cpu_util has been removed
-                                                  since Stein.
-============================ ============ ======= ===========================
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 

doc/source/strategies/node_resource_consolidation.rst

@@ -1,95 +0,0 @@
-====================================
-Node Resource Consolidation Strategy
-====================================
-
-Synopsis
---------
-
-**display name**: ``Node Resource Consolidation Strategy``
-
-**goal**: ``Server Consolidation``
-
-    .. watcher-term:: watcher.decision_engine.strategy.strategies.node_resource_consolidation.NodeResourceConsolidation
-
-Requirements
-------------
-
-None.
-
-Metrics
-*******
-
-None
-
-Cluster data model
-******************
-
-Default Watcher's Compute cluster data model:
-
-    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
-
-Actions
-*******
-
-Default Watcher's actions:
-
-    .. list-table::
-       :widths: 30 30
-       :header-rows: 1
-
-       * - action
-         - description
-       * - ``migration``
-         - .. watcher-term:: watcher.applier.actions.migration.Migrate
-       * - ``change_nova_service_state``
-         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
-
-Planner
-*******
-
-Default Watcher's planner:
-
-    .. watcher-term:: watcher.decision_engine.planner.weight.WeightPlanner
-
-Configuration
--------------
-
-Strategy parameters are:
-
-==================== ====== =======================================
-parameter            type   default Value description
-==================== ====== =======================================
-``host_choice``      String The way to select the server migration
-                            destination node, The value auto means
-                            that Nova schedular selects the
-                            destination node, and specify means
-                            the strategy specifies the destination.
-==================== ====== =======================================
-
-Efficacy Indicator
-------------------
-
-None
-
-Algorithm
----------
-
-For more information on the Node Resource Consolidation Strategy please refer
-to: https://specs.openstack.org/openstack/watcher-specs/specs/train/approved/node-resource-consolidation.html
-
-How to use it ?
----------------
-
-.. code-block:: shell
-
-    $ openstack optimize audittemplate create \
-      at1 server_consolidation \
-      --strategy node_resource_consolidation
-
-    $ openstack optimize audit create \
-      -a at1 -p host_choice=auto
-
-External Links
---------------
-
-None.

doc/source/strategies/vm_workload_consolidation.rst

@@ -19,15 +19,22 @@ Metrics
 
 The *vm_workload_consolidation* strategy requires the following metrics:
 
-============================ ============ ======= =========================
+============================ ============ ======= =======
 metric                       service name plugins comment
-============================ ============ ======= =========================
-``cpu_util``                 ceilometer_  none    cpu_util has been removed
-                                                  since Stein.
-``memory.resident``          ceilometer_  none
+============================ ============ ======= =======
 ``memory``                   ceilometer_  none
 ``disk.root.size``           ceilometer_  none
-============================ ============ ======= =========================
+============================ ============ ======= =======
+
+The following metrics are not required but increase the accuracy of
+the strategy if available:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory.resident``          ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 

doc/source/strategies/workload-stabilization.rst

@@ -19,21 +19,17 @@ Metrics
 
 The *workload_stabilization* strategy requires the following metrics:
 
-============================ ============ ======= =============================
+============================ ============ ======= =======
 metric                       service name plugins comment
-============================ ============ ======= =============================
-``compute.node.cpu.percent`` ceilometer_  none    need to set the
-                                                  ``compute_monitors`` option
-                                                  to ``cpu.virt_driver`` in the
-                                                  nova.conf.
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
 ``hardware.memory.used``     ceilometer_  SNMP_
-``cpu_util``                 ceilometer_  none    cpu_util has been removed
-                                                  since Stein.
+``cpu_util``                 ceilometer_  none
 ``memory.resident``          ceilometer_  none
-============================ ============ ======= =============================
+============================ ============ ======= =======
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
-.. _SNMP: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#snmp-based-meters
+.. _SNMP: http://docs.openstack.org/admin-guide/telemetry-measurements.html
 
 Cluster data model
 ******************

doc/source/strategies/workload_balance.rst

@@ -21,13 +21,12 @@ Metrics
 
 The *workload_balance* strategy requires the following metrics:
 
-======================= ============ ======= =========================
+======================= ============ ======= =======
 metric                  service name plugins comment
-======================= ============ ======= =========================
-``cpu_util``            ceilometer_  none    cpu_util has been removed
-                                             since Stein.
+======================= ============ ======= =======
+``cpu_util``            ceilometer_  none
 ``memory.resident``     ceilometer_  none
-======================= ============ ======= =========================
+======================= ============ ======= =======
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 

doc/source/user/event_type_audit.rst

@@ -1,195 +0,0 @@
-..
-  Licensed under the Apache License, Version 2.0 (the "License"); you may
-  not use this file except in compliance with the License. You may obtain
-  a copy of the License at
-
-          http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-  License for the specific language governing permissions and limitations
-  under the License.
-
-
-======================
-Audit using Aodh alarm
-======================
-
-Audit with EVENT type can be triggered by special alarm. This guide walks
-you through the steps to build an event-driven optimization solution by
-integrating Watcher with Ceilometer/Aodh.
-
-Step 1: Create an audit with EVENT type
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The first step is to create an audit with EVENT type,
-you can create an audit template firstly:
-
-.. code-block:: bash
-
-  $ openstack optimize audittemplate create your_template_name <your_goal> \
-    --strategy <your_strategy>
-
-or create an audit directly with special goal and strategy:
-
-.. code-block:: bash
-
-  $ openstack optimize audit create --goal <your_goal> \
-    --strategy <your_strategy> --audit_type EVENT
-
-This is an example for creating an audit with dummy strategy:
-
-.. code-block:: bash
-
-  $ openstack optimize audit create --goal dummy \
-    --strategy dummy --audit_type EVENT
-  +---------------+--------------------------------------+
-  | Field         | Value                                |
-  +---------------+--------------------------------------+
-  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
-  | Name          | dummy-2020-01-14T03:21:19.168467     |
-  | Created At    | 2020-01-14T03:21:19.200279+00:00     |
-  | Updated At    | None                                 |
-  | Deleted At    | None                                 |
-  | State         | PENDING                              |
-  | Audit Type    | EVENT                                |
-  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
-  | Interval      | None                                 |
-  | Goal          | dummy                                |
-  | Strategy      | dummy                                |
-  | Audit Scope   | []                                   |
-  | Auto Trigger  | False                                |
-  | Next Run Time | None                                 |
-  | Hostname      | None                                 |
-  | Start Time    | None                                 |
-  | End Time      | None                                 |
-  | Force         | False                                |
-  +---------------+--------------------------------------+
-
-We need to build Aodh action url using Watcher webhook API.
-For convenience we export the url into an environment variable:
-
-.. code-block:: bash
-
-  $ export AUDIT_UUID=a3326a6a-c18e-4e8e-adba-d0c61ad404c5
-  $ export ALARM_URL="trust+http://localhost/infra-optim/v1/webhooks/$AUDIT_UUID"
-
-Step 2: Create Aodh Alarm
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Once we have the audit created, we can continue to create Aodh alarm and
-set the alarm action to Watcher webhook API. The alarm type can be event(
-i.e. ``compute.instance.create.end``) or gnocchi_resources_threshold(i.e.
-``cpu_util``), more info refer to alarm-creation_
-
-For example:
-
-.. code-block:: bash
-
-  $ openstack alarm create \
-    --type event --name instance_create \
-    --event-type "compute.instance.create.end" \
-    --enable True --repeat-actions False \
-    --alarm-action $ALARM_URL
-  +---------------------------+------------------------------------------------------------------------------------------+
-  | Field                     | Value                                                                                    |
-  +---------------------------+------------------------------------------------------------------------------------------+
-  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5'] |
-  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                     |
-  | description               | Alarm when compute.instance.create.end event occurred.                                   |
-  | enabled                   | True                                                                                     |
-  | event_type                | compute.instance.create.end                                                              |
-  | insufficient_data_actions | []                                                                                       |
-  | name                      | instance_create                                                                          |
-  | ok_actions                | []                                                                                       |
-  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                         |
-  | query                     |                                                                                          |
-  | repeat_actions            | False                                                                                    |
-  | severity                  | low                                                                                      |
-  | state                     | insufficient data                                                                        |
-  | state_reason              | Not evaluated yet                                                                        |
-  | state_timestamp           | 2020-01-14T03:56:26.894416                                                               |
-  | time_constraints          | []                                                                                       |
-  | timestamp                 | 2020-01-14T03:56:26.894416                                                               |
-  | type                      | event                                                                                    |
-  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                         |
-  +---------------------------+------------------------------------------------------------------------------------------+
-
-.. _alarm-creation: https://docs.openstack.org/aodh/latest/admin/telemetry-alarms.html#alarm-creation
-
-Step 3: Trigger the alarm
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-In this example, you can create a new instance to trigger the alarm.
-The alarm state will translate from ``insufficient data`` to ``alarm``.
-
-.. code-block:: bash
-
-  $ openstack alarm show b9e381fc-8e3e-4943-82ee-647e7a2ef644
-  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
-  | Field                     | Value                                                                                                             |
-  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
-  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5']                          |
-  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                                              |
-  | description               | Alarm when compute.instance.create.end event occurred.                                                            |
-  | enabled                   | True                                                                                                              |
-  | event_type                | compute.instance.create.end                                                                                       |
-  | insufficient_data_actions | []                                                                                                                |
-  | name                      | instance_create                                                                                                   |
-  | ok_actions                | []                                                                                                                |
-  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                                                  |
-  | query                     |                                                                                                                   |
-  | repeat_actions            | False                                                                                                             |
-  | severity                  | low                                                                                                               |
-  | state                     | alarm                                                                                                             |
-  | state_reason              | Event <id=67dd0afa-2082-45a4-8825-9573b2cc60e5,event_type=compute.instance.create.end> hits the query <query=[]>. |
-  | state_timestamp           | 2020-01-14T03:56:26.894416                                                                                        |
-  | time_constraints          | []                                                                                                                |
-  | timestamp                 | 2020-01-14T06:17:40.350649                                                                                        |
-  | type                      | event                                                                                                             |
-  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                                                  |
-  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
-
-Step 4: Verify the audit
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-This can be verified to check if the audit state was ``SUCCEEDED``:
-
-.. code-block:: bash
-
-  $ openstack optimize audit show a3326a6a-c18e-4e8e-adba-d0c61ad404c5
-  +---------------+--------------------------------------+
-  | Field         | Value                                |
-  +---------------+--------------------------------------+
-  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
-  | Name          | dummy-2020-01-14T03:21:19.168467     |
-  | Created At    | 2020-01-14T03:21:19+00:00            |
-  | Updated At    | 2020-01-14T06:26:40+00:00            |
-  | Deleted At    | None                                 |
-  | State         | SUCCEEDED                            |
-  | Audit Type    | EVENT                                |
-  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
-  | Interval      | None                                 |
-  | Goal          | dummy                                |
-  | Strategy      | dummy                                |
-  | Audit Scope   | []                                   |
-  | Auto Trigger  | False                                |
-  | Next Run Time | None                                 |
-  | Hostname      | ubuntudbs                            |
-  | Start Time    | None                                 |
-  | End Time      | None                                 |
-  | Force         | False                                |
-  +---------------+--------------------------------------+
-
-and you can use the following command to check if the action plan
-was created:
-
-.. code-block:: bash
-
-  $ openstack optimize actionplan list --audit a3326a6a-c18e-4e8e-adba-d0c61ad404c5
-  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
-  | UUID                                 | Audit                                | State       | Updated At | Global efficacy |
-  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
-  | 673b3fcb-8c16-4a41-9ee3-2956d9f6ca9e | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 | RECOMMENDED | None       |                 |
-  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+

doc/source/user/index.rst

@@ -1,10 +1,4 @@
-==========
-User Guide
-==========
-
 .. toctree::
    :maxdepth: 2
 
-   ways-to-install
    user-guide
-   event_type_audit

doc/source/user/user-guide.rst

@@ -4,6 +4,8 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
+.. _user-guide:
+
 ==================
 Watcher User Guide
 ==================
@@ -53,13 +55,13 @@ plugin installation guide`_.
 
      $ watcher ...
 
-.. _`installation guide`: https://docs.openstack.org/watcher/latest/install/
+.. _`installation guide`: https://docs.openstack.org/python-watcherclient/latest
 .. _`Watcher Horizon plugin installation guide`: https://docs.openstack.org/watcher-dashboard/latest/install/installation.html
 .. _`OpenStack CLI`: https://docs.openstack.org/python-openstackclient/latest/cli/man/openstack.html
 .. _`Watcher CLI`: https://docs.openstack.org/python-watcherclient/latest/cli/index.html
 
-Watcher CLI Command
--------------------
+Seeing what the Watcher CLI can do ?
+------------------------------------
 We can see all of the commands available with Watcher CLI by running the
 watcher binary without options.
 
@@ -67,8 +69,8 @@ watcher binary without options.
 
   $ openstack help optimize
 
-Running an audit of the cluster
--------------------------------
+How do I run an audit of my cluster ?
+-------------------------------------
 
 First, you need to find the :ref:`goal <goal_definition>` you want to achieve:
 
@@ -180,7 +182,7 @@ periodically calling:
 
 .. code:: bash
 
-  $ openstack optimize action list --action-plan <the_action_plan_uuid>
+  $ openstack optimize action list
 
 You can also obtain more detailed information about a specific action:
 

etc/apache2/watcher

@@ -16,7 +16,7 @@ Listen 9322
 
 <VirtualHost *:9322>
     WSGIDaemonProcess watcher-api user=stack group=stack processes=2 threads=2 display-name=%{GROUP}
-    WSGIScriptAlias / /usr/local/bin/watcher-api-wsgi
+    WSGIScriptAlias / /opt/stack/watcher/watcher/api/app.wsgi
     WSGIProcessGroup watcher-api
 
     ErrorLog /var/log/httpd/watcher_error.log

lower-constraints.txt

@@ -5,6 +5,8 @@ appdirs==1.4.3
 APScheduler==3.5.1
 asn1crypto==0.24.0
 automaton==1.14.0
+Babel==2.5.3
+bandit==1.4.0
 beautifulsoup4==4.6.0
 cachetools==2.0.1
 certifi==2018.1.18
@@ -29,12 +31,15 @@ eventlet==0.20.0
 extras==1.0.0
 fasteners==0.14.1
 fixtures==3.0.0
+flake8==2.5.5
 freezegun==0.3.10
-futurist==1.8.0
+future==0.16.0
+futurist==1.6.0
 gitdb2==2.0.3
 GitPython==2.1.8
 gnocchiclient==7.0.1
 greenlet==0.4.13
+hacking==0.12.0
 idna==2.6
 imagesize==1.0.0
 iso8601==0.1.12
@@ -42,7 +47,7 @@ Jinja2==2.10
 jmespath==0.9.3
 jsonpatch==1.21
 jsonpointer==2.0
-jsonschema==3.2.0
+jsonschema==2.6.0
 keystoneauth1==3.4.0
 keystonemiddleware==4.21.0
 kombu==4.1.0
@@ -52,34 +57,34 @@ lxml==4.1.1
 Mako==1.0.7
 MarkupSafe==1.0
 mccabe==0.2.1
-microversion_parse==0.2.1
+mock==2.0.0
 monotonic==1.4
+mox3==0.25.0
 msgpack==0.5.6
 munch==2.2.0
 netaddr==0.7.19
 netifaces==0.10.6
-networkx==2.2
+networkx==1.11
+openstackdocstheme==1.20.0
 openstacksdk==0.12.0
 os-api-ref===1.4.0
 os-client-config==1.29.0
 os-service-types==1.2.0
 os-testr==1.0.0
 osc-lib==1.10.0
-os-resource-classes==0.4.0
 oslo.cache==1.29.0
 oslo.concurrency==3.26.0
 oslo.config==5.2.0
-oslo.context==2.21.0
+oslo.context==2.20.0
 oslo.db==4.35.0
 oslo.i18n==3.20.0
 oslo.log==3.37.0
-oslo.messaging==8.1.2
+oslo.messaging==5.36.0
 oslo.middleware==3.35.0
 oslo.policy==1.34.0
 oslo.reports==1.27.0
 oslo.serialization==2.25.0
 oslo.service==1.30.0
-oslo.upgradecheck==0.1.0
 oslo.utils==3.36.0
 oslo.versionedobjects==1.32.0
 oslotest==3.3.0
@@ -87,13 +92,15 @@ packaging==17.1
 Paste==2.0.3
 PasteDeploy==1.5.2
 pbr==3.1.1
-pecan==1.3.2
+pecan==1.2.1
+pep8==1.5.7
 pika==0.10.0
 pika-pool==0.1.3
 prettytable==0.7.2
 psutil==5.4.3
 pycadf==2.7.0
 pycparser==2.18
+pyflakes==0.8.1
 Pygments==2.2.0
 pyinotify==0.9.6
 pyOpenSSL==17.5.0
@@ -104,16 +111,17 @@ python-cinderclient==3.5.0
 python-dateutil==2.7.0
 python-editor==1.0.3
 python-glanceclient==2.9.1
-python-ironicclient==2.5.0
+python-ironicclient==2.3.0
 python-keystoneclient==3.15.0
 python-mimeparse==1.6.0
 python-monascaclient==1.12.0
 python-neutronclient==6.7.0
-python-novaclient==14.1.0
+python-novaclient==10.1.0
 python-openstackclient==3.14.0
 python-subunit==1.2.0
 pytz==2018.3
 PyYAML==3.12
+reno==2.7.0
 repoze.lru==0.7
 requests==2.18.4
 requestsexceptions==1.4.0
@@ -122,15 +130,20 @@ rfc3986==1.1.0
 Routes==2.4.1
 simplegeneric==0.8.1
 simplejson==3.13.2
+six==1.11.0
 smmap2==2.0.3
 snowballstemmer==1.2.1
+Sphinx==1.6.5
+sphinxcontrib-httpdomain==1.6.1
+sphinxcontrib-pecanwsme==0.8.0
+sphinxcontrib-websupport==1.0.1
 SQLAlchemy==1.2.5
 sqlalchemy-migrate==0.11.0
 sqlparse==0.2.4
 statsd==3.2.2
 stestr==2.0.0
 stevedore==1.28.0
-taskflow==3.7.1
+taskflow==3.1.0
 Tempita==0.5.2
 tenacity==4.9.0
 testresources==2.0.1
@@ -144,7 +157,7 @@ urllib3==1.22
 vine==1.1.4
 waitress==1.1.0
 warlock==1.3.0
-WebOb==1.8.5
+WebOb==1.7.4
 WebTest==2.0.29
 wrapt==1.10.11
 WSME==0.9.2

playbooks/orchestrate-tempest.yaml

@@ -0,0 +1,14 @@
+- hosts: all
+  # This is the default strategy, however since orchestrate-devstack requires
+  # "linear", it is safer to enforce it in case this is running in an
+  # environment configured with a different default strategy.
+  strategy: linear
+  roles:
+    - orchestrate-devstack
+
+- hosts: tempest
+  roles:
+    - setup-tempest-run-dir
+    - setup-tempest-data-dir
+    - acl-devstack-files
+    - run-tempest

playbooks/pre.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - add-hostnames-to-hosts

releasenotes/notes/add-baremetal-scoper-9ef23f5fb8f0be6a.yaml

@@ -1,3 +0,0 @@
----
-features:
-  - Baremetal Model gets Audit scoper with an ability to exclude Ironic nodes.

releasenotes/notes/add-force-field-to-audit-4bcaeedfe27233ad.yaml

@@ -1,7 +0,0 @@
----
-features:
-  - |
-    Add force field to Audit. User can set --force to enable the new option when
-    launching audit. If force is True, audit will be executed despite of ongoing
-    actionplan. The new audit may create a wrong actionplan if they use the same
-    data model.

releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml

@@ -1,5 +0,0 @@
----
-features:
-  - |
-    Add start_time and end_time fields in audits table. User can set the start
-    time and/or end time when creating CONTINUOUS audit.

releasenotes/notes/add-upgrade-check-framework-5bb9693c8a78931c.yaml

@@ -1,13 +0,0 @@
----
-prelude: >
-    Added new tool ``watcher-status upgrade check``.
-features:
-  - |
-    New framework for ``watcher-status upgrade check`` command is added.
-    This framework allows adding various checks which can be run before a
-    Watcher upgrade to ensure if the upgrade can be performed safely.
-upgrade:
-  - |
-    Operator can now use new CLI tool ``watcher-status upgrade check``
-    to check if Watcher deployment can be safely upgraded from
-    N-1 to N release.

releasenotes/notes/api-call-retry-fef741ac684c58dd.yaml

@@ -1,9 +0,0 @@
----
-features:
-  - |
-    API calls while building the Compute data model will be retried upon
-    failure. The amount of failures allowed before giving up and the time before
-    reattempting are configurable. The `api_call_retries` and
-    `api_query_timeout` parameters in the `[collector]` group can be used to
-    adjust these paremeters. 10 retries with a 1 second time in between
-    reattempts is the default.

releasenotes/notes/api-microversioning-7999a3ee8073bf32.yaml

@@ -1,8 +0,0 @@
----
-features:
-    Watcher starts to support API microversions since Stein cycle. From now
-    onwards all API changes should be made with saving backward compatibility.
-    To specify API version operator should use OpenStack-API-Version
-    HTTP header. If operator wants to know the mininum and maximum supported
-    versions by API, he/she can access /v1 resource and Watcher API will
-    return appropriate headers in response.

releasenotes/notes/consume-nova-versioned-notifications-f98361b37e546b4d.yaml

@@ -1,60 +0,0 @@
----
-features:
-  - |
-    Watcher consumes Nova notifications to update its internal
-    Compute CDM(Cluster Data Model).
-    All the notifications as below
-
-    pre-existing:
-
-    * service.update
-
-    * instance.update
-
-    * instance.delete.end
-
-    new:
-
-    * instance.lock
-
-    * instance.unlock
-
-    * instance.pause.end
-
-    * instance.power_off.end
-
-    * instance.power_on.end
-
-    * instance.resize_confirm.end
-
-    * instance.restore.end
-
-    * instance.resume.end
-
-    * instance.shelve.end
-
-    * instance.shutdown.end
-
-    * instance.suspend.end
-
-    * instance.unpause.end
-
-    * instance.unrescue.end
-
-    * instance.unshelve.end
-
-    * instance.rebuild.end
-
-    * instance.rescue.end
-
-    * instance.create.end
-
-    * instance.live_migration_force_complete.end
-
-    * instance.live_migration_post_dest.end
-
-    * instance.soft_delete.end
-
-    * service.create
-
-    * service.delete

releasenotes/notes/datasource-query-retry-00cba5f7e68aec39.yaml

@@ -1,17 +0,0 @@
----
-features:
-  - |
-    All datasources can now be configured to retry retrieving a metric upon
-    encountering an error. Between each attempt will be a set amount of time
-    which can be adjusted from the configuration. These configuration
-    options can be found in the `[watcher_datasources]` group and are named
-    `query_max_retries` and `query_timeout`.
-upgrade:
-  - |
-    If Gnocchi was configured to have a custom amount of retries and or a
-    custom timeout then the configuration needs to moved into the
-    `[watcher_datasources]` group instead of the `[gnocchi_client]` group.
-deprecations:
-  - |
-    The configuration options for query retries in `[gnocchi_client]` are
-    deprecated and the option in `[watcher_datasources]` should now be used.

releasenotes/notes/deprecate-ceilometer-datasource-446b0be70fbce28b.yaml

@@ -1,7 +0,0 @@
----
-deprecations:
-  - |
-    Ceilometer Datasource has been deprecated since its API has been
-    deprecated in Ocata cycle. Watcher has supported Ceilometer for some
-    releases after Ocata to let users migrate to Gnocchi/Monasca datasources.
-    Since Train release, Ceilometer support will be removed.

releasenotes/notes/drop-py-2-7-54f8e806d71f19a7.yaml

@@ -1,6 +0,0 @@
----
-upgrade:
-  - |
-    Python 2.7 support has been dropped. Last release of Watcher
-    to support py2.7 is OpenStack Train. The minimum version of Python now
-    supported by Watcher is Python 3.6.

releasenotes/notes/enhance-watcher-applier-engine-86c676ce8f179e68.yaml

@@ -1,16 +0,0 @@
----
-features:
-  - |
-    Added a new config option 'action_execution_rule' which is a dict type.
-    Its key field is strategy name and the value is 'ALWAYS' or 'ANY'.
-    'ALWAYS' means the callback function returns True as usual.
-    'ANY' means the return depends on the result of previous action execution.
-    The callback returns True if previous action gets failed, and the engine
-    continues to run the next action. If previous action executes success,
-    the callback returns False then the next action will be ignored.
-    For strategies that aren't in 'action_execution_rule', the callback always
-    returns True.
-    Please add the next section in the watcher.conf file
-    if your strategy needs this feature.
-    [watcher_workflow_engines.taskflow]
-    action_execution_rule = {'your strategy name': 'ANY'}

releasenotes/notes/event-driven-optimization-based-4870f112bef8a560.yaml

@@ -1,8 +0,0 @@
----
-features:
-  - |
-    Add a new webhook API and a new audit type EVENT, the microversion is 1.4.
-    Now Watcher user can create audit with EVENT type and the audit will be
-    triggered by webhook API.
-    The user guide is available online:
-    https://docs.openstack.org/watcher/latest/user/event_type_audit.html

releasenotes/notes/file-based-metric-map-c2af62b5067895df.yaml

@@ -1,11 +0,0 @@
----
-features:
-  - |
-    Allow using file to override metric map. Override the metric map of
-    each datasource as soon as it is created by the manager. This override
-    comes from a file whose path is provided by a setting in config file.
-    The setting is `watcher_decision_engine/metric_map_path`.  The file
-    contains a map per datasource whose keys are the metric names as
-    recognized by watcher and the value is the real name of the metric
-    in the datasource. This setting defaults to `/etc/watcher/metric_map.yaml`,
-    and presence of this file is optional.

releasenotes/notes/formal-datasource-interface-implementation-222769d55a127d33.yaml

@@ -1,17 +0,0 @@
----
-features:
-  - |
-    Improved interface for datasource baseclass that better defines expected
-    values and types for parameters and return types of all abstract methods.
-    This allows all strategies to work with every datasource provided the
-    metrics are configured for that given datasource.
-deprecations:
-  - |
-    The new strategy baseclass has significant changes in method parameters
-    and any out-of-tree strategies will have to be adopted.
-  - |
-    Several strategies have changed the `node` parameter to `compute_node` to
-    be better aligned with terminology. These strategies include
-    `basic_consolidation` and `workload_stabilzation`. The `node` parameter
-    will remain supported during Train release and will be removed in the
-    subsequent release.

releasenotes/notes/general-purpose-decision-engine-threadpool-0711b23abfc9d409.yaml

@@ -1,20 +0,0 @@
----
-prelude: >
-    Many operations in the decision engine will block on I/O. Such I/O
-    operations can stall the execution of a sequential application
-    significantly. To reduce the potential bottleneck of many operations
-    the general purpose decision engine threadpool is introduced.
-features:
-  - |
-    A new threadpool for the decision engine that contributors can use to
-    improve the performance of many operations, primarily I/O bound onces.
-    The amount of workers used by the decision engine threadpool can be
-    configured to scale according to the available infrastructure using
-    the `watcher_decision_engine.max_general_workers` config option.
-    Documentation for contributors to effectively use this threadpool is
-    available online:
-    https://docs.openstack.org/watcher/latest/contributor/concurrency.html
-  - |
-    The building of the compute (Nova) data model will be done using the
-    decision engine threadpool, thereby, significantly reducing the total
-    time required to build it.

releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml

@@ -1,11 +0,0 @@
----
-features:
-  - |
-    Watcher now supports configuring which datasource to use and in which
-    order. This configuration is done by specifying datasources in the
-    watcher_datasources section:
-
-    - ``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``
-
-    Specific strategies can override this order and use datasources which
-    are not listed in the global preference.

releasenotes/notes/grafana-datasource-b672367c23ffa0c6.yaml

@@ -1,10 +0,0 @@
----
-features:
-  - |
-    Grafana has been added as datasource that can be used for collecting
-    metrics. The configuration options allow to specify what metrics and how
-    they are stored in grafana so that no matter how Grafana is configured it
-    can still be used. The configuration can be done via the typical
-    configuration file but it is recommended to configure most options in the
-    yaml file for metrics. For a complete walkthrough on configuring Grafana
-    see: https://docs.openstack.org/watcher/latest/datasources/grafana.html

releasenotes/notes/improve-compute-data-model-b427c85e4ed2b6fb.yaml

@@ -1,23 +0,0 @@
----
-features:
-  - |
-    Watcher can get resource information such as total, allocation ratio and
-    reserved information from Placement API.
-    Now we add some new fields to the Watcher Data Model:
-
-    * vcpu_reserved: The amount of cpu a node has reserved for its own use.
-    * vcpu_ratio: CPU allocation ratio.
-    * memory_mb_reserved: The amount of memory a node has reserved for
-      its own use.
-    * memory_ratio: Memory allocation ratio.
-    * disk_gb_reserved: The amount of disk a node has reserved for its own use.
-    * disk_ratio: Disk allocation ratio.
-
-    We also add some new propeties:
-
-    * vcpu_capacity: The amount of vcpu, take allocation ratio into account,
-      but do not include reserved.
-    * memory_mb_capacity: The amount of memory, take allocation ratio into
-      account, but do not include reserved.
-    * disk_gb_capacity: The amount of disk, take allocation ratio into
-      account, but do not include reserved.

releasenotes/notes/min-required-nova-train-71f124192d88ae52.yaml

@@ -1,8 +0,0 @@
----
-upgrade:
-  - |
-    The minimum required version of the ``[nova_client]/api_version`` value
-    is now enforced to be ``2.56`` which is available since the Queens version
-    of the nova compute service.
-
-    A ``watcher-status upgrade check`` has been added for this.

releasenotes/notes/node-resource-consolidation-73bc0c0abfeb0b03.yaml

@@ -1,7 +0,0 @@
----
-features:
-  - |
-    Added strategy "node resource consolidation". This
-    strategy is used to centralize VMs to as few nodes
-    as possible by VM migration. User can set an input
-    parameter to decide how to select the destination node.

releasenotes/notes/remove-nova-legacy-notifications-e1b6d10eff58f30a.yaml

@@ -1,5 +0,0 @@
----
-deprecations:
-  - |
-    Watcher removes the support to Nova legacy notifications because of Nova
-    will deprecate them.

releasenotes/notes/scope-for-data-model-ea9792f90db14343.yaml

@@ -1,6 +0,0 @@
----
-features:
-  - |
-    For a large cloud infrastructure, retrieving data from Nova may take
-    a long time. To avoid getting too much data from Nova, building the
-    compute data model according to the scope of audit.

releasenotes/notes/show-datamodel-api-6945b744fd5d25d5.yaml

@@ -1,12 +0,0 @@
----
-features:
-  - |
-    Add show data model api for Watcher. New in version 1.3.
-    User can use 'openstack optimize datamodel list'
-    command to view the current data model information in memory.
-    User can also add '--audit <Audit_UUID>' to view specific data model
-    in memory filted by the scope in audit.
-    User can also add '--detail' to view detailed information about current data model.
-    User can also add '--type <type>' to specify the type of data model.
-    Default type is 'compute'. In the future, type 'storage' and 'baremetal'
-    will be supported.