Move queue declaration to project level

This moves the watcher queue declaration from the pipeline level
(where it is no longer valid) to the project level.

https: //lists.openstack.org/pipermail/openstack-discuss/2022-May/028603.html
Change-Id: I06923abb00f7eecd59587f44cd1f6a069e88a9fc
(cherry picked from commit 6003322711)

.gitignore

@@ -26,6 +26,7 @@ pip-log.txt
 .stestr/
 .venv
 .idea
+.testrepository/
 
 # Translations
 *.mo
@@ -74,3 +75,6 @@ releasenotes/build
 
 # Autogenerated sample config file
 etc/watcher/watcher.conf.sample
+
+# Atom
+.remote-sync.json

.gitreview

@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
-defaultbranch=stable/rocky
+defaultbranch=stable/stein

.zuul.yaml

@@ -1,25 +1,31 @@
 - project:
+    queue: watcher
     templates:
+      - check-requirements
+      - openstack-cover-jobs
+      - openstack-lower-constraints-jobs
       - openstack-python-jobs
       - openstack-python35-jobs
-      - publish-openstack-sphinx-docs
-      - check-requirements
-      - release-notes-jobs
+      - openstack-python36-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3
     check:
       jobs:
         - watcher-tempest-functional
+        - watcher-grenade:
+            voting: false
         - watcher-tempest-dummy_optim
         - watcher-tempest-actuator
         - watcher-tempest-basic_optim
+        - watcher-tempest-vm_workload_consolidation
         - watcher-tempest-workload_balancing
-        - watcherclient-tempest-functional
         - watcher-tempest-zone_migration
-        - openstack-tox-lower-constraints
+        - watcher-tempest-host_maintenance
+        - watcher-tempest-storage_balance
+        - watcher-tls-test
     gate:
-      queue: watcher
       jobs:
         - watcher-tempest-functional
-        - openstack-tox-lower-constraints
 
 - job:
     name: watcher-tempest-dummy_optim
@@ -42,6 +48,18 @@
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_basic_optim
 
+- job:
+    name: watcher-tempest-vm_workload_consolidation
+    parent: watcher-tempest-multinode
+    voting: false
+    vars:
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation
+      devstack_local_conf:
+        test-config:
+          $WATCHER_CONFIG:
+            watcher_strategies.vm_workload_consolidation:
+              datasource: ceilometer
+
 - job:
     name: watcher-tempest-workload_balancing
     parent: watcher-tempest-multinode
@@ -56,21 +74,48 @@
     vars:
       tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_zone_migration
 
+- job:
+    name: watcher-tempest-host_maintenance
+    parent: watcher-tempest-multinode
+    voting: false
+    vars:
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance
+
+- job:
+    name: watcher-tempest-storage_balance
+    parent: watcher-tempest-multinode
+    voting: false
+    vars:
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_storage_balance
+      devstack_local_conf:
+        test-config:
+          $TEMPEST_CONFIG:
+            volume:
+              backend_names: ['BACKEND_1', 'BACKEND_2']
+            volume-feature-enabled:
+              multi_backend: true
+
+- job:
+    name: watcher-tls-test
+    parent: watcher-tempest-multinode
+    group-vars:
+      subnode:
+          devstack_services:
+            tls-proxy: true
+    vars:
+      devstack_services:
+        tls-proxy: true
+
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
-    nodeset: openstack-two-node
-    pre-run: playbooks/pre.yaml
-    run: playbooks/orchestrate-tempest.yaml
+    nodeset: openstack-two-node-bionic
     roles:
       - zuul: openstack/tempest
     group-vars:
       subnode:
         devstack_local_conf:
           post-config:
-            $NOVA_CONF:
-              libvirt:
-                live_migration_uri: qemu+ssh://root@%s/system
             $WATCHER_CONF:
               watcher_cluster_data_model_collectors.compute:
                 period: 120
@@ -96,9 +141,6 @@
     vars:
       devstack_local_conf:
         post-config:
-          $NOVA_CONF:
-            libvirt:
-              live_migration_uri: qemu+ssh://root@%s/system
           $WATCHER_CONF:
             watcher_cluster_data_model_collectors.compute:
               period: 120
@@ -151,6 +193,30 @@
       zuul_copy_output:
         /etc/hosts: logs
 
+- job:
+    name: watcher-grenade
+    parent: legacy-dsvm-base
+    timeout: 10800
+    run: playbooks/legacy/grenade-devstack-watcher/run.yaml
+    post-run: playbooks/legacy/grenade-devstack-watcher/post.yaml
+    irrelevant-files:
+      - ^(test-|)requirements.txt$
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^watcher/hacking/.*$
+      - ^watcher/tests/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^tox.ini$
+    required-projects:
+      - openstack/grenade
+      - openstack/devstack-gate
+      - openstack/watcher
+      - openstack/python-watcherclient
+      - openstack/watcher-tempest-plugin
+
 - job:
     # This job is used in python-watcherclient repo
     name: watcherclient-tempest-functional
@@ -159,6 +225,4 @@
     timeout: 4200
     vars:
       tempest_concurrency: 1
-      devstack_localrc:
-        TEMPEST_PLUGINS: /opt/stack/python-watcherclient
-      tempest_test_regex: watcherclient.tests.functional
+      tempest_test_regex: watcher_tempest_plugin.tests.client_functional

api-ref/source/index.rst

@@ -6,6 +6,7 @@ Watcher API
 
 .. rest_expand_all::
 
+.. include:: watcher-api-versions.inc
 .. include:: watcher-api-v1-audittemplates.inc
 .. include:: watcher-api-v1-audits.inc
 .. include:: watcher-api-v1-actionplans.inc
@@ -13,4 +14,4 @@ Watcher API
 .. include:: watcher-api-v1-goals.inc
 .. include:: watcher-api-v1-strategies.inc
 .. include:: watcher-api-v1-services.inc
-.. include:: watcher-api-v1-scoring_engines.inc
+.. include:: watcher-api-v1-scoring_engines.inc

api-ref/source/parameters.yaml

@@ -1,3 +1,42 @@
+# variables in header
+header_version:
+  description: |
+    Specific API microversion used to generate this response.
+  in: header
+  required: true
+  type: string
+openstack-api-max-version:
+  description: |
+    Maximum API microversion supported by this endpoint, eg. "1.1"
+  in: header
+  required: true
+  type: string
+openstack-api-min-version:
+  description: |
+    Minimum API microversion supported by this endpoint, eg. "1.0"
+  in: header
+  required: true
+  type: string
+openstack-api-version:
+  description: >
+    A request SHOULD include this header to indicate to the Watcher API service what
+    version the client supports. The server will transform the response object into
+    compliance with the requested version, if it is supported, or return a
+    406 Not Acceptable error.
+    If this header is not supplied, the server will response with server minimum
+    supported version.
+  in: header
+  required: true
+  type: string
+openstack-request-id:
+  description: >
+    An unique ID for tracking the request. The request ID associated with the request
+    appears in the log lines for that request. By default, the middleware configuration
+    ensures that the request ID appears in the log files.
+  in: header
+  required: false
+  type: string
+
 # Path
 action_ident:
   description: |
@@ -170,6 +209,12 @@ actionplan_global_efficacy:
   in: body
   required: false
   type: array
+actionplan_hostname:
+  description: |
+    Hostname the actionplan is running on
+  in: body
+  required: false
+  type: string
 actionplan_state:
   description: |
     State of this action plan. To get more information about states and
@@ -181,16 +226,37 @@ actionplan_state:
 # Audit
 audit_autotrigger:
   description: |
-    Autoexecute action plan once audit is succeeded.
+    Auto execute action plan once audit is succeeded.
   in: body
   required: false
   type: boolean
+audit_endtime_req:
+  description: |
+    The local time after which audit can't be executed.
+    It will be converted to UTC time by Watcher.
+  in: body
+  required: false
+  type: string
+  min_version: 1.1
+audit_endtime_resp:
+  description: |
+    The UTC time after which audit can't be executed.
+  in: body
+  required: false
+  type: string
+  min_version: 1.1
 audit_goal:
   description: |
     The UUID or name of the Goal.
   in: body
   required: false
   type: string
+audit_hostname:
+  description: |
+    Hostname the audit is running on
+  in: body
+  required: false
+  type: string
 audit_interval:
   description: |
     Time interval between audit's execution.
@@ -217,6 +283,21 @@ audit_parameters:
   in: body
   required: false
   type: JSON
+audit_starttime_req:
+  description: |
+    The local time after which audit can be executed in accordance
+    with interval. It will be converted to UTC time by Watcher.
+  in: body
+  required: false
+  type: string
+  min_version: 1.1
+audit_starttime_resp:
+  description: |
+    The UTC time after which audit can be executed in accordance with interval.
+  in: body
+  required: false
+  type: string
+  min_version: 1.1
 audit_state:
   description: |
     State of this audit. To get more information about states and
@@ -232,7 +313,7 @@ audit_strategy:
   type: string
 audit_type:
   description: |
-    Type of this audit. Can be either ONESHOT or CONTINUOUS.
+    Type of this audit. Can only be either ONESHOT or CONTINUOUS.
   in: body
   required: true
   type: string
@@ -268,6 +349,12 @@ audittemplate_strategy:
   in: body
   required: false
   type: string
+audittemplate_uuid:
+  description: |
+    The UUID of the Audit template.
+  in: body
+  required: true
+  type: string
 
 created_at:
   description: |
@@ -431,3 +518,29 @@ uuid:
   in: body
   required: true
   type: string
+
+# Version
+version:
+  description: |
+    Versioning of this API response, eg. "1.1".
+  in: body
+  required: true
+  type: string
+version_description:
+  description: |
+    Descriptive text about the Watcher service.
+  in: body
+  required: true
+  type: string
+version_id:
+  description: |
+    Major API version, eg, "v1"
+  in: body
+  required: true
+  type: string
+versions:
+  description: |
+    Array of information about currently supported versions.
+  in: body
+  required: true
+  type: array

api-ref/source/samples/actionplan-list-detailed-response.json

@@ -20,7 +20,8 @@
             "deleted_at": null,
             "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
             "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
-            "created_at": "2018-04-10T11:59:52.640067+00:00"
+            "created_at": "2018-04-10T11:59:52.640067+00:00",
+            "hostname": "controller"
         }
     ]
 }

api-ref/source/samples/actionplan-show-response.json

@@ -16,5 +16,6 @@
     "updated_at": "2018-04-10T11:59:52.640067+00:00",
     "strategy_name": "dummy_with_resize",
     "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
-    "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a"
+    "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
+    "hostname": "controller"
 }

api-ref/source/samples/actionplan-start-response.json

@@ -18,5 +18,6 @@
     "uuid": "4cbc4ede-0d25-481b-b86e-998dbbd4f8bf",
     "audit_uuid": "7d100b05-0a86-491f-98a7-f93da19b272a",
     "created_at": "2018-04-10T11:59:12.592729+00:00",
-    "deleted_at": null
+    "deleted_at": null,
+    "hostname": null
 }

api-ref/source/samples/api-root-response.json

@@ -0,0 +1,30 @@
+{
+  "default_version": {
+    "id": "v1",
+    "links": [
+      {
+        "href": "http://controller:9322/v1/",
+        "rel": "self"
+      }
+    ],
+    "min_version": "1.0",
+    "status": "CURRENT",
+    "max_version": "1.1"
+  },
+  "description": "Watcher is an OpenStack project which aims to improve physical resources usage through better VM placement.",
+  "name": "OpenStack Watcher API",
+  "versions": [
+    {
+      "id": "v1",
+      "links": [
+        {
+          "href": "http://controller:9322/v1/",
+          "rel": "self"
+        }
+      ],
+      "min_version": "1.0",
+      "status": "CURRENT",
+      "max_version": "1.1"
+    }
+  ]
+}

api-ref/source/samples/api-v1-root-response.json

@@ -0,0 +1,80 @@
+{
+  "scoring_engines": [
+    {
+      "href": "http://controller:9322/v1/scoring_engines/",
+      "rel": "self"
+    },
+    {
+      "href": "http://controller:9322/scoring_engines/",
+      "rel": "bookmark"
+    }
+  ],
+  "media_types": [
+    {
+      "base": "application/json",
+      "type": "application/vnd.openstack.watcher.v1+json"
+    }
+  ],
+  "links": [
+    {
+      "href": "http://controller:9322/v1/",
+      "rel": "self"
+    },
+    {
+      "href": "http://docs.openstack.org/developer/watcher/dev/api-spec-v1.html",
+      "type": "text/html",
+      "rel": "describedby"
+    }
+  ],
+  "actions": [
+    {
+      "href": "http://controller:9322/v1/actions/",
+      "rel": "self"
+    },
+    {
+      "href": "http://controller:9322/actions/",
+      "rel": "bookmark"
+    }
+  ],
+  "audit_templates": [
+    {
+      "href": "http://controller:9322/v1/audit_templates/",
+      "rel": "self"
+    },
+    {
+      "href": "http://controller:9322/audit_templates/",
+      "rel": "bookmark"
+    }
+  ],
+  "action_plans": [
+    {
+      "href": "http://controller:9322/v1/action_plans/",
+      "rel": "self"
+    },
+    {
+      "href": "http://controller:9322/action_plans/",
+      "rel": "bookmark"
+    }
+  ],
+  "services": [
+    {
+      "href": "http://controller:9322/v1/services/",
+      "rel": "self"
+    },
+    {
+      "href": "http://controller:9322/services/",
+      "rel": "bookmark"
+    }
+  ],
+  "audits": [
+    {
+      "href": "http://controller:9322/v1/audits/",
+      "rel": "self"
+    },
+    {
+      "href": "http://controller:9322/audits/",
+      "rel": "bookmark"
+    }
+  ],
+  "id": "v1"
+}

api-ref/source/samples/audit-cancel-response.json

@@ -47,5 +47,8 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": "2018-04-06T11:56:00",
-    "updated_at": "2018-04-06T11:54:01.266447+00:00"
-}
+    "updated_at": "2018-04-06T11:54:01.266447+00:00",
+    "hostname": "controller",
+    "start_time": null,
+    "end_time": null
+}

api-ref/source/samples/audit-create-request-continuous.json

@@ -8,5 +8,7 @@
         ]
     },
     "audit_type": "CONTINUOUS",
-    "interval": "*/2 * * * *"
-}
+    "interval": "*/2 * * * *",
+    "start_time":"2018-04-02 20:30:00",
+    "end_time": "2018-04-04 20:30:00"
+}

api-ref/source/samples/audit-create-response.json

@@ -47,5 +47,8 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": null,
-    "updated_at": null
-}
+    "updated_at": null,
+    "hostname": null,
+    "start_time": null,
+    "end_time": null
+}

api-ref/source/samples/audit-list-detailed-response.json

@@ -49,7 +49,10 @@
             ],
             "strategy_name": "workload_stabilization",
             "next_run_time": "2018-04-06T09:46:00",
-            "updated_at": "2018-04-06T09:44:01.604146+00:00"
+            "updated_at": "2018-04-06T09:44:01.604146+00:00",
+            "hostname": "controller",
+            "start_time": null,
+            "end_time": null
         }
     ]
-}
+}

api-ref/source/samples/audit-show-response.json

@@ -47,5 +47,8 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": "2018-04-06T11:56:00",
-    "updated_at": "2018-04-06T11:54:01.266447+00:00"
-}
+    "updated_at": "2018-04-06T11:54:01.266447+00:00",
+    "hostname": "controller",
+    "start_time": null,
+    "end_time": null
+}

api-ref/source/samples/audit-update-response.json

@@ -47,5 +47,8 @@
     ],
     "strategy_name": "workload_stabilization",
     "next_run_time": "2018-04-06T11:56:00",
-    "updated_at": "2018-04-06T11:54:01.266447+00:00"
-}
+    "updated_at": "2018-04-06T11:54:01.266447+00:00",
+    "hostname": "controller",
+    "start_time": null,
+    "end_time": null
+}

api-ref/source/watcher-api-v1-actionplans.inc

@@ -50,6 +50,7 @@ version 1:
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
+    - hostname: actionplan_hostname
 
 **Example JSON representation of an Action Plan:**
 
@@ -137,6 +138,7 @@ Response
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
+    - hostname: actionplan_hostname
 
 **Example JSON representation of an Action Plan:**
 
@@ -174,6 +176,7 @@ Response
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
+    - hostname: actionplan_hostname
 
 **Example JSON representation of an Audit:**
 
@@ -229,6 +232,7 @@ version 1:
     - efficacy_indicators: actionplan_efficacy_indicators
     - global_efficacy: actionplan_global_efficacy
     - links: links
+    - hostname: actionplan_hostname
 
 **Example JSON representation of an Action Plan:**
 

api-ref/source/watcher-api-v1-audits.inc

@@ -28,10 +28,6 @@ itself. In the first case, there also should be supplied
 ``audit_template_uuid``. If ``Audit`` is created without ``Audit Template``,
 ``goal`` should be provided.
 
-.. warning::
-    **Only ``audit_template_uuid`` can be used to create audit so far.**
-    It should be fixed during the ``Rocky`` cycle.
-
 Normal response codes: 201
 
 Error codes: 400,404
@@ -41,15 +37,16 @@ Request
 
 .. rest_parameters:: parameters.yaml
 
-   - audit_template_uuid: audittemplate_name
+   - audit_template_uuid: audittemplate_uuid
    - audit_type: audit_type
    - name: audit_name
    - goal: audit_goal
    - strategy: audit_strategy
    - parameters: audit_parameters
    - interval: audit_interval
-   - scope: audittemplate_scope
    - auto_trigger: audit_autotrigger
+   - start_time: audit_starttime_req
+   - end_time: audit_endtime_req
 
 **Example ONESHOT Audit creation request:**
 
@@ -83,6 +80,9 @@ version 1:
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
+    - hostname: audit_hostname
+    - start_time: audit_starttime_resp
+    - end_time: audit_endtime_resp
 
 **Example JSON representation of an Audit:**
 
@@ -178,6 +178,9 @@ Response
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
+    - hostname: audit_hostname
+    - start_time: audit_starttime_resp
+    - end_time: audit_endtime_resp
 
 **Example JSON representation of an Audit:**
 
@@ -221,6 +224,9 @@ Response
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
+    - hostname: audit_hostname
+    - start_time: audit_starttime_resp
+    - end_time: audit_endtime_resp
 
 **Example JSON representation of an Audit:**
 
@@ -272,6 +278,9 @@ version 1:
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
+    - hostname: audit_hostname
+    - start_time: audit_starttime_resp
+    - end_time: audit_endtime_resp
 
 **Example JSON representation of an Audit:**
 
@@ -323,6 +332,9 @@ Response
     - state: audit_state
     - scope: audittemplate_scope
     - links: links
+    - hostname: audit_hostname
+    - start_time: audit_starttime_resp
+    - end_time: audit_endtime_resp
 
 **Example JSON representation of an Audit:**
 
@@ -346,4 +358,4 @@ Request
 
 .. rest_parameters:: parameters.yaml
 
-   - audit_ident: audit_ident
+   - audit_ident: audit_ident

api-ref/source/watcher-api-versions.inc

@@ -0,0 +1,81 @@
+.. -*- rst -*-
+
+============
+API versions
+============
+
+In order to bring new features to users over time, the Watcher API
+supports versioning. There are two kinds of versions in Watcher.
+
+- ''major versions'', which have dedicated URLs.
+- ''microversions'', which can be requested using the
+  ``OpenStack-API-Version`` header.
+
+.. note:: The maximum microversion depends on release.
+  Please reference:
+  `API Microversion History
+  <https://docs.openstack.org/watcher/latest/contributor/api_microversion_history.html>`__
+  for API microversion history details.
+
+The Version API resource works differently from other API resources as they *do not*
+require authentication.
+
+If Watcher receives a request with unsupported version, it responds with a 406 Not Acceptable,
+along with the -Min- and -Max- headers that it can support.
+
+List API versions
+=================
+
+.. rest_method::  GET /
+
+This fetches all the information about all known major API versions in the
+deployment. Links to more specific information will be provided for each major
+API version, as well as information about supported min and max microversions.
+
+Normal response codes: 200
+
+Request
+-------
+
+Response Example
+----------------
+
+.. rest_parameters::  parameters.yaml
+
+    - description: version_description
+    - versions: versions
+    - version: version
+    - id: version_id
+    - links: links
+    - min_version: openstack-api-min-version
+    - max_version: openstack-api-max-version
+
+.. literalinclude:: samples/api-root-response.json
+   :language: javascript
+
+
+Show v1 API
+===========
+
+.. rest_method::  GET /v1/
+
+Show all the resources within the Watcher v1 API.
+
+Normal response codes: 200
+
+Request
+-------
+
+Response Example
+----------------
+
+.. rest_parameters::  parameters.yaml
+
+    - id: version_id
+    - links: links
+    - OpenStack-API-Version: header_version
+    - OpenStack-API-Minimum-Version: openstack-api-min-version
+    - OpenStack-API-Maximum-Version: openstack-api-max-version
+
+.. literalinclude:: samples/api-v1-root-response.json
+   :language: javascript

devstack/lib/watcher

@@ -47,9 +47,6 @@ WATCHER_POLICY_YAML=$WATCHER_CONF_DIR/policy.yaml.sample
 WATCHER_DEVSTACK_DIR=$WATCHER_DIR/devstack
 WATCHER_DEVSTACK_FILES_DIR=$WATCHER_DEVSTACK_DIR/files
 
-NOVA_CONF_DIR=/etc/nova
-NOVA_CONF=$NOVA_CONF_DIR/nova.conf
-
 if is_ssl_enabled_service "watcher" || is_service_enabled tls-proxy; then
     WATCHER_SERVICE_PROTOCOL="https"
 fi
@@ -151,12 +148,17 @@ function create_watcher_accounts {
 function _config_watcher_apache_wsgi {
     local watcher_apache_conf
     if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+        local service_port=$WATCHER_SERVICE_PORT
+        if is_service_enabled tls-proxy; then
+            service_port=$WATCHER_SERVICE_PORT_INT
+            service_protocol="http"
+        fi
         sudo mkdir -p $WATCHER_WSGI_DIR
         sudo cp $WATCHER_DIR/watcher/api/app.wsgi $WATCHER_WSGI_DIR/app.wsgi
         watcher_apache_conf=$(apache_site_config_for watcher-api)
         sudo cp $WATCHER_DEVSTACK_FILES_DIR/apache-watcher-api.template $watcher_apache_conf
         sudo sed -e "
-            s|%WATCHER_SERVICE_PORT%|$WATCHER_SERVICE_PORT|g;
+            s|%WATCHER_SERVICE_PORT%|$service_port|g;
             s|%WATCHER_WSGI_DIR%|$WATCHER_WSGI_DIR|g;
             s|%USER%|$STACK_USER|g;
             s|%APIWORKERS%|$API_WORKERS|g;
@@ -193,9 +195,6 @@ function create_watcher_conf {
 
     iniset $WATCHER_CONF oslo_messaging_notifications driver "messagingv2"
 
-    iniset $NOVA_CONF oslo_messaging_notifications topics "notifications,watcher_notifications"
-    iniset $NOVA_CONF notifications notify_on_state_change "vm_and_task_state"
-
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR "watcher_clients_auth"
 
@@ -290,19 +289,22 @@ function start_watcher_api {
         service_protocol="http"
     fi
     if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+        enable_apache_site watcher-api
         restart_apache_server
     else
         run_process watcher-api "$WATCHER_BIN_DIR/watcher-api --config-file $WATCHER_CONF"
     fi
-    echo "Waiting for watcher-api to start..."
-    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$WATCHER_SERVICE_HOST:$service_port; then
-        die $LINENO "watcher-api did not start"
-    fi
 
     # Start proxies if enabled
     if is_service_enabled tls-proxy; then
         start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
     fi
+
+    echo "Waiting for watcher-api to start..."
+    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$WATCHER_SERVICE_HOST:$service_port; then
+        die $LINENO "watcher-api did not start"
+    fi
+
 }
 
 # start_watcher() - Start running processes, including screen
@@ -317,7 +319,7 @@ function start_watcher {
 function stop_watcher {
     if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         disable_apache_site watcher-api
-	restart_apache_server
+        restart_apache_server
     else
         stop_process watcher-api
     fi

devstack/local.conf.compute

@@ -44,6 +44,3 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
-notify_on_state_change = vm_and_task_state
-[notifications]
-notify_on_state_change = vm_and_task_state

devstack/local.conf.controller

@@ -48,6 +48,3 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
-notify_on_state_change = vm_and_task_state
-[notifications]
-notify_on_state_change = vm_and_task_state

devstack/upgrade/from_rocky/upgrade-watcher

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# ``upgrade-watcher``
+
+function configure_watcher_upgrade {
+    XTRACE=$(set +o | grep xtrace)
+    set -o xtrace
+
+    # Copy release-specific files
+    sudo cp $TARGET_RELEASE_DIR/watcher/etc/watcher/watcher.conf $WATCHER_CONF_DIR/watcher.conf
+    sudo cp $TARGET_RELEASE_DIR/watcher/etc/watcher/policy.yaml.sample $WATCHER_CONF_DIR/policy.yaml.sample
+
+    # reset to previous state
+    $XTRACE
+}

devstack/upgrade/resources.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+set -o errexit
+
+source $GRENADE_DIR/grenaderc
+source $GRENADE_DIR/functions
+
+source $TOP_DIR/openrc admin demo
+
+set -o xtrace
+
+function _wait_for_status {
+     while :
+     do
+         state=$("${@:2}" -f value -c State)
+         [[ $state == "SUCCEEDED" ]] && break
+         if [ $state == "ERROR" ]; then
+             die $LINENO "ERROR creating audit"
+         fi
+         sleep 10
+     done
+ }
+
+function create_audit_template {
+    at_id=$(openstack optimize audittemplate create d1 dummy -s dummy -f value -c UUID)
+    resource_save watcher at_id $at_id
+ }
+
+function create_audit {
+    audit_id=$(openstack optimize audit create -s dummy -g dummy -f value -c UUID)
+    resource_save watcher audit_id $audit_id
+}
+
+function create_audit_with_autotrigger {
+    audit_at_id=$(openstack optimize audit create -s dummy -g dummy -f value -c UUID --auto-trigger)
+    resource_save watcher audit_at_id $audit_at_id
+}
+
+function verify_audit_template {
+    local at_id=$(resource_get watcher at_id)
+    openstack optimize audittemplate show $at_id
+}
+
+function verify_audit_with_autotrigger {
+    local audit_at_id=$(resource_get watcher audit_at_id)
+    _wait_for_status "SUCCEEDED" openstack optimize audit show $audit_at_id
+    local actionplan_at_id=$(openstack optimize actionplan list --audit $audit_at_id -c UUID -f value)
+    resource_save watcher actionplan_at $actionplan_at_id
+    actionplan_at_state=$(openstack optimize actionplan show $actionplan_at_id -c State -f value)
+    if [ $actionplan_at_state != "SUCCEEDED" ]; then
+        die $LINENO "ERROR executing actionplan"
+    fi
+}
+
+function verify_audit {
+    local audit_id=$(resource_get watcher audit_id)
+    _wait_for_status "SUCCEEDED" openstack optimize audit show $audit_id
+    local actionplan_id=$(openstack optimize actionplan list --audit $audit_id -c UUID -f value)
+    resource_save watcher actionplan $actionplan_id
+    actionplan_state=$(openstack optimize actionplan show $actionplan_id -c State -f value)
+    if [ $actionplan_state != "RECOMMENDED" ]; then
+        die $LINENO "ERROR creating actionplan"
+    fi
+}
+
+function verify_noapi {
+    # currently no good way
+    :
+}
+
+function delete_audit {
+  local audit_id=$(resource_get watcher audit_id)
+  local actionplan_id=$(resource_get watcher actionplan)
+  watcher actionplan delete $actionplan_id
+  openstack optimize audit delete $audit_id
+}
+
+function delete_audit_with_autotrigger {
+  local audit_at_id=$(resource_get watcher audit_at_id)
+  local actionplan_id=$(resource_get watcher actionplan_at)
+  watcher actionplan delete $actionplan_id
+  openstack optimize audit delete $audit_at_id
+}
+
+function delete_audit_template {
+  local at_id=$(resource_get watcher at_id)
+  openstack optimize audittemplate delete $at_id
+}
+
+function create {
+  create_audit_template
+  create_audit
+  create_audit_with_autotrigger
+}
+
+function verify {
+  verify_audit_template
+  verify_audit
+  verify_audit_with_autotrigger
+}
+
+function destroy {
+  delete_audit_template
+  delete_audit
+  delete_audit_with_autotrigger
+}
+
+# Dispatcher
+case $1 in
+    "create")
+        create
+        ;;
+    "verify_noapi")
+        verify_noapi
+        ;;
+    "verify")
+        verify
+        ;;
+    "destroy")
+        destroy
+        ;;
+    "force_destroy")
+        set +o errexit
+        destroy
+        ;;
+esac

devstack/upgrade/settings

@@ -0,0 +1,11 @@
+register_project_for_upgrade watcher
+register_db_to_save watcher
+
+devstack_localrc base enable_plugin watcher https://git.openstack.org/openstack/watcher stable/rocky
+devstack_localrc target enable_plugin watcher https://git.openstack.org/openstack/watcher
+
+devstack_localrc base enable_service watcher-api watcher-decision-engine watcher-applier
+devstack_localrc target enable_service watcher-api watcher-decision-engine watcher-applier
+
+BASE_RUN_SMOKE=False
+TARGET_RUN_SMOKE=False

devstack/upgrade/shutdown.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -o errexit
+
+source $GRENADE_DIR/grenaderc
+source $GRENADE_DIR/functions
+
+# We need base DevStack functions for this
+source $BASE_DEVSTACK_DIR/functions
+source $BASE_DEVSTACK_DIR/stackrc # needed for status directory
+source $BASE_DEVSTACK_DIR/lib/tls
+source $BASE_DEVSTACK_DIR/lib/apache
+
+WATCHER_DEVSTACK_DIR=$(dirname $(dirname $0))
+source $WATCHER_DEVSTACK_DIR/settings
+source $WATCHER_DEVSTACK_DIR/plugin.sh
+source $WATCHER_DEVSTACK_DIR/lib/watcher
+
+set -o xtrace
+
+stop_watcher
+
+# sanity check that service is actually down
+ensure_services_stopped watcher-api watcher-decision-engine watcher-applier

devstack/upgrade/upgrade.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# ``upgrade-watcher``
+
+echo "*********************************************************************"
+echo "Begin $0"
+echo "*********************************************************************"
+
+# Clean up any resources that may be in use
+cleanup() {
+    set +o errexit
+
+    echo "********************************************************************"
+    echo "ERROR: Abort $0"
+    echo "********************************************************************"
+
+    # Kill ourselves to signal any calling process
+    trap 2; kill -2 $$
+}
+
+trap cleanup SIGHUP SIGINT SIGTERM
+
+# Keep track of the grenade directory
+RUN_DIR=$(cd $(dirname "$0") && pwd)
+
+# Source params
+source $GRENADE_DIR/grenaderc
+
+# Import common functions
+source $GRENADE_DIR/functions
+
+# This script exits on an error so that errors don't compound and you see
+# only the first error that occurred.
+set -o errexit
+
+# Upgrade watcher
+# ============
+
+# Get functions from current DevStack
+source $TARGET_DEVSTACK_DIR/stackrc
+source $TARGET_DEVSTACK_DIR/lib/apache
+source $TARGET_DEVSTACK_DIR/lib/tls
+source $(dirname $(dirname $BASH_SOURCE))/settings
+source $(dirname $(dirname $BASH_SOURCE))/plugin.sh
+
+# Print the commands being run so that we can see the command that triggers
+# an error.  It is also useful for following allowing as the install occurs.
+set -o xtrace
+
+# Save current config files for posterity
+[[ -d $SAVE_DIR/etc.watcher ]] || cp -pr $WATCHER_CONF_DIR $SAVE_DIR/etc.watcher
+
+# Install the target watcher
+install_watcher
+
+# calls upgrade-watcher for specific release
+upgrade_project watcher $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
+
+# Migrate the database
+watcher-db-manage upgrade || die $LINO "DB migration error"
+
+start_watcher
+
+# Don't succeed unless the services come up
+ensure_services_started watcher-api watcher-decision-engine watcher-applier
+
+set +o xtrace
+echo "*********************************************************************"
+echo "SUCCESS: End $0"
+echo "*********************************************************************"

doc/requirements.txt

@@ -0,0 +1,10 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+openstackdocstheme>=1.20.0 # Apache-2.0
+sphinx>=1.6.5,!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7' # BSD
+sphinx>=1.6.5,!=1.6.6,!=1.6.7;python_version>='3.4' # BSD
+sphinxcontrib-pecanwsme>=0.8.0 # Apache-2.0
+reno>=2.7.0 # Apache-2.0
+sphinxcontrib-apidoc>=0.2.0  # BSD
+os-api-ref>=1.4.0 # Apache-2.0

doc/source/admin/apache-mod-wsgi.rst

@@ -44,6 +44,6 @@ Installing API behind mod_wsgi
     Fedora/RHEL7/CentOS7:
       sudo systemctl reload httpd
 
-   Debian/Ubuntu:
-    sudo a2ensite watcher
-    sudo service apache2 reload
+    Debian/Ubuntu:
+      sudo a2ensite watcher
+      sudo service apache2 reload

doc/source/api/index.rst

@@ -1,8 +0,0 @@
-==================================================
-OpenStack Infrastructure Optimization Service APIs
-==================================================
-
-.. toctree::
-   :maxdepth: 1
-
-   v1

doc/source/api/v1.rst

@@ -1,100 +0,0 @@
-..
-      Except where otherwise noted, this document is licensed under Creative
-      Commons Attribution 3.0 License.  You can view the license at:
-
-          https://creativecommons.org/licenses/by/3.0/
-
-====================
-RESTful Web API (v1)
-====================
-
-Goals
-=====
-
-.. rest-controller:: watcher.api.controllers.v1.goal:GoalsController
-   :webprefix: /v1/goal
-
-.. autotype:: watcher.api.controllers.v1.goal.GoalCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.goal.Goal
-   :members:
-
-Strategies
-==========
-
-.. rest-controller:: watcher.api.controllers.v1.strategy:StrategiesController
-   :webprefix: /v1/strategies
-
-.. autotype:: watcher.api.controllers.v1.strategy.StrategyCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.strategy.Strategy
-   :members:
-
-Audit Templates
-===============
-
-.. rest-controller:: watcher.api.controllers.v1.audit_template:AuditTemplatesController
-   :webprefix: /v1/audit_templates
-
-.. autotype:: watcher.api.controllers.v1.audit_template.AuditTemplateCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.audit_template.AuditTemplate
-   :members:
-
-Audits
-======
-
-.. rest-controller:: watcher.api.controllers.v1.audit:AuditsController
-   :webprefix: /v1/audits
-
-.. autotype:: watcher.api.controllers.v1.audit.AuditCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.audit.Audit
-   :members:
-
-Links
-=====
-
-.. autotype:: watcher.api.controllers.link.Link
-   :members:
-
-Action Plans
-============
-
-.. rest-controller:: watcher.api.controllers.v1.action_plan:ActionPlansController
-   :webprefix: /v1/action_plans
-
-.. autotype:: watcher.api.controllers.v1.action_plan.ActionPlanCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.action_plan.ActionPlan
-   :members:
-
-
-Actions
-=======
-
-.. rest-controller:: watcher.api.controllers.v1.action:ActionsController
-   :webprefix: /v1/actions
-
-.. autotype:: watcher.api.controllers.v1.action.ActionCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.action.Action
-   :members:
-
-Scoring Engine
-==============
-
-.. rest-controller:: watcher.api.controllers.v1.scoring_engine:ScoringEngineController
-   :webprefix: /v1/scoring_engine
-
-.. autotype:: watcher.api.controllers.v1.scoring_engine.ScoringEngineCollection
-   :members:
-
-.. autotype:: watcher.api.controllers.v1.scoring_engine.ScoringEngine
-   :members:

doc/source/architecture.rst

@@ -76,6 +76,7 @@ Watcher Applier
 This component is in charge of executing the
 :ref:`Action Plan <action_plan_definition>` built by the
 :ref:`Watcher Decision Engine <watcher_decision_engine_definition>`.
+Taskflow is the default workflow engine for Watcher.
 
 It connects to the :ref:`message bus <amqp_bus_definition>` and launches the
 :ref:`Action Plan <action_plan_definition>` whenever a triggering message is
@@ -110,6 +111,23 @@ If the :ref:`Action <action_definition>` fails, the
 previous state of the :ref:`Managed resource <managed_resource_definition>`
 (i.e. before the command was sent to the underlying OpenStack service).
 
+In Stein, added a new config option 'action_execution_rule' which is a
+dict type. Its key field is strategy name and the value is 'ALWAYS' or 'ANY'.
+'ALWAYS' means the callback function returns True as usual.
+'ANY' means the return depends on the result of previous action execution.
+The callback returns True if previous action gets failed, and the engine
+continues to run the next action. If previous action executes success,
+the callback returns False then the next action will be ignored.
+For strategies that aren't in 'action_execution_rule', the callback always
+returns True.
+Please add the next section in the watcher.conf file
+if your strategy needs this feature.
+
+::
+
+    [watcher_workflow_engines.taskflow]
+    action_execution_rule = {'your strategy name': 'ANY'}
+
 .. _archi_watcher_cli_definition:
 
 Watcher CLI
@@ -461,4 +479,4 @@ change to a new value:
 
 
 
-.. _Watcher API: webapi/v1.html
+.. _Watcher API: https://developer.openstack.org/api-ref/resource-optimization/

doc/source/conf.py

@@ -32,7 +32,7 @@ sys.path.insert(0, os.path.abspath('./'))
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'oslo_config.sphinxext',
-    'sphinx.ext.autodoc',
+    'sphinxcontrib.apidoc',
     'sphinx.ext.viewcode',
     'sphinxcontrib.httpdomain',
     'sphinxcontrib.pecanwsme.rest',
@@ -55,6 +55,18 @@ sample_config_basename = 'watcher'
 # text edit cycles.
 # execute "export SPHINX_DEBUG=1" in your terminal to disable
 
+# sphinxcontrib.apidoc options
+apidoc_module_dir = '../../watcher'
+apidoc_output_dir = 'api'
+apidoc_excluded_paths = [
+    'tests/*',
+    'db',
+    'decision_engine',
+    'doc',
+    'objects',
+]
+apidoc_separate_modules = True
+
 # The suffix of source filenames.
 source_suffix = '.rst'
 

doc/source/configuration/configuring.rst

@@ -159,7 +159,7 @@ MySQL database that is used by other OpenStack services.
    ``watcher`` user. Replace WATCHER_DBPASSWORD
    with the actual password::
 
-    $ mysql -u root -p
+    # mysql
 
     mysql> CREATE DATABASE watcher CHARACTER SET utf8;
     mysql> GRANT ALL PRIVILEGES ON watcher.* TO 'watcher'@'localhost' \
@@ -403,26 +403,22 @@ Watcher can consume notifications generated by the Nova services, in order to
 build or update, in real time, its cluster data model related to computing
 resources.
 
-Nova publishes, by default, notifications on ``notifications`` AMQP queue
-(configurable) and ``versioned_notifications`` AMQP queue (not
-configurable). ``notifications`` queue is mainly used by ceilometer, so we can
-not use it. And some events, related to nova-compute service state, are only
-sent into the ``versioned_notifications`` queue.
+Nova emits unversioned(legacy) and versioned notifications on different
+topics. Because legacy notifications will be deprecated, Watcher consumes
+Nova versioned notifications.
 
-By default, Watcher listens to AMQP queues named ``watcher_notifications``
-and ``versioned_notifications``. So you have to update the Nova
-configuration file on controller and compute nodes, in order
-to Watcher receives Nova notifications in ``watcher_notifications`` as well.
-
-  * In the file ``/etc/nova/nova.conf``, update the section
-    ``[oslo_messaging_notifications]``, by redefining the list of topics
-    into which Nova services will publish events ::
+  * In the file ``/etc/nova/nova.conf``, the value of driver in the section
+    ``[oslo_messaging_notifications]`` can't be noop, and the value of
+    notification_format in the section ``[notifications]``
+    should be both or versioned ::
 
       [oslo_messaging_notifications]
       driver = messagingv2
-      topics = notifications,watcher_notifications
 
-  * Restart the Nova services.
+      ...
+
+      [notifications]
+      notification_format = both
 
 
 Configure Cinder Notifications

doc/source/contributor/api_microversion_history.rst

@@ -0,0 +1 @@
+.. include:: ../../../watcher/api/controllers/rest_api_version_history.rst

doc/source/contributor/contributing.rst

@@ -49,7 +49,7 @@ Bug tracker
     https://launchpad.net/watcher
 
 Mailing list (prefix subjects with ``[watcher]`` for faster responses)
-    http://lists.openstack.org/pipermail/openstack-dev/
+    http://lists.openstack.org/pipermail/openstack-discuss/
 
 Wiki
     https://wiki.openstack.org/Watcher

doc/source/contributor/plugin/strategy-plugin.rst

@@ -245,15 +245,16 @@ Querying metrics
 
 A large set of metrics, generated by OpenStack modules, can be used in your
 strategy implementation. To collect these metrics, Watcher provides a
-`Helper`_ for two data sources which are `Ceilometer`_ and `Monasca`_. If you
-wish to query metrics from a different data source, you can implement your own
-and directly use it from within your new strategy. Indeed, strategies in
-Watcher have the cluster data models decoupled from the data sources which
-means that you may keep the former while changing the latter.
-The recommended way for you to support a new data source is to implement a new
-helper that would encapsulate within separate methods the queries you need to
-perform. To then use it, you would just have to instantiate it within your
-strategy.
+`DataSourceManager`_ for two data sources which are `Ceilometer`_
+(with `Gnocchi`_ as API) and `Monasca`_. If you wish to query metrics from a
+different data source, you can implement your own and use it via
+DataSourceManager from within your new strategy. Indeed, strategies in Watcher
+have the cluster data models decoupled from the data sources which means that
+you may keep the former while changing the latter. The recommended way for you
+to support a new data source is to implement a new helper that would
+encapsulate within separate methods the queries you need to perform. To then
+use it, you would just have to add it to appropriate watcher_strategies.*
+section in config file.
 
 If you want to use Ceilometer but with your own metrics database backend,
 please refer to the `Ceilometer developer guide`_. The list of the available
@@ -263,52 +264,38 @@ requires new metrics not covered by Ceilometer, you can add them through a
 `Ceilometer plugin`_.
 
 
-.. _`Helper`: https://github.com/openstack/watcher/blob/master/watcher/datasource/ceilometer.py
+.. _`DataSourceManager`: https://github.com/openstack/watcher/blob/master/watcher/datasource/manager.py
 .. _`Ceilometer developer guide`: https://docs.openstack.org/ceilometer/latest/contributor/architecture.html#storing-accessing-the-data
 .. _`Ceilometer`: https://docs.openstack.org/ceilometer/latest
 .. _`Monasca`: https://github.com/openstack/monasca-api/blob/master/docs/monasca-api-spec.md
 .. _`here`: https://docs.openstack.org/ceilometer/latest/contributor/install/dbreco.html#choosing-a-database-backend
 .. _`Ceilometer plugin`: https://docs.openstack.org/ceilometer/latest/contributor/plugins.html
 .. _`Ceilosca`: https://github.com/openstack/monasca-ceilometer/blob/master/ceilosca/ceilometer/storage/impl_monasca.py
+.. _`Gnocchi`: https://gnocchi.xyz/
 
 Read usage metrics using the Watcher Datasource Helper
 ------------------------------------------------------
 
-The following code snippet shows how to invoke a Datasource Helper class:
+The following code snippet shows how datasource_backend is defined:
 
 .. code-block:: py
 
-    from watcher.datasource import ceilometer as ceil
-    from watcher.datasource import monasca as mon
+    from watcher.datasource import manager as ds_manager
 
     @property
-    def ceilometer(self):
-        if self._ceilometer is None:
-            self._ceilometer = ceil.CeilometerHelper(osc=self.osc)
-        return self._ceilometer
-
-    @property
-    def monasca(self):
-        if self._monasca is None:
-            self._monasca = mon.MonascaHelper(osc=self.osc)
-        return self._monasca
+    def datasource_backend(self):
+        if not self._datasource_backend:
+            self._datasource_backend = ds_manager.DataSourceManager(
+                config=self.config,
+                osc=self.osc
+            ).get_backend(self.DATASOURCE_METRICS)
+        return self._datasource_backend
 
 Using that you can now query the values for that specific metric:
 
 .. code-block:: py
 
-        if self.config.datasource == "ceilometer":
-            resource_id = "%s_%s" % (node.uuid, node.hostname)
-            return self.ceilometer.statistic_aggregation(
-                resource_id=resource_id,
-                meter_name='compute.node.cpu.percent',
-                period="7200",
-                aggregate='avg',
-            )
-        elif self.config.datasource == "monasca":
-            statistics = self.monasca.statistic_aggregation(
-                meter_name='compute.node.cpu.percent',
-                dimensions=dict(hostname=node.uuid),
-                period=7200,
-                aggregate='avg'
-            )
+    avg_meter = self.datasource_backend.statistic_aggregation(
+        instance.uuid, 'cpu_util', self.periods['instance'],
+        self.granularity,
+        aggregation=self.aggregation_method['instance'])

doc/source/contributor/testing.rst

@@ -13,39 +13,32 @@ Testing
 Unit tests
 ==========
 
-All unit tests should be run using `tox`_. To run the same unit tests that are
-executing onto `Gerrit`_ which includes ``py35``, ``py27`` and ``pep8``, you
-can issue the following command::
+All unit tests should be run using `tox`_. Before running the unit tests, you
+should download the latest `watcher`_ from the github. To run the same unit
+tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
+``pep8``, you can issue the following command::
 
-    $ workon watcher
-    (watcher) $ pip install tox
-    (watcher) $ cd watcher
-    (watcher) $ tox
+    $ git clone https://git.openstack.org/openstack/watcher
+    $ cd watcher
+    $ pip install tox
+    $ tox
 
-If you want to only run one of the aforementioned, you can then issue one of
+If you only want to run one of the aforementioned, you can then issue one of
 the following::
 
-    $ workon watcher
-    (watcher) $ tox -e py35
-    (watcher) $ tox -e py27
-    (watcher) $ tox -e pep8
+    $ tox -e py35
+    $ tox -e py27
+    $ tox -e pep8
 
 .. _tox: https://tox.readthedocs.org/
+.. _watcher: https://git.openstack.org/cgit/openstack/watcher
 .. _Gerrit: https://review.openstack.org/
 
-You may pass options to the test programs using positional arguments. To run a
-specific unit test, you can pass extra options to `os-testr`_ after putting
-the ``--`` separator. So using the ``-r`` option followed by a regex string,
-you can run the desired test::
+If you only want to run specific unit test code and don't like to waste time
+waiting for all unit tests to execute, you can add parameters ``--`` followed
+by a regex string::
 
-    $ workon watcher
-    (watcher) $ tox -e py27 -- -r watcher.tests.api
-
-.. _os-testr: https://docs.openstack.org/os-testr/latest
-
-When you're done, deactivate the virtualenv::
-
-    $ deactivate
+    $ tox -e py27 -- watcher.tests.api
 
 .. _tempest_tests:
 
@@ -55,4 +48,4 @@ Tempest tests
 Tempest tests for Watcher has been migrated to the external repo
 `watcher-tempest-plugin`_.
 
-.. _watcher-tempest-plugin: https://github.com/openstack/watcher-tempest-plugin
+.. _watcher-tempest-plugin: https://git.openstack.org/cgit/openstack/watcher-tempest-plugin

doc/source/image_src/plantuml/audit_state_machine.txt

@@ -2,7 +2,7 @@
 
 [*] --> PENDING: Audit requested by Administrator
 PENDING --> ONGOING: Audit request is received\nby the Watcher Decision Engine
-ONGOING --> FAILED: Audit fails\n(no solution found, technical error, ...)
+ONGOING --> FAILED: Audit fails\n(Exception occurred)
 ONGOING --> SUCCEEDED: The Watcher Decision Engine\ncould find at least one Solution
 ONGOING --> SUSPENDED: Administrator wants to\nsuspend the Audit
 SUSPENDED --> ONGOING: Administrator wants to\nresume the Audit

doc/source/index.rst

@@ -78,12 +78,13 @@ User Guide
   user/index
 
 API References
---------------
+==============
 
 .. toctree::
   :maxdepth: 1
 
-  api/index
+  API Reference <https://developer.openstack.org/api-ref/resource-optimization/>
+  Watcher API Microversion History </contributor/api_microversion_history>
 
 Plugins
 -------
@@ -117,7 +118,7 @@ Watcher Manual Pages
 .. toctree::
    :hidden:
 
-   api/autoindex
+   api/modules
 
 
 Indices and tables

doc/source/install/common_prerequisites.rst

@@ -11,7 +11,7 @@ you must create a database, service credentials, and API endpoints.
 
      .. code-block:: console
 
-        $ mysql -u root -p
+        # mysql
 
    * Create the ``watcher`` database:
 

doc/source/install/install-ubuntu.rst

@@ -4,7 +4,7 @@ Install and configure for Ubuntu
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 This section describes how to install and configure the Infrastructure
-Optimization service for Ubuntu 14.04 (LTS).
+Optimization service for Ubuntu 16.04 (LTS).
 
 .. include:: common_prerequisites.rst
 
@@ -25,10 +25,15 @@ Install and configure components
 Finalize installation
 ---------------------
 
-Restart the Infrastructure Optimization services:
+Start the Infrastructure Optimization services and configure them to start when
+the system boots:
 
 .. code-block:: console
 
-   # service watcher-api restart
-   # service watcher-decision-engine restart
-   # service watcher-applier restart
+   # systemctl enable watcher-api.service \
+     watcher-decision-engine.service \
+     watcher-applier.service
+
+   # systemctl start watcher-api.service \
+     watcher-decision-engine.service \
+     watcher-applier.service

doc/source/man/index.rst

@@ -5,4 +5,5 @@
    watcher-api
    watcher-applier
    watcher-db-manage
-   watcher-decision-engine
+   watcher-decision-engine
+   watcher-status

doc/source/man/watcher-status.rst

@@ -0,0 +1,83 @@
+==============
+watcher-status
+==============
+
+-----------------------------------------
+CLI interface for Watcher status commands
+-----------------------------------------
+
+Synopsis
+========
+
+::
+
+  watcher-status <category> <command> [<args>]
+
+Description
+===========
+
+:program:`watcher-status` is a tool that provides routines for checking the
+status of a Watcher deployment.
+
+Options
+=======
+
+The standard pattern for executing a :program:`watcher-status` command is::
+
+    watcher-status <category> <command> [<args>]
+
+Run without arguments to see a list of available command categories::
+
+    watcher-status
+
+Categories are:
+
+* ``upgrade``
+
+Detailed descriptions are below:
+
+You can also run with a category argument such as ``upgrade`` to see a list of
+all commands in that category::
+
+    watcher-status upgrade
+
+These sections describe the available categories and arguments for
+:program:`Watcher-status`.
+
+Upgrade
+~~~~~~~
+
+.. _watcher-status-checks:
+
+``watcher-status upgrade check``
+  Performs a release-specific readiness check before restarting services with
+  new code. For example, missing or changed configuration options,
+  incompatible object states, or other conditions that could lead to
+  failures while upgrading.
+
+  **Return Codes**
+
+  .. list-table::
+     :widths: 20 80
+     :header-rows: 1
+
+     * - Return code
+       - Description
+     * - 0
+       - All upgrade readiness checks passed successfully and there is nothing
+         to do.
+     * - 1
+       - At least one check encountered an issue and requires further
+         investigation. This is considered a warning but the upgrade may be OK.
+     * - 2
+       - There was an upgrade status check failure that needs to be
+         investigated. This should be considered something that stops an
+         upgrade.
+     * - 255
+       - An unexpected error occurred.
+
+  **History of Checks**
+
+  **1.12.0 (Stein)**
+
+  * Sample check to be filled in with checks as they are added in Stein.

doc/source/strategies/vm_workload_consolidation.rst

@@ -22,20 +22,12 @@ The *vm_workload_consolidation* strategy requires the following metrics:
 ============================ ============ ======= =======
 metric                       service name plugins comment
 ============================ ============ ======= =======
+``cpu_util``                 ceilometer_  none
+``memory.resident``          ceilometer_  none
 ``memory``                   ceilometer_  none
 ``disk.root.size``           ceilometer_  none
 ============================ ============ ======= =======
 
-The following metrics are not required but increase the accuracy of
-the strategy if available:
-
-============================ ============ ======= =======
-metric                       service name plugins comment
-============================ ============ ======= =======
-``memory.resident``          ceilometer_  none
-``cpu_util``                 ceilometer_  none
-============================ ============ ======= =======
-
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
 
 Cluster data model

doc/source/user/user-guide.rst

@@ -182,7 +182,7 @@ periodically calling:
 
 .. code:: bash
 
-  $ openstack optimize action list
+  $ openstack optimize action list --action-plan <the_action_plan_uuid>
 
 You can also obtain more detailed information about a specific action:
 

lower-constraints.txt

@@ -57,6 +57,7 @@ lxml==4.1.1
 Mako==1.0.7
 MarkupSafe==1.0
 mccabe==0.2.1
+microversion_parse==0.2.1
 mock==2.0.0
 monotonic==1.4
 mox3==0.25.0
@@ -85,6 +86,7 @@ oslo.policy==1.34.0
 oslo.reports==1.27.0
 oslo.serialization==2.25.0
 oslo.service==1.30.0
+oslo.upgradecheck==0.1.0
 oslo.utils==3.36.0
 oslo.versionedobjects==1.32.0
 oslotest==3.3.0

playbooks/legacy/grenade-devstack-watcher/post.yaml

@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/grenade-devstack-watcher/run.yaml

@@ -0,0 +1,60 @@
+- hosts: all
+  name: legacy-grenade-dsvm-watcher
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              https://opendev.org \
+              openstack/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+
+          export PROJECTS="openstack/grenade $PROJECTS"
+          export PROJECTS="openstack/watcher $PROJECTS"
+          export PROJECTS="openstack/watcher-tempest-plugin $PROJECTS"
+          export PROJECTS="openstack/python-watcherclient $PROJECTS"
+          export DEVSTACK_PROJECT_FROM_GIT="python-watcherclient $DEVSTACK_PROJECT_FROM_GIT"
+
+          export GRENADE_PLUGINRC="enable_grenade_plugin watcher https://opendev.org/openstack/watcher"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"export TEMPEST_PLUGINS='/opt/stack/new/watcher-tempest-plugin'"
+
+          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
+          export DEVSTACK_GATE_GRENADE=pullup
+
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+          # Add configuration values for enabling security features in local.conf
+          function pre_test_hook {
+              if [ -f /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh ] ; then
+                  . /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh
+              fi
+          }
+          export -f pre_test_hook
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/orchestrate-tempest.yaml

@@ -1,14 +0,0 @@
-- hosts: all
-  # This is the default strategy, however since orchestrate-devstack requires
-  # "linear", it is safer to enforce it in case this is running in an
-  # environment configured with a different default strategy.
-  strategy: linear
-  roles:
-    - orchestrate-devstack
-
-- hosts: tempest
-  roles:
-    - setup-tempest-run-dir
-    - setup-tempest-data-dir
-    - acl-devstack-files
-    - run-tempest

playbooks/pre.yaml

@@ -1,3 +0,0 @@
-- hosts: all
-  roles:
-    - add-hostnames-to-hosts

releasenotes/notes/add-baremetal-scoper-9ef23f5fb8f0be6a.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Baremetal Model gets Audit scoper with an ability to exclude Ironic nodes.

releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add start_time and end_time fields in audits table. User can set the start
+    time and/or end time when creating CONTINUOUS audit.

releasenotes/notes/add-upgrade-check-framework-5bb9693c8a78931c.yaml

@@ -0,0 +1,13 @@
+---
+prelude: >
+    Added new tool ``watcher-status upgrade check``.
+features:
+  - |
+    New framework for ``watcher-status upgrade check`` command is added.
+    This framework allows adding various checks which can be run before a
+    Watcher upgrade to ensure if the upgrade can be performed safely.
+upgrade:
+  - |
+    Operator can now use new CLI tool ``watcher-status upgrade check``
+    to check if Watcher deployment can be safely upgraded from
+    N-1 to N release.

releasenotes/notes/api-microversioning-7999a3ee8073bf32.yaml

@@ -0,0 +1,8 @@
+---
+features:
+    Watcher starts to support API microversions since Stein cycle. From now
+    onwards all API changes should be made with saving backward compatibility.
+    To specify API version operator should use OpenStack-API-Version
+    HTTP header. If operator wants to know the mininum and maximum supported
+    versions by API, he/she can access /v1 resource and Watcher API will
+    return appropriate headers in response.

releasenotes/notes/consume-nova-versioned-notifications-f98361b37e546b4d.yaml

@@ -0,0 +1,60 @@
+---
+features:
+  - |
+    Watcher consumes Nova notifications to update its internal
+    Compute CDM(Cluster Data Model).
+    All the notifications as below
+
+    pre-existing:
+
+    * service.update
+
+    * instance.update
+
+    * instance.delete.end
+
+    new:
+
+    * instance.lock
+
+    * instance.unlock
+
+    * instance.pause.end
+
+    * instance.power_off.end
+
+    * instance.power_on.end
+
+    * instance.resize_confirm.end
+
+    * instance.restore.end
+
+    * instance.resume.end
+
+    * instance.shelve.end
+
+    * instance.shutdown.end
+
+    * instance.suspend.end
+
+    * instance.unpause.end
+
+    * instance.unrescue.end
+
+    * instance.unshelve.end
+
+    * instance.rebuild.end
+
+    * instance.rescue.end
+
+    * instance.create.end
+
+    * instance.live_migration_force_complete.end
+
+    * instance.live_migration_post_dest.end
+
+    * instance.soft_delete.end
+
+    * service.create
+
+    * service.delete

releasenotes/notes/deprecate-ceilometer-datasource-446b0be70fbce28b.yaml

@@ -0,0 +1,7 @@
+---
+deprecations:
+  - |
+    Ceilometer Datasource has been deprecated since its API has been
+    deprecated in Ocata cycle. Watcher has supported Ceilometer for some
+    releases after Ocata to let users migrate to Gnocchi/Monasca datasources.
+    Since Train release, Ceilometer support will be removed.

releasenotes/notes/enhance-watcher-applier-engine-86c676ce8f179e68.yaml

@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    Added a new config option 'action_execution_rule' which is a dict type.
+    Its key field is strategy name and the value is 'ALWAYS' or 'ANY'.
+    'ALWAYS' means the callback function returns True as usual.
+    'ANY' means the return depends on the result of previous action execution.
+    The callback returns True if previous action gets failed, and the engine
+    continues to run the next action. If previous action executes success,
+    the callback returns False then the next action will be ignored.
+    For strategies that aren't in 'action_execution_rule', the callback always
+    returns True.
+    Please add the next section in the watcher.conf file
+    if your strategy needs this feature.
+    [watcher_workflow_engines.taskflow]
+    action_execution_rule = {'your strategy name': 'ANY'}

releasenotes/notes/remove-nova-legacy-notifications-e1b6d10eff58f30a.yaml

@@ -0,0 +1,5 @@
+---
+deprecations:
+  - |
+    Watcher removes the support to Nova legacy notifications because of Nova
+    will deprecate them.

releasenotes/notes/scope-for-data-model-ea9792f90db14343.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    For a large cloud infrastructure, retrieving data from Nova may take
+    a long time. To avoid getting too much data from Nova, building the
+    compute data model according to the scope of audit.

releasenotes/source/index.rst

@@ -21,6 +21,7 @@ Contents:
    :maxdepth: 1
 
    unreleased
+   rocky
    queens
    pike
    ocata

releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po

@@ -2,13 +2,13 @@
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
 msgid ""
 msgstr ""
-"Project-Id-Version: watcher\n"
+"Project-Id-Version: python-watcher\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-02-28 12:27+0000\n"
+"POT-Creation-Date: 2018-11-08 01:22+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2018-02-16 07:20+0000\n"
+"PO-Revision-Date: 2018-11-07 06:15+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -27,6 +27,12 @@ msgstr "1.0.0"
 msgid "1.1.0"
 msgstr "1.1.0"
 
+msgid "1.10.0"
+msgstr "1.10.0"
+
+msgid "1.11.0"
+msgstr "1.11.0"
+
 msgid "1.3.0"
 msgstr "1.3.0"
 
@@ -45,6 +51,9 @@ msgstr "1.6.0"
 msgid "1.7.0"
 msgstr "1.7.0"
 
+msgid "1.9.0"
+msgstr "1.9.0"
+
 msgid "Add a service supervisor to watch Watcher deamons."
 msgstr "Add a service supervisor to watch Watcher daemons."
 
@@ -70,6 +79,13 @@ msgstr "Add notifications related to Audit object."
 msgid "Add notifications related to Service object."
 msgstr "Add notifications related to Service object."
 
+msgid ""
+"Add start_time and end_time fields in audits table. User can set the start "
+"time and/or end time when creating CONTINUOUS audit."
+msgstr ""
+"Add start_time and end_time fields in audits table. User can set the start "
+"time and/or end time when creating CONTINUOUS audit."
+
 msgid ""
 "Add superseded state for an action plan if the cluster data model has "
 "changed after it has been created."
@@ -120,6 +136,19 @@ msgstr ""
 "hypervisor balanced, when the total VM workloads of hypervisor reaches "
 "threshold."
 
+msgid ""
+"Added a strategy for one compute node maintenance, without having the user's "
+"application been interrupted. If given one backup node, the strategy will "
+"firstly migrate all instances from the maintenance node to the backup node. "
+"If the backup node is not provided, it will migrate all instances, relying "
+"on nova-scheduler."
+msgstr ""
+"Added a strategy for one compute node maintenance, without having the user's "
+"application been interrupted. If given one backup node, the strategy will "
+"firstly migrate all instances from the maintenance node to the backup node. "
+"If the backup node is not provided, it will migrate all instances, relying "
+"on nova-scheduler."
+
 msgid ""
 "Added a strategy that monitors if there is a higher load on some hosts "
 "compared to other hosts in the cluster and re-balances the work across hosts "
@@ -194,6 +223,9 @@ msgstr ""
 "Added Gnocchi support as data source for metrics. Administrator can change "
 "data source for each strategy using config file."
 
+msgid "Added new tool ``watcher-status upgrade check``."
+msgstr "Added new tool ``watcher-status upgrade check``."
+
 msgid ""
 "Added notifications about cancelling of action plan. Now event based plugins "
 "know when action plan cancel started and completed."
@@ -268,6 +300,9 @@ msgstr ""
 "Audits have 'name' field now, that is more friendly to end users. Audit's "
 "name can't exceed 63 characters."
 
+msgid "Bug Fixes"
+msgstr "Bug Fixes"
+
 msgid "Centralize all configuration options for Watcher."
 msgstr "Centralise all configuration options for Watcher."
 
@@ -284,6 +319,9 @@ msgstr ""
 msgid "Current Series Release Notes"
 msgstr "Current Series Release Notes"
 
+msgid "Deprecation Notes"
+msgstr "Deprecation Notes"
+
 msgid ""
 "Each CDM collector can have its own CDM scoper now. This changed Scope JSON "
 "schema definition for the audit template POST data. Please see audit "
@@ -313,18 +351,60 @@ msgstr ""
 "feature improves the strategy. By the input parameter \"metrics\", it makes "
 "decision to migrate a VM base on CPU or memory utilisation."
 
+msgid ""
+"Feature to exclude instances from audit scope based on project_id is added. "
+"Now instances from particular project in OpenStack can be excluded from "
+"audit defining scope in audit templates."
+msgstr ""
+"Feature to exclude instances from audit scope based on project_id is added. "
+"Now instances from particular project in OpenStack can be excluded from "
+"audit defining scope in audit templates."
+
+msgid ""
+"Instance cold migration logic is now replaced with using Nova migrate "
+"Server(migrate Action) API which has host option since v2.56."
+msgstr ""
+"Instance cold migration logic is now replaced with using Nova migrate "
+"Server(migrate Action) API which has host option since v2.56."
+
 msgid "New Features"
 msgstr "New Features"
 
+msgid ""
+"New framework for ``watcher-status upgrade check`` command is added. This "
+"framework allows adding various checks which can be run before a Watcher "
+"upgrade to ensure if the upgrade can be performed safely."
+msgstr ""
+"New framework for ``watcher-status upgrade check`` command is added. This "
+"framework allows adding various checks which can be run before a Watcher "
+"upgrade to ensure if the upgrade can be performed safely."
+
 msgid "Newton Series Release Notes"
 msgstr "Newton Series Release Notes"
 
+msgid ""
+"Nova API version is now set to 2.56 by default. This needs the migrate "
+"action of migration type cold with destination_node parameter to work."
+msgstr ""
+"Nova API version is now set to 2.56 by default. This needs the migrate "
+"action of migration type cold with destination_node parameter to work."
+
 msgid "Ocata Series Release Notes"
 msgstr "Ocata Series Release Notes"
 
+msgid ""
+"Operator can now use new CLI tool ``watcher-status upgrade check`` to check "
+"if Watcher deployment can be safely upgraded from N-1 to N release."
+msgstr ""
+"Operator can now use new CLI tool ``watcher-status upgrade check`` to check "
+"if Watcher deployment can be safely upgraded from N-1 to N release."
+
 msgid "Pike Series Release Notes"
 msgstr "Pike Series Release Notes"
 
+msgid "Prelude"
+msgstr "Prelude"
+
 msgid ""
 "Provide a notification mechanism into Watcher that supports versioning. "
 "Whenever a Watcher object is created, updated or deleted, a versioned "
@@ -352,6 +432,9 @@ msgstr ""
 msgid "Queens Series Release Notes"
 msgstr "Queens Series Release Notes"
 
+msgid "Rocky Series Release Notes"
+msgstr "Rocky Series Release Notes"
+
 msgid ""
 "The graph model describes how VMs are associated to compute hosts. This "
 "allows for seeing relationships upfront between the entities and hence can "
@@ -363,6 +446,15 @@ msgstr ""
 "be used to identify hot/cold spots in the data centre and influence a "
 "strategy decision."
 
+msgid ""
+"The migrate action of migration type cold with destination_node parameter "
+"was fixed. Before fixing, it booted an instance in the service project as a "
+"migrated instance."
+msgstr ""
+"The migrate action of migration type cold with destination_node parameter "
+"was fixed. Before fixing, it booted an instance in the service project as a "
+"migrated instance."
+
 msgid ""
 "There is new ability to create Watcher continuous audits with cron interval. "
 "It means you may use, for example, optional argument '--interval \"\\*/5 \\* "
@@ -376,6 +468,9 @@ msgstr ""
 "best effort basis and therefore, we recommend you to use a minimal cron "
 "interval of at least one minute."
 
+msgid "Upgrade Notes"
+msgstr "Upgrade Notes"
+
 msgid ""
 "Watcher can continuously optimize the OpenStack cloud for a specific "
 "strategy or goal by triggering an audit periodically which generates an "
@@ -392,6 +487,13 @@ msgstr ""
 "Watcher can now run specific actions in parallel improving the performance "
 "dramatically when executing an action plan."
 
+msgid ""
+"Watcher consumes Nova notifications to update its internal Compute "
+"CDM(Cluster Data Model). All the notifications as below"
+msgstr ""
+"Watcher consumes Nova notifications to update its internal Compute "
+"CDM(Cluster Data Model). All the notifications as below"
+
 msgid "Watcher database can now be upgraded thanks to Alembic."
 msgstr "Watcher database can now be upgraded thanks to Alembic."
 
@@ -406,6 +508,48 @@ msgstr ""
 "resource types (like volumes, instances, network) if strategy supports "
 "efficacy indicators."
 
+msgid ""
+"Watcher has a whole scope of the cluster, when building compute CDM which "
+"includes all instances. It filters excluded instances when migration during "
+"the audit."
+msgstr ""
+"Watcher has a whole scope of the cluster, when building compute CDM which "
+"includes all instances. It filters excluded instances when migration during "
+"the audit."
+
+msgid ""
+"Watcher removes the support to Nova legacy notifications because of Nova "
+"will deprecate them."
+msgstr ""
+"Watcher removes the support to Nova legacy notifications because of Nova "
+"will deprecate them."
+
+msgid ""
+"Watcher services can be launched in HA mode. From now on Watcher Decision "
+"Engine and Watcher Applier services may be deployed on different nodes to "
+"run in active-active or active-passive mode. Any ONGOING Audits or Action "
+"Plans will be CANCELLED if service they are executed on is restarted."
+msgstr ""
+"Watcher services can be launched in HA mode. From now on Watcher Decision "
+"Engine and Watcher Applier services may be deployed on different nodes to "
+"run in active-active or active-passive mode. Any ONGOING Audits or Action "
+"Plans will be CANCELLED if service they are executed on is restarted."
+
+msgid ""
+"Watcher starts to support API microversions since Stein cycle. From now "
+"onwards all API changes should be made with saving backward compatibility. "
+"To specify API version operator should use OpenStack-API-Version HTTP "
+"header. If operator wants to know the mininum and maximum supported versions "
+"by API, he/she can access /v1 resource and Watcher API will return "
+"appropriate headers in response."
+msgstr ""
+"Watcher starts to support API microversions since the Stein cycle. From now "
+"onwards all API changes should be made with saving backward compatibility. "
+"To specify API version operator should use OpenStack-API-Version HTTP "
+"header. If operator wants to know the minimum and maximum supported versions "
+"by API, he/she can access /v1 resource and Watcher API will return "
+"appropriate headers in response."
+
 msgid ""
 "Watcher supports multiple metrics backend and relies on Ceilometer and "
 "Monasca."
@@ -424,3 +568,84 @@ msgstr ""
 "all Watcher objects have been refactored to support OVO (oslo."
 "versionedobjects) which was a prerequisite step in order to implement "
 "versioned notifications."
+
+msgid "instance.create.end"
+msgstr "instance.create.end"
+
+msgid "instance.delete.end"
+msgstr "instance.delete.end"
+
+msgid "instance.live_migration_force_complete.end"
+msgstr "instance.live_migration_force_complete.end"
+
+msgid "instance.live_migration_post_dest.end"
+msgstr "instance.live_migration_post_dest.end"
+
+msgid "instance.lock"
+msgstr "instance.lock"
+
+msgid "instance.pause.end"
+msgstr "instance.pause.end"
+
+msgid "instance.power_off.end"
+msgstr "instance.power_off.end"
+
+msgid "instance.power_on.end"
+msgstr "instance.power_on.end"
+
+msgid "instance.rebuild.end"
+msgstr "instance.rebuild.end"
+
+msgid "instance.rescue.end"
+msgstr "instance.rescue.end"
+
+msgid "instance.resize_confirm.end"
+msgstr "instance.resize_confirm.end"
+
+msgid "instance.restore.end"
+msgstr "instance.restore.end"
+
+msgid "instance.resume.end"
+msgstr "instance.resume.end"
+
+msgid "instance.shelve.end"
+msgstr "instance.shelve.end"
+
+msgid "instance.shutdown.end"
+msgstr "instance.shutdown.end"
+
+msgid "instance.soft_delete.end"
+msgstr "instance.soft_delete.end"
+
+msgid "instance.suspend.end"
+msgstr "instance.suspend.end"
+
+msgid "instance.unlock"
+msgstr "instance.unlock"
+
+msgid "instance.unpause.end"
+msgstr "instance.unpause.end"
+
+msgid "instance.unrescue.end"
+msgstr "instance.unrescue.end"
+
+msgid "instance.unshelve.end"
+msgstr "instance.unshelve.end"
+
+msgid "instance.update"
+msgstr "instance.update"
+
+msgid "new:"
+msgstr "new:"
+
+msgid "pre-existing:"
+msgstr "pre-existing:"
+
+msgid "service.create"
+msgstr "service.create"
+
+msgid "service.delete"
+msgstr "service.delete"
+
+msgid "service.update"
+msgstr "service.update"

releasenotes/source/rocky.rst

@@ -0,0 +1,6 @@
+===================================
+ Rocky Series Release Notes
+===================================
+
+.. release-notes::
+   :branch: stable/rocky

requirements.txt

@@ -22,6 +22,7 @@ oslo.policy>=1.34.0 # Apache-2.0
 oslo.reports>=1.27.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
 oslo.service>=1.30.0 # Apache-2.0
+oslo.upgradecheck>=0.1.0 # Apache-2.0
 oslo.utils>=3.36.0 # Apache-2.0
 oslo.versionedobjects>=1.32.0 # Apache-2.0
 PasteDeploy>=1.5.2 # MIT
@@ -44,5 +45,7 @@ stevedore>=1.28.0 # Apache-2.0
 taskflow>=3.1.0 # Apache-2.0
 WebOb>=1.7.4 # MIT
 WSME>=0.9.2 # MIT
-networkx>=1.11 # BSD
-
+# NOTE(fdegir): NetworkX 2.3 dropped support for Python 2
+networkx>=1.11,<2.3;python_version<'3.0'  # BSD
+networkx>=1.11;python_version>='3.4' # BSD
+microversion_parse>=0.2.1 # Apache-2.0

roles/add-hostnames-to-hosts/tasks/main.yaml

@@ -1,16 +0,0 @@
-- name: Set up the list of hostnames and addresses
-  set_fact:
-    hostname_addresses: >
-      {% set hosts = {} -%}
-      {% for host, vars in hostvars.items() -%}
-      {% set _ = hosts.update({vars['ansible_hostname']: vars['nodepool']['private_ipv4']}) -%}
-      {% endfor -%}
-      {{- hosts -}}
-- name: Add inventory hostnames to the hosts file
-  become: yes
-  lineinfile:
-    dest: /etc/hosts
-    state: present
-    insertafter: EOF
-    line: "{{ item.value }} {{ item.key }}"
-  with_dict: "{{ hostname_addresses }}"

setup.cfg

@@ -4,7 +4,7 @@ summary = OpenStack Watcher provides a flexible and scalable resource optimizati
 description-file =
     README.rst
 author = OpenStack
-author-email = openstack-dev@lists.openstack.org
+author-email = openstack-discuss@lists.openstack.org
 home-page = https://docs.openstack.org/watcher/latest/
 classifier =
     Environment :: OpenStack
@@ -44,6 +44,7 @@ console_scripts =
     watcher-decision-engine = watcher.cmd.decisionengine:main
     watcher-applier = watcher.cmd.applier:main
     watcher-sync = watcher.cmd.sync:main
+    watcher-status = watcher.cmd.status:main
 
 watcher.database.migration_backend =
     sqlalchemy = watcher.db.sqlalchemy.migration
@@ -105,26 +106,6 @@ watcher_cluster_data_model_collectors =
     baremetal = watcher.decision_engine.model.collector.ironic:BaremetalClusterDataModelCollector
 
 
-[pbr]
-autodoc_index_modules = true
-autodoc_exclude_modules =
-    watcher.db.sqlalchemy.alembic.env
-    watcher.db.sqlalchemy.alembic.versions.*
-    watcher.tests.*
-    watcher.doc
-
-
-[build_sphinx]
-source-dir = doc/source
-build-dir = doc/build
-fresh_env = 1
-all_files = 1
-warning-is-error = 1
-
-[upload_sphinx]
-upload-dir = doc/build/html
-
-
 [compile_catalog]
 directory = watcher/locale
 domain = watcher

test-requirements.txt

@@ -5,24 +5,12 @@
 coverage>=4.5.1 # Apache-2.0
 doc8>=0.8.0 # Apache-2.0
 freezegun>=0.3.10 # Apache-2.0
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
+hacking>=1.1.0,<1.2.0 # Apache-2.0
 mock>=2.0.0 # BSD
 oslotest>=3.3.0 # Apache-2.0
 os-testr>=1.0.0 # Apache-2.0
 testscenarios>=0.5.0 # Apache-2.0/BSD
 testtools>=2.3.0 # MIT
 stestr>=2.0.0 # Apache-2.0
-
-# Doc requirements
-openstackdocstheme>=1.20.0 # Apache-2.0
-sphinx>=1.6.5,!=1.6.6,!=1.6.7 # BSD
-sphinxcontrib-pecanwsme>=0.8.0 # Apache-2.0
-
-# api-ref
 os-api-ref>=1.4.0 # Apache-2.0
-
-# releasenotes
-reno>=2.7.0 # Apache-2.0
-
-# bandit
-bandit>=1.1.0 # Apache-2.0
+bandit>=1.6.0 # Apache-2.0

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-minversion = 1.8
+minversion = 2.0
 envlist = py35,py27,pep8
 skipsdist = True
 
@@ -7,7 +7,7 @@ skipsdist = True
 usedevelop = True
 whitelist_externals = find
                       rm
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/rocky} {opts} {packages}
+install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/stein} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
@@ -22,7 +22,7 @@ basepython = python3
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    bandit -r watcher -x tests -n5 -ll -s B320
+    bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:venv]
 basepython = python3
@@ -34,7 +34,7 @@ basepython = python3
 setenv =
     PYTHON=coverage run --source watcher --parallel-mode
 commands =
-    stestr run '{posargs}'
+    stestr run {posargs}
     coverage combine
     coverage html -d cover
     coverage xml -o cover/coverage.xml
@@ -43,13 +43,12 @@ commands =
 [testenv:docs]
 basepython = python3
 setenv = PYTHONHASHSEED=0
-commands =
-    doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
-    python setup.py build_sphinx
+deps = -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -W -b html doc/source doc/build/html
 
 [testenv:api-ref]
-# This environment is called from CI scripts to test and publish
-# the API Ref to developer.openstack.org.
+basepython = python3
+deps = -r{toxinidir}/doc/requirements.txt
 whitelist_externals = bash
 commands =
   bash -c 'rm -rf api-ref/build'
@@ -93,12 +92,13 @@ ignore-path=doc/source/image_src,doc/source/man,doc/source/api
 
 [testenv:releasenotes]
 basepython = python3
+deps = -r{toxinidir}/doc/requirements.txt
 commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
 
 [testenv:bandit]
 basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r watcher -x tests -n5 -ll -s B320
+commands = bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:lower-constraints]
 basepython = python3

watcher/api/app.py

@@ -21,7 +21,7 @@ import pecan
 
 from watcher.api import acl
 from watcher.api import config as api_config
-from watcher.api import middleware
+from watcher.api.middleware import parsable_error
 from watcher import conf
 
 CONF = conf.CONF
@@ -42,7 +42,7 @@ def setup_app(config=None):
         app_conf.pop('root'),
         logging=getattr(config, 'logging', {}),
         debug=CONF.debug,
-        wrap_app=middleware.ParsableErrorMiddleware,
+        wrap_app=parsable_error.ParsableErrorMiddleware,
         **app_conf
     )
 

watcher/api/controllers/base.py

@@ -14,7 +14,10 @@
 # limitations under the License.
 
 import datetime
+import functools
 
+import microversion_parse
+from webob import exc
 import wsme
 from wsme import types as wtypes
 
@@ -49,3 +52,84 @@ class APIBase(wtypes.Base):
         for k in self.as_dict():
             if k not in except_list:
                 setattr(self, k, wsme.Unset)
+
+
+@functools.total_ordering
+class Version(object):
+    """API Version object."""
+
+    string = 'OpenStack-API-Version'
+    """HTTP Header string carrying the requested version"""
+
+    min_string = 'OpenStack-API-Minimum-Version'
+    """HTTP response header"""
+
+    max_string = 'OpenStack-API-Maximum-Version'
+    """HTTP response header"""
+
+    def __init__(self, headers, default_version, latest_version):
+        """Create an API Version object from the supplied headers.
+
+        :param headers: webob headers
+        :param default_version: version to use if not specified in headers
+        :param latest_version: version to use if latest is requested
+        :raises: webob.HTTPNotAcceptable
+
+        """
+        (self.major, self.minor) = Version.parse_headers(
+            headers, default_version, latest_version)
+
+    def __repr__(self):
+        return '%s.%s' % (self.major, self.minor)
+
+    @staticmethod
+    def parse_headers(headers, default_version, latest_version):
+        """Determine the API version requested based on the headers supplied.
+
+        :param headers: webob headers
+        :param default_version: version to use if not specified in headers
+        :param latest_version: version to use if latest is requested
+        :returns: a tuple of (major, minor) version numbers
+        :raises: webob.HTTPNotAcceptable
+
+        """
+        version_str = microversion_parse.get_version(
+            headers,
+            service_type='infra-optim')
+
+        minimal_version = (1, 0)
+
+        if version_str is None:
+            # If requested header is wrong, Watcher answers with the minimal
+            # supported version.
+            return minimal_version
+
+        if version_str.lower() == 'latest':
+            parse_str = latest_version
+        else:
+            parse_str = version_str
+
+        try:
+            version = tuple(int(i) for i in parse_str.split('.'))
+        except ValueError:
+            version = minimal_version
+
+        # NOTE (alexchadin): Old python-watcherclient sends requests with
+        # value of version header is "1". It should be transformed to 1.0 as
+        # it was supposed to be.
+        if len(version) == 1 and version[0] == 1:
+            version = minimal_version
+
+        if len(version) != 2:
+            raise exc.HTTPNotAcceptable(
+                "Invalid value for %s header" % Version.string)
+        return version
+
+    def __gt__(self, other):
+        return (self.major, self.minor) > (other.major, other.minor)
+
+    def __eq__(self, other):
+        return (self.major, self.minor) == (other.major, other.minor)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)

watcher/api/controllers/rest_api_version_history.rst

@@ -0,0 +1,27 @@
+REST API Version History
+========================
+
+This documents the changes made to the REST API with every
+microversion change. The description for each version should be a
+verbose one which has enough information to be suitable for use in
+user documentation.
+
+1.0 (Initial version)
+-----------------------
+This is the initial version of the Watcher API which supports
+microversions.
+
+A user can specify a header in the API request::
+
+  OpenStack-API-Version: infra-optim <version>
+
+where ``<version>`` is any valid api version for this API.
+
+If no version is specified then the API will behave as if version 1.0
+was requested.
+
+1.1
+---
+Added the parameters ``start_time`` and ``end_time`` to
+create audit request. Supported for start and end time of continuous
+audits.

watcher/api/controllers/root.py

@@ -14,6 +14,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import importlib
+
 import pecan
 from pecan import rest
 from wsme import types as wtypes
@@ -24,19 +26,39 @@ from watcher.api.controllers import link
 from watcher.api.controllers import v1
 
 
+class APIStatus(object):
+    CURRENT = "CURRENT"
+    SUPPORTED = "SUPPORTED"
+    DEPRECATED = "DEPRECATED"
+    EXPERIMENTAL = "EXPERIMENTAL"
+
+
 class Version(base.APIBase):
     """An API version representation."""
 
     id = wtypes.text
     """The ID of the version, also acts as the release number"""
 
+    status = wtypes.text
+    """The state of this API version"""
+
+    max_version = wtypes.text
+    """The maximum version supported"""
+
+    min_version = wtypes.text
+    """The minimum version supported"""
+
     links = [link.Link]
     """A Link that point to a specific version of the API"""
 
     @staticmethod
-    def convert(id):
+    def convert(id, status=APIStatus.CURRENT):
+        v = importlib.import_module('watcher.api.controllers.%s.versions' % id)
         version = Version()
         version.id = id
+        version.status = status
+        version.max_version = v.max_version_string()
+        version.min_version = v.min_version_string()
         version.links = [link.Link.make_link('self', pecan.request.host_url,
                                              id, '', bookmark=True)]
         return version

watcher/api/controllers/v1/__init__.py

@@ -24,10 +24,12 @@ import datetime
 
 import pecan
 from pecan import rest
+from webob import exc
 import wsme
 from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
+from watcher.api.controllers import base
 from watcher.api.controllers import link
 from watcher.api.controllers.v1 import action
 from watcher.api.controllers.v1 import action_plan
@@ -37,6 +39,21 @@ from watcher.api.controllers.v1 import goal
 from watcher.api.controllers.v1 import scoring_engine
 from watcher.api.controllers.v1 import service
 from watcher.api.controllers.v1 import strategy
+from watcher.api.controllers.v1 import versions
+
+
+def min_version():
+    return base.Version(
+        {base.Version.string: ' '.join([versions.service_type_string(),
+                                        versions.min_version_string()])},
+        versions.min_version_string(), versions.max_version_string())
+
+
+def max_version():
+    return base.Version(
+        {base.Version.string: ' '.join([versions.service_type_string(),
+                                        versions.max_version_string()])},
+        versions.min_version_string(), versions.max_version_string())
 
 
 class APIBase(wtypes.Base):
@@ -193,5 +210,51 @@ class Controller(rest.RestController):
         #       the request object to make the links.
         return V1.convert()
 
+    def _check_version(self, version, headers=None):
+        if headers is None:
+            headers = {}
+        # ensure that major version in the URL matches the header
+        if version.major != versions.BASE_VERSION:
+            raise exc.HTTPNotAcceptable(
+                "Mutually exclusive versions requested. Version %(ver)s "
+                "requested but not supported by this service. The supported "
+                "version range is: [%(min)s, %(max)s]." %
+                {'ver': version, 'min': versions.min_version_string(),
+                 'max': versions.max_version_string()},
+                headers=headers)
+        # ensure the minor version is within the supported range
+        if version < min_version() or version > max_version():
+            raise exc.HTTPNotAcceptable(
+                "Version %(ver)s was requested but the minor version is not "
+                "supported by this service. The supported version range is: "
+                "[%(min)s, %(max)s]." %
+                {'ver': version, 'min': versions.min_version_string(),
+                 'max': versions.max_version_string()},
+                headers=headers)
+
+    @pecan.expose()
+    def _route(self, args, request=None):
+        v = base.Version(pecan.request.headers, versions.min_version_string(),
+                         versions.max_version_string())
+
+        # The Vary header is used as a hint to caching proxies and user agents
+        # that the response is also dependent on the OpenStack-API-Version and
+        # not just the body and query parameters. See RFC 7231 for details.
+        pecan.response.headers['Vary'] = base.Version.string
+
+        # Always set the min and max headers
+        pecan.response.headers[base.Version.min_string] = (
+            versions.min_version_string())
+        pecan.response.headers[base.Version.max_string] = (
+            versions.max_version_string())
+
+        # assert that requested version is supported
+        self._check_version(v, pecan.response.headers)
+        pecan.response.headers[base.Version.string] = (
+            ' '.join([versions.service_type_string(), str(v)]))
+        pecan.request.version = v
+
+        return super(Controller, self)._route(args, request)
+
 
 __all__ = ("Controller", )

watcher/api/controllers/v1/action.py

@@ -74,6 +74,16 @@ from watcher.common import policy
 from watcher import objects
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class ActionPatchType(types.JsonPatchType):
 
     @staticmethod
@@ -174,6 +184,8 @@ class Action(base.APIBase):
             description = ""
         setattr(action, 'description', description)
 
+        hide_fields_in_newer_versions(action)
+
         return cls._convert_with_links(action, pecan.request.host_url, expand)
 
     @classmethod

watcher/api/controllers/v1/action_plan.py

@@ -80,6 +80,16 @@ from watcher.objects import action_plan as ap_objects
 LOG = log.getLogger(__name__)
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class ActionPlanPatchType(types.JsonPatchType):
 
     @staticmethod
@@ -273,6 +283,7 @@ class ActionPlan(base.APIBase):
     @classmethod
     def convert_with_links(cls, rpc_action_plan, expand=True):
         action_plan = ActionPlan(**rpc_action_plan.as_dict())
+        hide_fields_in_newer_versions(action_plan)
         return cls._convert_with_links(action_plan, pecan.request.host_url,
                                        expand)
 

watcher/api/controllers/v1/audit.py

@@ -30,11 +30,13 @@ states, visit :ref:`the Audit State machine <audit_state_machine>`.
 """
 
 import datetime
+from dateutil import tz
 
 import pecan
 from pecan import rest
 import wsme
 from wsme import types as wtypes
+from wsme import utils as wutils
 import wsmeext.pecan as wsme_pecan
 
 from oslo_log import log
@@ -54,6 +56,25 @@ from watcher import objects
 LOG = log.getLogger(__name__)
 
 
+def _get_object_by_value(context, class_name, value):
+    if utils.is_uuid_like(value) or utils.is_int_like(value):
+        return class_name.get(context, value)
+    else:
+        return class_name.get_by_name(context, value)
+
+
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    if not api_utils.allow_start_end_audit_time():
+        obj.start_time = wsme.Unset
+        obj.end_time = wsme.Unset
+
+
 class AuditPostType(wtypes.Base):
 
     name = wtypes.wsattr(wtypes.text, mandatory=False)
@@ -79,6 +100,10 @@ class AuditPostType(wtypes.Base):
 
     hostname = wtypes.wsattr(wtypes.text, readonly=True, mandatory=False)
 
+    start_time = wsme.wsattr(datetime.datetime, mandatory=False)
+
+    end_time = wsme.wsattr(datetime.datetime, mandatory=False)
+
     def as_audit(self, context):
         audit_type_values = [val.value for val in objects.audit.AuditType]
         if self.audit_type not in audit_type_values:
@@ -93,6 +118,21 @@ class AuditPostType(wtypes.Base):
             raise exception.AuditIntervalNotSpecified(
                 audit_type=self.audit_type)
 
+        if self.audit_template_uuid and self.goal:
+            raise exception.Invalid('Either audit_template_uuid '
+                                    'or goal should be provided.')
+
+        if (self.audit_type == objects.audit.AuditType.ONESHOT.value and
+                (self.start_time not in (wtypes.Unset, None) or
+                    self.end_time not in (wtypes.Unset, None))):
+            raise exception.AuditStartEndTimeNotAllowed(
+                audit_type=self.audit_type)
+
+        if not api_utils.allow_start_end_audit_time():
+            for field in ('start_time', 'end_time'):
+                if getattr(self, field) not in (wsme.Unset, None):
+                    raise exception.NotAcceptable()
+
         # If audit_template_uuid was provided, we will provide any
         # variables not included in the request, but not override
         # those variables that were included.
@@ -123,7 +163,8 @@ class AuditPostType(wtypes.Base):
         # Note: If audit name was not provided, used a default name
         if not self.name:
             if self.strategy:
-                strategy = objects.Strategy.get(context, self.strategy)
+                strategy = _get_object_by_value(context, objects.Strategy,
+                                                self.strategy)
                 self.name = "%s-%s" % (strategy.name,
                                        datetime.datetime.utcnow().isoformat())
             elif self.audit_template_uuid:
@@ -132,7 +173,7 @@ class AuditPostType(wtypes.Base):
                 self.name = "%s-%s" % (audit_template.name,
                                        datetime.datetime.utcnow().isoformat())
             else:
-                goal = objects.Goal.get(context, self.goal)
+                goal = _get_object_by_value(context, objects.Goal, self.goal)
                 self.name = "%s-%s" % (goal.name,
                                        datetime.datetime.utcnow().isoformat())
         # No more than 63 characters
@@ -149,7 +190,9 @@ class AuditPostType(wtypes.Base):
             strategy_id=self.strategy,
             interval=self.interval,
             scope=self.scope,
-            auto_trigger=self.auto_trigger)
+            auto_trigger=self.auto_trigger,
+            start_time=self.start_time,
+            end_time=self.end_time)
 
 
 class AuditPatchType(types.JsonPatchType):
@@ -310,6 +353,12 @@ class Audit(base.APIBase):
     hostname = wsme.wsattr(wtypes.text, mandatory=False)
     """Hostname the audit is running on"""
 
+    start_time = wsme.wsattr(datetime.datetime, mandatory=False)
+    """The start time for continuous audit launch"""
+
+    end_time = wsme.wsattr(datetime.datetime, mandatory=False)
+    """The end time that stopping continuous audit"""
+
     def __init__(self, **kwargs):
         self.fields = []
         fields = list(objects.Audit.fields)
@@ -356,6 +405,7 @@ class Audit(base.APIBase):
     @classmethod
     def convert_with_links(cls, rpc_audit, expand=True):
         audit = Audit(**rpc_audit.as_dict())
+        hide_fields_in_newer_versions(audit)
         return cls._convert_with_links(audit, pecan.request.host_url, expand)
 
     @classmethod
@@ -370,7 +420,9 @@ class Audit(base.APIBase):
                      interval='7200',
                      scope=[],
                      auto_trigger=False,
-                     next_run_time=datetime.datetime.utcnow())
+                     next_run_time=datetime.datetime.utcnow(),
+                     start_time=datetime.datetime.utcnow(),
+                     end_time=datetime.datetime.utcnow())
 
         sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
         sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff'
@@ -471,7 +523,7 @@ class AuditsController(rest.RestController):
         return audits_collection
 
     @wsme_pecan.wsexpose(AuditCollection, types.uuid, int, wtypes.text,
-                         wtypes.text, wtypes.text, wtypes.text, int)
+                         wtypes.text, wtypes.text, wtypes.text)
     def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc',
                 goal=None, strategy=None):
         """Retrieve a list of audits.
@@ -572,6 +624,17 @@ class AuditsController(rest.RestController):
                                       'parameter spec in predefined strategy'))
 
         audit_dict = audit.as_dict()
+        # convert local time to UTC time
+        start_time_value = audit_dict.get('start_time')
+        end_time_value = audit_dict.get('end_time')
+        if start_time_value:
+            audit_dict['start_time'] = start_time_value.replace(
+                tzinfo=tz.tzlocal()).astimezone(
+                    tz.tzutc()).replace(tzinfo=None)
+        if end_time_value:
+            audit_dict['end_time'] = end_time_value.replace(
+                tzinfo=tz.tzlocal()).astimezone(
+                    tz.tzutc()).replace(tzinfo=None)
 
         new_audit = objects.Audit(context, **audit_dict)
         new_audit.create()
@@ -616,6 +679,16 @@ class AuditsController(rest.RestController):
                     reason=error_message % dict(
                         initial_state=initial_state, new_state=new_state))
 
+            patch_path = api_utils.get_patch_key(patch, 'path')
+            if patch_path in ('start_time', 'end_time'):
+                patch_value = api_utils.get_patch_value(patch, patch_path)
+                # convert string format to UTC time
+                new_patch_value = wutils.parse_isodatetime(
+                    patch_value).replace(
+                        tzinfo=tz.tzlocal()).astimezone(
+                            tz.tzutc()).replace(tzinfo=None)
+                api_utils.set_patch_value(patch, patch_path, new_patch_value)
+
             audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch))
         except api_utils.JSONPATCH_EXCEPTIONS as e:
             raise exception.PatchError(patch=patch, reason=e)

watcher/api/controllers/v1/audit_template.py

@@ -65,6 +65,16 @@ from watcher.decision_engine.loading import default as default_loading
 from watcher import objects
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class AuditTemplatePostType(wtypes.Base):
     _ctx = context_utils.make_context()
 
@@ -410,6 +420,7 @@ class AuditTemplate(base.APIBase):
     @classmethod
     def convert_with_links(cls, rpc_audit_template, expand=True):
         audit_template = AuditTemplate(**rpc_audit_template.as_dict())
+        hide_fields_in_newer_versions(audit_template)
         return cls._convert_with_links(audit_template, pecan.request.host_url,
                                        expand)
 

watcher/api/controllers/v1/goal.py

@@ -48,6 +48,16 @@ from watcher.common import policy
 from watcher import objects
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class Goal(base.APIBase):
     """API representation of a goal.
 
@@ -97,6 +107,7 @@ class Goal(base.APIBase):
     @classmethod
     def convert_with_links(cls, goal, expand=True):
         goal = Goal(**goal.as_dict())
+        hide_fields_in_newer_versions(goal)
         return cls._convert_with_links(goal, pecan.request.host_url, expand)
 
     @classmethod

watcher/api/controllers/v1/scoring_engine.py

@@ -43,6 +43,16 @@ from watcher.common import policy
 from watcher import objects
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class ScoringEngine(base.APIBase):
     """API representation of a scoring engine.
 
@@ -95,6 +105,7 @@ class ScoringEngine(base.APIBase):
     @classmethod
     def convert_with_links(cls, scoring_engine, expand=True):
         scoring_engine = ScoringEngine(**scoring_engine.as_dict())
+        hide_fields_in_newer_versions(scoring_engine)
         return cls._convert_with_links(
             scoring_engine, pecan.request.host_url, expand)
 

watcher/api/controllers/v1/service.py

@@ -44,6 +44,16 @@ CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class Service(base.APIBase):
     """API representation of a service.
 
@@ -59,8 +69,8 @@ class Service(base.APIBase):
 
     def _set_status(self, id):
         service = objects.Service.get(pecan.request.context, id)
-        last_heartbeat = (service.last_seen_up or service.updated_at
-                          or service.created_at)
+        last_heartbeat = (service.last_seen_up or service.updated_at or
+                          service.created_at)
         if isinstance(last_heartbeat, six.string_types):
             # NOTE(russellb) If this service came in over rpc via
             # conductor, then the timestamp will be a string and needs to be
@@ -126,6 +136,7 @@ class Service(base.APIBase):
     @classmethod
     def convert_with_links(cls, service, expand=True):
         service = Service(**service.as_dict())
+        hide_fields_in_newer_versions(service)
         return cls._convert_with_links(
             service, pecan.request.host_url, expand)
 

watcher/api/controllers/v1/strategy.py

@@ -45,6 +45,16 @@ from watcher.decision_engine import rpcapi
 from watcher import objects
 
 
+def hide_fields_in_newer_versions(obj):
+    """This method hides fields that were added in newer API versions.
+
+    Certain node fields were introduced at certain API versions.
+    These fields are only made available when the request's API version
+    matches or exceeds the versions when these fields were introduced.
+    """
+    pass
+
+
 class Strategy(base.APIBase):
     """API representation of a strategy.
 
@@ -146,6 +156,7 @@ class Strategy(base.APIBase):
     @classmethod
     def convert_with_links(cls, strategy, expand=True):
         strategy = Strategy(**strategy.as_dict())
+        hide_fields_in_newer_versions(strategy)
         return cls._convert_with_links(
             strategy, pecan.request.host_url, expand)
 

watcher/api/controllers/v1/utils.py

@@ -23,6 +23,7 @@ import pecan
 import wsme
 
 from watcher._i18n import _
+from watcher.api.controllers.v1 import versions
 from watcher.common import utils
 from watcher import objects
 
@@ -101,6 +102,18 @@ def get_patch_value(patch, key):
             return p['value']
 
 
+def set_patch_value(patch, key, value):
+    for p in patch:
+        if p['op'] == 'replace' and p['path'] == '/%s' % key:
+            p['value'] = value
+
+
+def get_patch_key(patch, key):
+    for p in patch:
+        if p['op'] == 'replace' and key in p.keys():
+            return p[key][1:]
+
+
 def check_audit_state_transition(patch, initial):
     is_transition_valid = True
     state_value = get_patch_value(patch, "state")
@@ -143,3 +156,12 @@ def get_resource(resource, resource_id, eager=False):
         return _get(pecan.request.context, resource_id, eager=eager)
 
     return _get(pecan.request.context, resource_id)
+
+
+def allow_start_end_audit_time():
+    """Check if we should support optional start/end attributes for Audit.
+
+    Version 1.1 of the API added support for start and end time of continuous
+    audits.
+    """
+    return pecan.request.version.minor >= versions.MINOR_1_START_END_TIMING

watcher/api/controllers/v1/versions.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2015 Intel Corporation
+# Copyright (c) 2018 SBCloud
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+# This is the version 1 API
+BASE_VERSION = 1
+
+# Here goes a short log of changes in every version.
+#
+# v1.0: corresponds to Rocky API
+# v1.1: Add start/end time for continuous audit
+
+MINOR_0_ROCKY = 0
+MINOR_1_START_END_TIMING = 1
+
+MINOR_MAX_VERSION = MINOR_1_START_END_TIMING
+
+# String representations of the minor and maximum versions
+_MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_0_ROCKY)
+_MAX_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_MAX_VERSION)
+
+
+def service_type_string():
+    return 'infra-optim'
+
+
+def min_version_string():
+    """Returns the minimum supported API version (as a string)"""
+    return _MIN_VERSION_STRING
+
+
+def max_version_string():
+    """Returns the maximum supported API version (as a string).
+
+    If the service is pinned, the maximum API version is the pinned
+    version. Otherwise, it is the maximum supported API version.
+
+    """
+    return _MAX_VERSION_STRING

watcher/api/middleware/__init__.py

@@ -1,25 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from watcher.api.middleware import auth_token
-from watcher.api.middleware import parsable_error
-
-
-ParsableErrorMiddleware = parsable_error.ParsableErrorMiddleware
-AuthTokenMiddleware = auth_token.AuthTokenMiddleware
-
-__all__ = (ParsableErrorMiddleware,
-           AuthTokenMiddleware)

watcher/api/scheduling.py

@@ -82,8 +82,8 @@ class APISchedulingService(scheduling.BackgroundSchedulerService):
 
     def get_service_status(self, context, service_id):
         service = objects.Service.get(context, service_id)
-        last_heartbeat = (service.last_seen_up or service.updated_at
-                          or service.created_at)
+        last_heartbeat = (service.last_seen_up or service.updated_at or
+                          service.created_at)
         if isinstance(last_heartbeat, six.string_types):
             # NOTE(russellb) If this service came in over rpc via
             # conductor, then the timestamp will be a string and needs to be

watcher/applier/workflow_engine/base.py

@@ -58,6 +58,7 @@ class BaseWorkFlowEngine(loadable.Loadable):
         self._action_factory = factory.ActionFactory()
         self._osc = None
         self._is_notified = False
+        self.execution_rule = None
 
     @classmethod
     def get_config_opts(cls):
@@ -191,7 +192,7 @@ class BaseTaskFlowActionContainer(flow_task.Task):
                     fields.NotificationPhase.END)
             except Exception as e:
                 LOG.exception(e)
-                LOG.error('The workflow engine has failed'
+                LOG.error('The workflow engine has failed '
                           'to execute the action: %s', self.name)
                 db_action = self.engine.notify(self._db_action,
                                                objects.action.State.FAILED)
@@ -206,11 +207,14 @@ class BaseTaskFlowActionContainer(flow_task.Task):
         et = eventlet.spawn(_do_execute_action, *args, **kwargs)
         # NOTE: check for the state of action plan periodically,so that if
         # action is finished or action plan is cancelled we can exit from here.
+        result = False
         while True:
             action_object = objects.Action.get_by_uuid(
                 self.engine.context, self._db_action.uuid, eager=True)
             action_plan_object = objects.ActionPlan.get_by_id(
                 self.engine.context, action_object.action_plan_id)
+            if action_object.state == objects.action.State.SUCCEEDED:
+                result = True
             if (action_object.state in [objects.action.State.SUCCEEDED,
                objects.action.State.FAILED] or
                action_plan_object.state in CANCEL_STATE):
@@ -226,6 +230,7 @@ class BaseTaskFlowActionContainer(flow_task.Task):
             if (action_plan_object.state in CANCEL_STATE and abort):
                 et.kill()
             et.wait()
+            return result
 
             # NOTE: catch the greenlet exit exception due to thread kill,
             # taskflow will call revert for the action,
@@ -236,7 +241,8 @@ class BaseTaskFlowActionContainer(flow_task.Task):
 
         except Exception as e:
             LOG.exception(e)
-            raise
+            # return False instead of raising an exception
+            return False
 
     def post_execute(self):
         try:

watcher/applier/workflow_engine/default.py

@@ -38,8 +38,6 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
     """
 
     def decider(self, history):
-        # FIXME(jed) not possible with the current Watcher Planner
-        #
         # decider – A callback function that will be expected to
         # decide at runtime whether v should be allowed to execute
         # (or whether the execution of v should be ignored,
@@ -48,7 +46,11 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
         # all u decidable links that have v as a target. It is expected
         # to return a single boolean
         # (True to allow v execution or False to not).
-        return True
+        LOG.info("decider history: %s", history)
+        if history and self.execution_rule == 'ANY':
+            return not list(history.values())[0]
+        else:
+            return True
 
     @classmethod
     def get_config_opts(cls):
@@ -59,9 +61,27 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
                 min=1,
                 required=True,
                 help='Number of workers for taskflow engine '
-                     'to execute actions.')
+                     'to execute actions.'),
+            cfg.DictOpt(
+                'action_execution_rule',
+                default={},
+                help='The execution rule for linked actions,'
+                     'the key is strategy name and '
+                     'value ALWAYS means all actions will be executed,'
+                     'value ANY means if previous action executes '
+                     'success, the next action will be ignored.'
+                     'None means ALWAYS.')
             ]
 
+    def get_execution_rule(self, actions):
+        if actions:
+            actionplan_object = objects.ActionPlan.get_by_id(
+                self.context, actions[0].action_plan_id)
+            strategy_object = objects.Strategy.get_by_id(
+                self.context, actionplan_object.strategy_id)
+            return self.config.action_execution_rule.get(
+                strategy_object.name)
+
     def execute(self, actions):
         try:
             # NOTE(jed) We want to have a strong separation of concern
@@ -72,6 +92,7 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
             # the users to change it.
             # The current implementation uses graph with linked actions.
             # todo(jed) add olso conf for retry and name
+            self.execution_rule = self.get_execution_rule(actions)
             flow = gf.Flow("watcher_flow")
             actions_uuid = {}
             for a in actions:

watcher/cmd/__init__.py

@@ -0,0 +1,20 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# NOTE(licanwei): Do eventlet monkey patching here, instead of in
+# common/service.py.  This allows the API service to run without monkey
+# patching under Apache (which uses its own concurrency model). Mixing
+# concurrency models can cause undefined behavior and potentially API timeouts.
+import eventlet
+
+eventlet.monkey_patch()

watcher/cmd/status.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2018 NEC, Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sys
+
+from oslo_upgradecheck import upgradecheck
+
+from watcher._i18n import _
+from watcher import conf
+
+CONF = conf.CONF
+
+
+class Checks(upgradecheck.UpgradeCommands):
+
+    """Contains upgrade checks
+
+    Various upgrade checks should be added as separate methods in this class
+    and added to _upgrade_checks tuple.
+    """
+
+    def _sample_check(self):
+        """This is sample check added to test the upgrade check framework
+
+        It needs to be removed after adding any real upgrade check
+        """
+        return upgradecheck.Result(upgradecheck.Code.SUCCESS, 'Sample detail')
+
+    _upgrade_checks = (
+        # Sample check added for now.
+        # Whereas in future real checks must be added here in tuple
+        (_('Sample Check'), _sample_check),
+    )
+
+
+def main():
+    return upgradecheck.main(
+        CONF, project='watcher', upgrade_command=Checks())
+
+
+if __name__ == '__main__':
+    sys.exit(main())

watcher/common/context.py

@@ -38,11 +38,11 @@ class RequestContext(context.RequestContext):
         tenant = kwargs.pop('tenant', None)
         super(RequestContext, self).__init__(
             auth_token=auth_token,
-            user=user_id or user,
-            tenant=project_id or tenant,
-            domain=kwargs.pop('domain', None) or domain_name or domain_id,
-            user_domain=kwargs.pop('user_domain', None),
-            project_domain=kwargs.pop('project_domain', None),
+            user_id=user_id or user,
+            project_id=project_id or tenant,
+            domain_id=kwargs.pop('domain', None) or domain_name or domain_id,
+            user_domain_id=kwargs.pop('user_domain', None),
+            project_domain_id=kwargs.pop('project_domain', None),
             is_admin=is_admin,
             read_only=kwargs.pop('read_only', False),
             show_deleted=kwargs.pop('show_deleted', False),
@@ -50,23 +50,15 @@ class RequestContext(context.RequestContext):
             resource_uuid=kwargs.pop('resource_uuid', None),
             is_admin_project=kwargs.pop('is_admin_project', True),
             overwrite=overwrite,
-            roles=roles)
+            roles=roles,
+            global_request_id=kwargs.pop('global_request_id', None),
+            system_scope=kwargs.pop('system_scope', None))
 
         self.remote_address = kwargs.pop('remote_address', None)
-        self.instance_lock_checked = kwargs.pop('instance_lock_checked', None)
         self.read_deleted = kwargs.pop('read_deleted', None)
         self.service_catalog = kwargs.pop('service_catalog', None)
         self.quota_class = kwargs.pop('quota_class', None)
 
-        # oslo_context's RequestContext.to_dict() generates this field, we can
-        # safely ignore this as we don't use it.
-        kwargs.pop('user_identity', None)
-        kwargs.pop('global_request_id', None)
-        kwargs.pop('project', None)
-        if kwargs:
-            LOG.warning('Arguments dropped when creating context: %s',
-                        str(kwargs))
-
         # FIXME(dims): user_id and project_id duplicate information that is
         # already present in the oslo_context's RequestContext. We need to
         # get rid of them.

watcher/common/exception.py

@@ -122,6 +122,11 @@ class NotAuthorized(WatcherException):
     code = 403
 
 
+class NotAcceptable(WatcherException):
+    msg_fmt = _("Request not acceptable.")
+    code = 406
+
+
 class PolicyNotAuthorized(NotAuthorized):
     msg_fmt = _("Policy doesn't allow %(action)s to be performed.")
 
@@ -260,6 +265,11 @@ class AuditIntervalNotAllowed(Invalid):
     msg_fmt = _("Interval of audit must not be set for %(audit_type)s.")
 
 
+class AuditStartEndTimeNotAllowed(Invalid):
+    msg_fmt = _("Start or End time of audit must not be set for "
+                "%(audit_type)s.")
+
+
 class AuditReferenced(Invalid):
     msg_fmt = _("Audit %(audit)s is referenced by one or multiple action "
                 "plans")
@@ -366,18 +376,6 @@ class KeystoneFailure(WatcherException):
     msg_fmt = _("Keystone API endpoint is missing")
 
 
-class ClusterEmpty(WatcherException):
-    msg_fmt = _("The list of compute node(s) in the cluster is empty")
-
-
-class ComputeClusterEmpty(WatcherException):
-    msg_fmt = _("The list of compute node(s) in the cluster is empty")
-
-
-class StorageClusterEmpty(WatcherException):
-    msg_fmt = _("The list of storage node(s) in the cluster is empty")
-
-
 class MetricCollectorNotDefined(WatcherException):
     msg_fmt = _("The metrics resource collector is not defined")
 

watcher/common/nova_helper.py

@@ -53,6 +53,9 @@ class NovaHelper(object):
         # We need to pass an object with an 'id' attribute to make it work
         return self.nova.hypervisors.get(utils.Struct(id=node_id))
 
+    def get_compute_node_by_name(self, node_name, servers=False):
+        return self.nova.hypervisors.search(node_name, servers)
+
     def get_compute_node_by_hostname(self, node_hostname):
         """Get compute node by hostname"""
         try:
@@ -73,9 +76,25 @@ class NovaHelper(object):
             LOG.exception(exc)
             raise exception.ComputeNodeNotFound(name=node_hostname)
 
-    def get_instance_list(self):
-        return self.nova.servers.list(search_opts={'all_tenants': True},
-                                      limit=-1)
+    def get_instance_list(self, filters=None, limit=-1):
+        """List servers for all tenants with details.
+
+        This always gets servers with the all_tenants=True filter.
+
+        :param filters: dict of additional filters (optional).
+        :param limit: Maximum number of servers to return (optional).
+                      If limit == -1, all servers will be returned,
+                      note that limit == -1 will have a performance
+                      penalty. For details, please see:
+                      https://bugs.launchpad.net/watcher/+bug/1834679
+        :returns: list of novaclient Server objects
+        """
+        search_opts = {'all_tenants': True}
+        if filters:
+            search_opts.update(filters)
+        # TODO(chenker) Add marker param to list Server objects.
+        return self.nova.servers.list(search_opts=search_opts,
+                                      limit=limit)
 
     def get_flavor_list(self):
         return self.nova.flavors.list(**{'is_public': None})
@@ -218,7 +237,7 @@ class NovaHelper(object):
         flavor_id = None
 
         try:
-            flavor_id = self.nova.flavors.get(flavor)
+            flavor_id = self.nova.flavors.get(flavor).id
         except nvexceptions.NotFound:
             flavor_id = [f.id for f in self.nova.flavors.list() if
                          f.name == flavor][0]
@@ -406,38 +425,6 @@ class NovaHelper(object):
 
         return status
 
-    def set_host_offline(self, hostname):
-        # See API on https://developer.openstack.org/api-ref/compute/
-        # especially the PUT request
-        # regarding this resource : /v2.1/os-hosts/​{host_name}​
-        #
-        # The following body should be sent :
-        # {
-        # "host": {
-        # "host": "65c5d5b7e3bd44308e67fc50f362aee6",
-        # "maintenance_mode": "off_maintenance",
-        # "status": "enabled"
-        # }
-        # }
-
-        # Voir ici
-        # https://github.com/openstack/nova/
-        # blob/master/nova/virt/xenapi/host.py
-        # set_host_enabled(self, enabled):
-        # Sets the compute host's ability to accept new instances.
-        # host_maintenance_mode(self, host, mode):
-        # Start/Stop host maintenance window.
-        # On start, it triggers guest instances evacuation.
-        host = self.nova.hosts.get(hostname)
-
-        if not host:
-            LOG.debug("host not found: %s", hostname)
-            return False
-        else:
-            host[0].update(
-                {"maintenance_mode": "disable", "status": "disable"})
-            return True
-
     def create_image_from_instance(self, instance_id, image_name,
                                    metadata={"reason": "instance_migrate"}):
         """This method creates a new image from a given instance.

watcher/common/observable.py

@@ -1,57 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from watcher.common import synchronization
-
-
-class Observable(synchronization.Synchronization):
-    def __init__(self):
-        super(Observable, self).__init__()
-        self.__observers = []
-        self.changed = 0
-
-    def set_changed(self):
-        self.changed = 1
-
-    def clear_changed(self):
-        self.changed = 0
-
-    def has_changed(self):
-        return self.changed
-
-    def register_observer(self, observer):
-        if observer not in self.__observers:
-            self.__observers.append(observer)
-
-    def unregister_observer(self, observer):
-        try:
-            self.__observers.remove(observer)
-        except ValueError:
-            pass
-
-    def notify(self, ctx=None, publisherid=None, event_type=None,
-               metadata=None, payload=None, modifier=None):
-        self.mutex.acquire()
-        try:
-            if not self.changed:
-                return
-            for observer in self.__observers:
-                if modifier != observer:
-                    observer.update(self, ctx, metadata, publisherid,
-                                    event_type, payload)
-            self.clear_changed()
-        finally:
-            self.mutex.release()

watcher/common/policies/goal.py

@@ -24,7 +24,7 @@ rules = [
         operations=[
             {
                 'path': '/v1/goals/detail',
-                'method': 'DELETE'
+                'method': 'GET'
             }
         ]
     ),

watcher/common/service.py

@@ -17,7 +17,6 @@
 import datetime
 import socket
 
-import eventlet
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import _options
@@ -42,12 +41,6 @@ from watcher.objects import base
 from watcher.objects import fields as wfields
 from watcher import version
 
-# NOTE:
-# Ubuntu 14.04 forces librabbitmq when kombu is used
-# Unfortunately it forces a version that has a crash
-# bug.  Calling eventlet.monkey_patch() tells kombu
-# to use libamqp instead.
-eventlet.monkey_patch()
 
 NOTIFICATION_OPTS = [
     cfg.StrOpt('notification_level',
@@ -68,7 +61,7 @@ _DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'qpid.messaging=INFO',
                        'oslo.messaging=INFO', 'sqlalchemy=WARN',
                        'keystoneclient=INFO', 'stevedore=INFO',
                        'eventlet.wsgi.server=WARN', 'iso8601=WARN',
-                       'paramiko=WARN', 'requests=WARN', 'neutronclient=WARN',
+                       'requests=WARN', 'neutronclient=WARN',
                        'glanceclient=WARN', 'watcher.openstack.common=WARN',
                        'apscheduler=WARN']
 
@@ -274,7 +267,7 @@ class Service(service.ServiceBase):
             self.heartbeat.start()
 
     def stop(self):
-        LOG.debug("Disconnecting from '%s' (%s)'", CONF.transport_url)
+        LOG.debug("Disconnecting from '%s'", CONF.transport_url)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.stop()
         if self.notification_handler:

watcher/common/synchronization.py

@@ -1,22 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import threading
-
-
-class Synchronization(object):
-    def __init__(self):
-        self.mutex = threading.RLock()

watcher/conf/applier.py

@@ -19,7 +19,7 @@
 from oslo_config import cfg
 
 watcher_applier = cfg.OptGroup(name='watcher_applier',
-                               title='Options for the Applier messaging'
+                               title='Options for the Applier messaging '
                                'core')
 
 APPLIER_MANAGER_OPTS = [
@@ -30,7 +30,7 @@ APPLIER_MANAGER_OPTS = [
                help='Number of workers for applier, default value is 1.'),
     cfg.StrOpt('conductor_topic',
                default='watcher.applier.control',
-               help='The topic name used for'
+               help='The topic name used for '
                     'control events, this topic '
                     'used for rpc call '),
     cfg.StrOpt('publisher_id',

watcher/conf/ceilometer_client.py

@@ -24,14 +24,32 @@ ceilometer_client = cfg.OptGroup(name='ceilometer_client',
 CEILOMETER_CLIENT_OPTS = [
     cfg.StrOpt('api_version',
                default='2',
+               deprecated_for_removal=True,
+               deprecated_since="1.13.0",
+               deprecated_reason="""
+               Ceilometer API is deprecated since Ocata release.
+               Any related configuration options are deprecated too.
+               """,
                help='Version of Ceilometer API to use in '
                     'ceilometerclient.'),
     cfg.StrOpt('endpoint_type',
                default='internalURL',
-               help='Type of endpoint to use in ceilometerclient.'
-                    'Supported values: internalURL, publicURL, adminURL'
+               deprecated_for_removal=True,
+               deprecated_since="1.13.0",
+               deprecated_reason="""
+               Ceilometer API is deprecated since Ocata release.
+               Any related configuration options are deprecated too.
+               """,
+               help='Type of endpoint to use in ceilometerclient. '
+                    'Supported values: internalURL, publicURL, adminURL. '
                     'The default is internalURL.'),
     cfg.StrOpt('region_name',
+               deprecated_for_removal=True,
+               deprecated_since="1.13.0",
+               deprecated_reason="""
+               Ceilometer API is deprecated since Ocata release.
+               Any related configuration options are deprecated too.
+               """,
                help='Region in Identity service catalog to use for '
                     'communication with the OpenStack service.')]
 

watcher/conf/cinder_client.py

@@ -27,8 +27,8 @@ CINDER_CLIENT_OPTS = [
                help='Version of Cinder API to use in cinderclient.'),
     cfg.StrOpt('endpoint_type',
                default='publicURL',
-               help='Type of endpoint to use in cinderclient.'
-                    'Supported values: internalURL, publicURL, adminURL'
+               help='Type of endpoint to use in cinderclient. '
+                    'Supported values: internalURL, publicURL, adminURL. '
                     'The default is publicURL.'),
     cfg.StrOpt('region_name',
                help='Region in Identity service catalog to use for '