Merge "Update .gitreview for stable/wallaby" into stable/wallaby

Update the URL to the upper-constraints file to point to the redirect
rule on releases.openstack.org so that anyone working on this branch
will switch to the correct upper-constraints list automatically when
the requirements repository branches.

Until the requirements repository has as stable/wallaby branch, tests will
continue to use the upper-constraints list on master.

Change-Id: I5986087281694580acc3c19feed461b5949f7001

.gitreview

@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
-defaultbranch=stable/train
+defaultbranch=stable/wallaby

.zuul.yaml

@@ -2,9 +2,7 @@
     templates:
       - check-requirements
       - openstack-cover-jobs
-      - openstack-lower-constraints-jobs
-      - openstack-python-jobs
-      - openstack-python3-train-jobs
+      - openstack-python3-wallaby-jobs
       - publish-openstack-docs-pti
       - release-notes-jobs-python3
     check:
@@ -103,7 +101,7 @@
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
-    nodeset: openstack-two-node-bionic
+    nodeset: openstack-two-node-focal
     roles:
       - zuul: openstack/tempest
     group-vars:
@@ -161,7 +159,6 @@
     timeout: 7200
     required-projects: &base_required_projects
       - openstack/ceilometer
-      - openstack/devstack-gate
       - openstack/python-openstackclient
       - openstack/python-watcherclient
       - openstack/watcher
@@ -180,13 +177,10 @@
         s-container: false
         s-object: false
         s-proxy: false
-      devstack_localrc:
-        TEMPEST_PLUGINS: /opt/stack/watcher-tempest-plugin
+      tempest_plugins:
+        - watcher-tempest-plugin
       tempest_test_regex: watcher_tempest_plugin.tests.api
       tox_envlist: all
-      tox_environment:
-        # Do we really need to set this? It's cargo culted
-        PYTHONUNBUFFERED: 'true'
       zuul_copy_output:
         /etc/hosts: logs
 
@@ -200,10 +194,12 @@
 
 - job:
     name: watcher-grenade
-    parent: legacy-dsvm-base
-    timeout: 10800
-    run: playbooks/legacy/grenade-devstack-watcher/run.yaml
-    post-run: playbooks/legacy/grenade-devstack-watcher/post.yaml
+    parent: grenade
+    required-projects:
+      - openstack/watcher
+      - openstack/python-watcherclient
+      - openstack/watcher-tempest-plugin
+    vars: *base_vars
     irrelevant-files:
       - ^(test-|)requirements.txt$
       - ^.*\.rst$
@@ -215,12 +211,6 @@
       - ^setup.cfg$
       - ^tools/.*$
       - ^tox.ini$
-    required-projects:
-      - openstack/grenade
-      - openstack/devstack-gate
-      - openstack/watcher
-      - openstack/python-watcherclient
-      - openstack/watcher-tempest-plugin
 
 - job:
     # This job is used in python-watcherclient repo

README.rst

@@ -1,6 +1,6 @@
-========================
-Team and repository tags
-========================
+=======
+Watcher
+=======
 
 .. image:: https://governance.openstack.org/tc/badges/watcher.svg
     :target: https://governance.openstack.org/tc/reference/tags/index.html
@@ -13,10 +13,6 @@ Team and repository tags
 
           https://creativecommons.org/licenses/by/3.0/
 
-=======
-Watcher
-=======
-
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
 Watcher provides a robust framework to realize a wide range of cloud

api-ref/source/conf.py

@@ -22,9 +22,6 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-from watcher import version as watcher_version
-
-
 extensions = [
     'openstackdocstheme',
     'os_api_ref',
@@ -46,21 +43,13 @@ project = u'Infrastructure Optimization API Reference'
 copyright = u'2010-present, OpenStack Foundation'
 
 # openstackdocstheme options
-repository_name = 'openstack/watcher'
-bug_project = 'watcher'
-bug_tag = ''
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The full version, including alpha/beta/rc tags.
-release = watcher_version.version_info.release_string()
-# The short X.Y version.
-version = watcher_version.version_string
+openstackdocs_repo_name = 'openstack/watcher'
+openstackdocs_auto_name = False
+openstackdocs_bug_project = 'watcher'
+openstackdocs_bug_tag = ''
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # -- Options for HTML output --------------------------------------------------
 
@@ -75,10 +64,6 @@ html_theme_options = {
     "sidebar_mode": "toc",
 }
 
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%Y-%m-%d %H:%M'
-
 # -- Options for LaTeX output -------------------------------------------------
 
 # Grouping the document tree into LaTeX files. List of tuples

api-ref/source/index.rst

@@ -16,3 +16,4 @@ Watcher API
 .. include:: watcher-api-v1-services.inc
 .. include:: watcher-api-v1-scoring_engines.inc
 .. include:: watcher-api-v1-datamodel.inc
+.. include:: watcher-api-v1-webhooks.inc

api-ref/source/watcher-api-v1-datamodel.inc

@@ -4,6 +4,8 @@
 Data Model
 ==========
 
+.. versionadded:: 1.3
+
 ``Data Model`` is very important for Watcher to generate resource
 optimization solutions. Users can easily view the data model by the
 API.
@@ -18,7 +20,7 @@ Returns the information about Data Model.
 
 Normal response codes: 200
 
-Error codes: 400,401
+Error codes: 400,401,406
 
 Request
 -------

api-ref/source/watcher-api-v1-webhooks.inc

@@ -0,0 +1,26 @@
+.. -*- rst -*-
+
+========
+Webhooks
+========
+
+.. versionadded:: 1.4
+
+Triggers an event based Audit.
+
+
+Trigger EVENT Audit
+===================
+
+.. rest_method::  POST /v1/webhooks/{audit_ident}
+
+Normal response codes: 202
+
+Error codes: 400,404
+
+Request
+-------
+
+.. rest_parameters:: parameters.yaml
+
+   - audit_ident: audit_ident

babel.cfg

@@ -1,2 +0,0 @@
-[python: **.py]
-

devstack/lib/watcher

@@ -298,7 +298,7 @@ function start_watcher_api {
         service_protocol="http"
     fi
     if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        run_process "watcher-api" "$WATCHER_BIN_DIR/uwsgi --ini $WATCHER_UWSGI_CONF"
+        run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
         watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
     else
         watcher_url=$service_protocol://$SERVICE_HOST:$service_port

doc/ext/term.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import unicode_literals
-
 import importlib
 import inspect
 

doc/requirements.txt

@@ -1,11 +1,10 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
-openstackdocstheme>=1.20.0 # Apache-2.0
-sphinx>=1.6.5,!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7' # BSD
-sphinx>=1.6.5,!=1.6.6,!=1.6.7,!=2.1.0;python_version>='3.4' # BSD
+openstackdocstheme>=2.2.1 # Apache-2.0
+sphinx>=2.0.0,!=2.1.0 # BSD
 sphinxcontrib-pecanwsme>=0.8.0 # Apache-2.0
 sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD
-reno>=2.7.0 # Apache-2.0
+reno>=3.1.0 # Apache-2.0
 sphinxcontrib-apidoc>=0.2.0  # BSD
 os-api-ref>=1.4.0 # Apache-2.0

doc/source/admin/index.rst

@@ -8,6 +8,7 @@ Administrator Guide
    apache-mod-wsgi
    gmr
    policy
-   ways-to-install
    ../strategies/index
    ../datasources/index
+   ../contributor/notifications
+   ../contributor/concurrency

doc/source/admin/policy.rst

@@ -17,6 +17,14 @@
 Policies
 ========
 
+.. warning::
+
+   JSON formatted policy file is deprecated since Watcher 6.0.0 (Wallaby).
+   This `oslopolicy-convert-json-to-yaml`__ tool will migrate your existing
+   JSON-formatted policy file to YAML in a backward-compatible way.
+
+.. __: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html
+
 Watcher's public API calls may be restricted to certain sets of users using a
 policy configuration file. This document explains exactly how policies are
 configured and what they apply to.

doc/source/architecture.rst

@@ -281,11 +281,13 @@ previously created :ref:`Audit template <audit_template_definition>`:
    :width: 100%
 
 The :ref:`Administrator <administrator_definition>` also can specify type of
-Audit and interval (in case of CONTINUOUS type). There is two types of Audit:
-ONESHOT and CONTINUOUS. Oneshot Audit is launched once and if it succeeded
-executed new action plan list will be provided. Continuous Audit creates
-action plans with specified interval (in seconds); if action plan
-has been created, all previous action plans get CANCELLED state.
+Audit and interval (in case of CONTINUOUS type). There is three types of Audit:
+ONESHOT, CONTINUOUS and EVENT. ONESHOT Audit is launched once and if it
+succeeded executed new action plan list will be provided; CONTINUOUS Audit
+creates action plans with specified interval (in seconds or cron format, cron
+inteval can be used like: `*/5 * * * *`), if action plan
+has been created, all previous action plans get CANCELLED state;
+EVENT audit is launched when receiving webhooks API.
 
 A message is sent on the :ref:`AMQP bus <amqp_bus_definition>` which triggers
 the Audit in the

doc/source/conf.py

@@ -14,7 +14,6 @@
 import os
 import sys
 
-from watcher import version as watcher_version
 from watcher import objects
 
 objects.register_all()
@@ -36,7 +35,6 @@ extensions = [
     'sphinxcontrib.httpdomain',
     'sphinxcontrib.pecanwsme.rest',
     'stevedore.sphinxext',
-    'wsmeext.sphinxext',
     'ext.term',
     'ext.versioned_notifications',
     'oslo_config.sphinxconfiggen',
@@ -61,16 +59,6 @@ master_doc = 'index'
 project = u'Watcher'
 copyright = u'OpenStack Foundation'
 
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-# The full version, including alpha/beta/rc tags.
-release = watcher_version.version_info.release_string()
-# The short X.Y version.
-version = watcher_version.version_string
-
 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['watcher.']
 
@@ -95,7 +83,7 @@ add_module_names = True
 suppress_warnings = ['app.add_directive']
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # -- Options for man page output --------------------------------------------
 
@@ -126,12 +114,13 @@ html_theme = 'openstackdocs'
 # Output file base name for HTML help builder.
 htmlhelp_basename = '%sdoc' % project
 
-html_last_updated_fmt = '%Y-%m-%d %H:%M'
 
 #openstackdocstheme options
-repository_name = 'openstack/watcher'
-bug_project = 'watcher'
-bug_tag = ''
+openstackdocs_repo_name = 'openstack/watcher'
+openstackdocs_pdf_link = True
+openstackdocs_auto_name = False
+openstackdocs_bug_project = 'watcher'
+openstackdocs_bug_tag = ''
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass
@@ -139,7 +128,7 @@ bug_tag = ''
 latex_documents = [
     ('index',
      'doc-watcher.tex',
-     u'%s Documentation' % project,
+     u'Watcher Documentation',
      u'OpenStack Foundation', 'manual'),
 ]
 

doc/source/contributor/concurrency.rst

@@ -0,0 +1,248 @@
+===========
+Concurrency
+===========
+
+Introduction
+************
+
+Modern processors typically contain multiple cores all capable of executing
+instructions in parallel. Ensuring applications can fully utilize modern
+underlying hardware requires developing with these concepts in mind. The
+OpenStack foundation maintains a number of libraries to facilitate this
+utilization, combined with constructs like CPython's GIL_ the proper use of
+these concepts becomes more straightforward compared to other programming
+languages.
+
+The primary libraries maintained by OpenStack to facilitate concurrency are
+futurist_ and taskflow_. Here futurist is a more straightforward and
+lightweight library while taskflow is more advanced supporting features like
+rollback mechanisms. Within Watcher both libraries are used to facilitate
+concurrency.
+
+.. _GIL: https://wiki.python.org/moin/GlobalInterpreterLock
+.. _futurist: https://docs.openstack.org/futurist/latest/
+.. _taskflow: https://docs.openstack.org/taskflow/latest/
+
+Threadpool
+**********
+
+A threadpool is a collection of one or more threads typically called *workers*
+to which tasks can be submitted. These submitted tasks will be scheduled by a
+threadpool and subsequently executed. In the case of Python tasks typically are
+bounded or unbounded methods while other programming languages like Java
+require implementing an interface.
+
+The order and amount of concurrency with which these tasks are executed is up
+to the threadpool to decide. Some libraries like taskflow allow for either
+strong or loose ordering of tasks while others like futurist might only support
+loose ordering. Taskflow supports building tree-based hierarchies of dependent
+tasks for example.
+
+Upon submission of a task to a threadpool a so called future_ is returned.
+These objects allow to determine information about the task such as if it is
+currently being executed or if it has finished execution. When the task has
+finished execution the future can also be used to retrieve what was returned by
+the method.
+
+Some libraries like futurist provide synchronization primitives for collections
+of futures such as wait_for_any_. The following sections will cover different
+types of concurrency used in various services of Watcher.
+
+.. _future: https://docs.python.org/3/library/concurrent.futures.html
+.. _wait_for_any: https://docs.openstack.org/futurist/latest/reference/index.html#waiters
+
+
+Decision engine concurrency
+***************************
+
+The concurrency in the decision engine is governed by two independent
+threadpools. Both of these threadpools are GreenThreadPoolExecutor_ from the
+futurist_ library. One of these is used automatically and most contributors
+will not interact with it while developing new features. The other threadpool
+can frequently be used while developing new features or updating existing ones.
+It is known as the DecisionEngineThreadpool and allows to achieve performance
+improvements in network or I/O bound operations.
+
+.. _GreenThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#executors
+
+AuditEndpoint
+#############
+
+The first threadpool is used to allow multiple audits to be run in parallel.
+In practice, however, only one audit can be run in parallel. This is due to
+the data model used by audits being a singleton. To prevent audits destroying
+each others data model one must wait for the other to complete before being
+allowed to access this data model. A performance improvement could be achieved
+by being more intelligent in the use, caching and construction of these
+data models.
+
+DecisionEngineThreadPool
+########################
+
+The second threadpool is used for generic tasks, typically networking and I/O
+could benefit the most of this threadpool. Upon execution of an audit this
+threadpool can be utilized to retrieve information from the Nova compute
+service for instance. This second threadpool is a singleton and is shared
+amongst concurrently running audits as a result the amount of workers is static
+and independent from the amount of workers in the first threadpool. The use of
+the :class:`~.DecisionEngineThreadpool` while building the Nova compute data
+model is demonstrated to show how it can effectively be used.
+
+In the following example a reference to the
+:class:`~.DecisionEngineThreadpool` is stored in ``self.executor``. Here two
+tasks are submitted one with function ``self._collect_aggregates`` and the
+other function ``self._collect_zones``. With both ``self.executor.submit``
+calls subsequent arguments are passed to the function. All subsequent arguments
+are passed to the function being submitted as task following the common
+``(fn, *args, **kwargs)`` signature. One of the original signatures would be
+``def _collect_aggregates(host_aggregates, compute_nodes)`` for example.
+
+.. code-block:: python
+
+    zone_aggregate_futures = {
+        self.executor.submit(
+            self._collect_aggregates, host_aggregates, compute_nodes),
+        self.executor.submit(
+            self._collect_zones, availability_zones, compute_nodes)
+    }
+    waiters.wait_for_all(zone_aggregate_futures)
+
+The last statement of the example above waits on all futures to complete.
+Similarly, ``waiters.wait_for_any`` will wait for any future of the specified
+collection to complete. To simplify the usage of ``wait_for_any`` the
+:class:`~.DecisiongEngineThreadpool` defines a ``do_while_futures`` method.
+This method will iterate in a do_while loop over a collection of futures until
+all of them have completed. The advantage of ``do_while_futures`` is that it
+allows to immediately call a method as soon as a future finishes. The arguments
+for this callback method can be supplied when calling ``do_while_futures``,
+however, the first argument to the callback is always the future itself! If
+the collection of futures can safely be modified ``do_while_futures_modify``
+can be used and should have slightly better performance. The following example
+will show how ``do_while_futures`` is used in the decision engine.
+
+.. code-block:: python
+
+    # For every compute node from compute_nodes submit a task to gather the node it's information.
+    # List comprehension is used to store all the futures of the submitted tasks in node_futures.
+    node_futures = [self.executor.submit(
+        self.nova_helper.get_compute_node_by_name,
+        node, servers=True, detailed=True)
+        for node in compute_nodes]
+    LOG.debug("submitted {0} jobs".format(len(compute_nodes)))
+
+    future_instances = []
+    # do_while iterate over node_futures and upon completion of a future call
+    # self._compute_node_future with the future and future_instances as arguments.
+    self.executor.do_while_futures_modify(
+        node_futures, self._compute_node_future, future_instances)
+
+    # Wait for all instance jobs to finish
+    waiters.wait_for_all(future_instances)
+
+Finally, let's demonstrate how powerful this ``do_while_futures`` can be by
+showing what the ``compute_node_future`` callback does. First, it retrieves the
+result from the future and adds the compute node to the data model. Afterwards,
+it checks if the compute node has any associated instances and if so it submits
+an additional task to the :class:`~.DecisionEngineThreadpool`. The future is
+appended to the ``future_instances`` so ``waiters.wait_for_all`` can be called
+on this list. This is important as otherwise the building of the data model
+might return before all tasks for instances have finished.
+
+.. code-block:: python
+
+    # Get the result from the future.
+    node_info = future.result()[0]
+
+    # Filter out baremetal nodes.
+    if node_info.hypervisor_type == 'ironic':
+        LOG.debug("filtering out baremetal node: %s", node_info)
+        return
+
+    # Add the compute node to the data model.
+    self.add_compute_node(node_info)
+    # Get the instances from the compute node.
+    instances = getattr(node_info, "servers", None)
+    # Do not submit job if there are no instances on compute node.
+    if instances is None:
+        LOG.info("No instances on compute_node: {0}".format(node_info))
+        return
+    # Submit a job to retrieve detailed information about the instances.
+    future_instances.append(
+        self.executor.submit(
+            self.add_instance_node, node_info, instances)
+    )
+
+Without ``do_while_futures`` an additional ``waiters.wait_for_all`` would be
+required in between the compute node tasks and the instance tasks. This would
+cause the progress of the decision engine to stall as less and less tasks
+remain active before the instance tasks could be submitted. This demonstrates
+how ``do_while_futures`` can be used to achieve more constant utilization of
+the underlying hardware.
+
+Applier concurrency
+*******************
+
+The applier does not use the futurist_ GreenThreadPoolExecutor_ directly but
+instead uses taskflow_. However, taskflow still utilizes a greenthreadpool.
+This threadpool is initialized in the workflow engine called
+:class:`~.DefaultWorkFlowEngine`. Currently Watcher supports one workflow
+engine but the base class allows contributors to develop other workflow engines
+as well. In taskflow tasks are created using different types of flows such as a
+linear, unordered or a graph flow. The linear and graph flow allow for strong
+ordering between individual tasks and it is for this reason that the workflow
+engine utilizes a graph flow. The creation of tasks, subsequently linking them
+into a graph like structure and submitting them is shown below.
+
+.. code-block:: python
+
+    self.execution_rule = self.get_execution_rule(actions)
+    flow = gf.Flow("watcher_flow")
+    actions_uuid = {}
+    for a in actions:
+        task = TaskFlowActionContainer(a, self)
+        flow.add(task)
+        actions_uuid[a.uuid] = task
+
+    for a in actions:
+        for parent_id in a.parents:
+            flow.link(actions_uuid[parent_id], actions_uuid[a.uuid],
+                      decider=self.decider)
+
+    e = engines.load(
+        flow, executor='greenthreaded', engine='parallel',
+        max_workers=self.config.max_workers)
+    e.run()
+
+    return flow
+
+In the applier tasks are contained in a :class:`~.TaskFlowActionContainer`
+which allows them to trigger events in the workflow engine. This way the
+workflow engine can halt or take other actions while the action plan is being
+executed based on the success or failure of individual actions. However, the
+base workflow engine simply uses these notifies to store the result of
+individual actions in the database. Additionally, since taskflow uses a graph
+flow if any of the tasks would fail all childs of this tasks not be executed
+while ``do_revert`` will be triggered for all parents.
+
+.. code-block:: python
+
+    class TaskFlowActionContainer(...):
+        ...
+        def do_execute(self, *args, **kwargs):
+            ...
+            result = self.action.execute()
+            if result is True:
+                return self.engine.notify(self._db_action,
+                                          objects.action.State.SUCCEEDED)
+            else:
+                self.engine.notify(self._db_action,
+                                   objects.action.State.FAILED)
+
+    class BaseWorkFlowEngine(...):
+        ...
+        def notify(self, action, state):
+            db_action = objects.Action.get_by_uuid(self.context, action.uuid,
+                                                   eager=True)
+            db_action.state = state
+            db_action.save()
+            return db_action

doc/source/contributor/contributing.rst

@@ -1,71 +1,111 @@
-..
-      Except where otherwise noted, this document is licensed under Creative
-      Commons Attribution 3.0 License.  You can view the license at:
+============================
+So You Want to Contribute...
+============================
 
-          https://creativecommons.org/licenses/by/3.0/
+For general information on contributing to OpenStack, please check out the
+`contributor guide <https://docs.openstack.org/contributors/>`_ to get started.
+It covers all the basics that are common to all OpenStack projects:
+the accounts you need, the basics of interacting with our Gerrit review system,
+how we communicate as a community, etc.
 
-.. _contributing:
+Below will cover the more project specific information you need to get started
+with Watcher.
 
-=======================
-Contributing to Watcher
-=======================
-
-If you're interested in contributing to the Watcher project,
-the following will help get you started.
-
-Contributor License Agreement
------------------------------
-
-.. index::
-   single: license; agreement
-
-In order to contribute to the Watcher project, you need to have
-signed OpenStack's contributor's agreement.
-
-.. seealso::
-
-   * https://docs.openstack.org/infra/manual/developers.html
-   * https://wiki.openstack.org/CLA
-
-LaunchPad Project
------------------
-
-Most of the tools used for OpenStack depend on a launchpad.net ID for
-authentication. After signing up for a launchpad account, join the
-"openstack" team to have access to the mailing list and receive
-notifications of important events.
-
-.. seealso::
-
-   * https://launchpad.net
-   * https://launchpad.net/watcher
-   * https://launchpad.net/openstack
-
-
-Project Hosting Details
------------------------
-
-Bug tracker
-    https://launchpad.net/watcher
-
-Mailing list (prefix subjects with ``[watcher]`` for faster responses)
-    http://lists.openstack.org/pipermail/openstack-discuss/
-
-Wiki
-    https://wiki.openstack.org/Watcher
-
-Code Hosting
-    https://opendev.org/openstack/watcher
-
-Code Review
-    https://review.opendev.org/#/q/status:open+project:openstack/watcher,n,z
+Communication
+~~~~~~~~~~~~~~
+.. This would be a good place to put the channel you chat in as a project; when/
+   where your meeting is, the tags you prepend to your ML threads, etc.
 
 IRC Channel
     ``#openstack-watcher`` (changelog_)
 
+Mailing list(prefix subjects with ``[watcher]``)
+    http://lists.openstack.org/pipermail/openstack-discuss/
+
 Weekly Meetings
     Bi-weekly, on Wednesdays at 08:00 UTC on odd weeks in the
     ``#openstack-meeting-alt`` IRC channel (`meetings logs`_)
 
+Meeting Agenda
+    https://wiki.openstack.org/wiki/Watcher_Meeting_Agenda
+
 .. _changelog: http://eavesdrop.openstack.org/irclogs/%23openstack-watcher/
 .. _meetings logs:  http://eavesdrop.openstack.org/meetings/watcher/
+
+Contacting the Core Team
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. This section should list the core team, their irc nicks, emails, timezones etc.
+   If all this info is maintained elsewhere (i.e. a wiki), you can link to that
+   instead of enumerating everyone here.
+
++--------------------+---------------+------------------------------------+
+| Name               | IRC           | Email                              |
++====================+===============+====================================+
+| `Li Canwei`_       | licanwei      | li.canwei2@zte.com.cn              |
++--------------------+---------------+------------------------------------+
+| `chen ke`_         | chenke        | chen.ke14@zte.com.cn               |
++--------------------+---------------+------------------------------------+
+| `Corne Lukken`_    | dantalion     | info@dantalion.nl                  |
++--------------------+---------------+------------------------------------+
+| `su zhengwei`_     | suzhengwei    | sugar-2008@163.com                 |
++--------------------+---------------+------------------------------------+
+| `Yumeng Bao`_      | Yumeng        | yumeng_bao@yahoo.com               |
++--------------------+---------------+------------------------------------+
+
+.. _Corne Lukken: https://launchpad.net/~dantalion
+.. _Li Canwei: https://launchpad.net/~li-canwei2
+.. _su zhengwei: https://launchpad.net/~sue.sam
+.. _Yumeng Bao: https://launchpad.net/~yumeng-bao
+.. _chen ke: https://launchpad.net/~chenker
+
+New Feature Planning
+~~~~~~~~~~~~~~~~~~~~
+.. This section is for talking about the process to get a new feature in. Some
+   projects use blueprints, some want specs, some want both! Some projects
+   stick to a strict schedule when selecting what new features will be reviewed
+   for a release.
+
+New feature will be discussed via IRC or ML (with [Watcher] prefix).
+Watcher team uses blueprints in `Launchpad`_ to manage the new features.
+
+.. _Launchpad: https://launchpad.net/watcher
+
+Task Tracking
+~~~~~~~~~~~~~~
+.. This section is about where you track tasks- launchpad? storyboard?
+   is there more than one launchpad project? what's the name of the project
+   group in storyboard?
+
+We track our tasks in Launchpad.
+If you're looking for some smaller, easier work item to pick up and get started
+on, search for the 'low-hanging-fruit' tag.
+
+.. NOTE: If your tag is not 'low-hanging-fruit' please change the text above.
+
+Reporting a Bug
+~~~~~~~~~~~~~~~
+.. Pretty self explanatory section, link directly to where people should report bugs for
+   your project.
+
+You found an issue and want to make sure we are aware of it? You can do so
+`HERE`_.
+
+.. _HERE: https://bugs.launchpad.net/watcher
+
+Getting Your Patch Merged
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. This section should have info about what it takes to get something merged.
+   Do you require one or two +2's before +W? Do some of your repos require
+   unit test changes with all patches? etc.
+
+Due to the small number of core reviewers of the Watcher project,
+we only need one +2 before +W (merge). All patches excepting for documentation
+or typos fixes must have unit test.
+
+Project Team Lead Duties
+------------------------
+.. this section is where you can put PTL specific duties not already listed in
+   the common PTL guide (linked below)  or if you already have them written
+   up elsewhere, you can link to that doc here.
+
+All common PTL duties are enumerated here in the `PTL guide <https://docs.openstack.org/project-team-guide/ptl.html>`_.

doc/source/contributor/index.rst

@@ -1,8 +1,12 @@
-.. toctree::
-   :maxdepth: 1
+==================
+Contribution Guide
+==================
 
+.. toctree::
+   :maxdepth: 2
+
+   contributing
    environment
    devstack
-   notifications
    testing
    rally_link

doc/source/contributor/plugin/index.rst

@@ -1,3 +1,7 @@
+============
+Plugin Guide
+============
+
 .. toctree::
    :maxdepth: 1
 

doc/source/contributor/plugin/strategy-plugin.rst

@@ -56,9 +56,6 @@ Here is an example showing how you can write a plugin called ``NewStrategy``:
     # filepath: thirdparty/new.py
     # import path: thirdparty.new
     import abc
-
-    import six
-
     from watcher._i18n import _
     from watcher.decision_engine.strategy.strategies import base
 

doc/source/contributor/testing.rst

@@ -4,9 +4,9 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-=======
-Testing
-=======
+=================
+Developer Testing
+=================
 
 .. _unit_tests:
 
@@ -15,7 +15,7 @@ Unit tests
 
 All unit tests should be run using `tox`_. Before running the unit tests, you
 should download the latest `watcher`_ from the github. To run the same unit
-tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
+tests that are executing onto `Gerrit`_ which includes ``py36``, ``py37`` and
 ``pep8``, you can issue the following command::
 
     $ git clone https://opendev.org/openstack/watcher
@@ -26,8 +26,8 @@ tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
 If you only want to run one of the aforementioned, you can then issue one of
 the following::
 
-    $ tox -e py35
-    $ tox -e py27
+    $ tox -e py36
+    $ tox -e py37
     $ tox -e pep8
 
 .. _tox: https://tox.readthedocs.org/
@@ -38,7 +38,7 @@ If you only want to run specific unit test code and don't like to waste time
 waiting for all unit tests to execute, you can add parameters ``--`` followed
 by a regex string::
 
-    $ tox -e py27 -- watcher.tests.api
+    $ tox -e py37 -- watcher.tests.api
 
 .. _tempest_tests:
 

doc/source/index.rst

@@ -32,91 +32,21 @@ specific prior release.
 .. _python-watcherclient: https://opendev.org/openstack/python-watcherclient/
 .. _watcher-dashboard: https://opendev.org/openstack/watcher-dashboard/
 
-Developer Guide
-===============
-
-Introduction
-------------
-
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
-  glossary
   architecture
-  contributor/contributing
-
-
-Getting Started
----------------
-
-.. toctree::
-  :maxdepth: 1
-
   contributor/index
-
-Installation
-============
-.. toctree::
-  :maxdepth: 2
-
   install/index
-
-Admin Guide
-===========
-
-.. toctree::
-  :maxdepth: 2
-
   admin/index
-
-User Guide
-==========
-
-.. toctree::
-  :maxdepth: 2
-
   user/index
-
-API References
-==============
+  configuration/index
+  contributor/plugin/index
+  man/index
 
 .. toctree::
   :maxdepth: 1
 
   API Reference <https://docs.openstack.org/api-ref/resource-optimization/>
   Watcher API Microversion History </contributor/api_microversion_history>
-
-Plugins
--------
-
-.. toctree::
-  :maxdepth: 1
-
-  contributor/plugin/index
-
-Watcher Configuration Options
-=============================
-
-.. toctree::
-  :maxdepth: 2
-
-  configuration/index
-
-Watcher Manual Pages
-====================
-
-.. toctree::
-   :glob:
-   :maxdepth: 1
-
-   man/index
-
-
-.. only:: html
-
-Indices and tables
-==================
-
-   * :ref:`genindex`
-   * :ref:`modindex`
-   * :ref:`search`
+  glossary

doc/source/install/index.rst

@@ -1,6 +1,6 @@
-===================================
-Infrastructure Optimization service
-===================================
+=============
+Install Guide
+=============
 
 .. toctree::
    :maxdepth: 2

doc/source/man/index.rst

@@ -1,3 +1,7 @@
+====================
+Watcher Manual Pages
+====================
+
 .. toctree::
    :glob:
    :maxdepth: 1

doc/source/user/event_type_audit.rst

@@ -0,0 +1,195 @@
+..
+  Licensed under the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License. You may obtain
+  a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+
+
+======================
+Audit using Aodh alarm
+======================
+
+Audit with EVENT type can be triggered by special alarm. This guide walks
+you through the steps to build an event-driven optimization solution by
+integrating Watcher with Ceilometer/Aodh.
+
+Step 1: Create an audit with EVENT type
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first step is to create an audit with EVENT type,
+you can create an audit template firstly:
+
+.. code-block:: bash
+
+  $ openstack optimize audittemplate create your_template_name <your_goal> \
+    --strategy <your_strategy>
+
+or create an audit directly with special goal and strategy:
+
+.. code-block:: bash
+
+  $ openstack optimize audit create --goal <your_goal> \
+    --strategy <your_strategy> --audit_type EVENT
+
+This is an example for creating an audit with dummy strategy:
+
+.. code-block:: bash
+
+  $ openstack optimize audit create --goal dummy \
+    --strategy dummy --audit_type EVENT
+  +---------------+--------------------------------------+
+  | Field         | Value                                |
+  +---------------+--------------------------------------+
+  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
+  | Name          | dummy-2020-01-14T03:21:19.168467     |
+  | Created At    | 2020-01-14T03:21:19.200279+00:00     |
+  | Updated At    | None                                 |
+  | Deleted At    | None                                 |
+  | State         | PENDING                              |
+  | Audit Type    | EVENT                                |
+  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
+  | Interval      | None                                 |
+  | Goal          | dummy                                |
+  | Strategy      | dummy                                |
+  | Audit Scope   | []                                   |
+  | Auto Trigger  | False                                |
+  | Next Run Time | None                                 |
+  | Hostname      | None                                 |
+  | Start Time    | None                                 |
+  | End Time      | None                                 |
+  | Force         | False                                |
+  +---------------+--------------------------------------+
+
+We need to build Aodh action url using Watcher webhook API.
+For convenience we export the url into an environment variable:
+
+.. code-block:: bash
+
+  $ export AUDIT_UUID=a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  $ export ALARM_URL="trust+http://localhost/infra-optim/v1/webhooks/$AUDIT_UUID"
+
+Step 2: Create Aodh Alarm
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Once we have the audit created, we can continue to create Aodh alarm and
+set the alarm action to Watcher webhook API. The alarm type can be event(
+i.e. ``compute.instance.create.end``) or gnocchi_resources_threshold(i.e.
+``cpu_util``), more info refer to alarm-creation_
+
+For example:
+
+.. code-block:: bash
+
+  $ openstack alarm create \
+    --type event --name instance_create \
+    --event-type "compute.instance.create.end" \
+    --enable True --repeat-actions False \
+    --alarm-action $ALARM_URL
+  +---------------------------+------------------------------------------------------------------------------------------+
+  | Field                     | Value                                                                                    |
+  +---------------------------+------------------------------------------------------------------------------------------+
+  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5'] |
+  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                     |
+  | description               | Alarm when compute.instance.create.end event occurred.                                   |
+  | enabled                   | True                                                                                     |
+  | event_type                | compute.instance.create.end                                                              |
+  | insufficient_data_actions | []                                                                                       |
+  | name                      | instance_create                                                                          |
+  | ok_actions                | []                                                                                       |
+  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                         |
+  | query                     |                                                                                          |
+  | repeat_actions            | False                                                                                    |
+  | severity                  | low                                                                                      |
+  | state                     | insufficient data                                                                        |
+  | state_reason              | Not evaluated yet                                                                        |
+  | state_timestamp           | 2020-01-14T03:56:26.894416                                                               |
+  | time_constraints          | []                                                                                       |
+  | timestamp                 | 2020-01-14T03:56:26.894416                                                               |
+  | type                      | event                                                                                    |
+  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                         |
+  +---------------------------+------------------------------------------------------------------------------------------+
+
+.. _alarm-creation: https://docs.openstack.org/aodh/latest/admin/telemetry-alarms.html#alarm-creation
+
+Step 3: Trigger the alarm
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this example, you can create a new instance to trigger the alarm.
+The alarm state will translate from ``insufficient data`` to ``alarm``.
+
+.. code-block:: bash
+
+  $ openstack alarm show b9e381fc-8e3e-4943-82ee-647e7a2ef644
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+  | Field                     | Value                                                                                                             |
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5']                          |
+  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                                              |
+  | description               | Alarm when compute.instance.create.end event occurred.                                                            |
+  | enabled                   | True                                                                                                              |
+  | event_type                | compute.instance.create.end                                                                                       |
+  | insufficient_data_actions | []                                                                                                                |
+  | name                      | instance_create                                                                                                   |
+  | ok_actions                | []                                                                                                                |
+  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                                                  |
+  | query                     |                                                                                                                   |
+  | repeat_actions            | False                                                                                                             |
+  | severity                  | low                                                                                                               |
+  | state                     | alarm                                                                                                             |
+  | state_reason              | Event <id=67dd0afa-2082-45a4-8825-9573b2cc60e5,event_type=compute.instance.create.end> hits the query <query=[]>. |
+  | state_timestamp           | 2020-01-14T03:56:26.894416                                                                                        |
+  | time_constraints          | []                                                                                                                |
+  | timestamp                 | 2020-01-14T06:17:40.350649                                                                                        |
+  | type                      | event                                                                                                             |
+  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                                                  |
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+
+Step 4: Verify the audit
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can be verified to check if the audit state was ``SUCCEEDED``:
+
+.. code-block:: bash
+
+  $ openstack optimize audit show a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  +---------------+--------------------------------------+
+  | Field         | Value                                |
+  +---------------+--------------------------------------+
+  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
+  | Name          | dummy-2020-01-14T03:21:19.168467     |
+  | Created At    | 2020-01-14T03:21:19+00:00            |
+  | Updated At    | 2020-01-14T06:26:40+00:00            |
+  | Deleted At    | None                                 |
+  | State         | SUCCEEDED                            |
+  | Audit Type    | EVENT                                |
+  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
+  | Interval      | None                                 |
+  | Goal          | dummy                                |
+  | Strategy      | dummy                                |
+  | Audit Scope   | []                                   |
+  | Auto Trigger  | False                                |
+  | Next Run Time | None                                 |
+  | Hostname      | ubuntudbs                            |
+  | Start Time    | None                                 |
+  | End Time      | None                                 |
+  | Force         | False                                |
+  +---------------+--------------------------------------+
+
+and you can use the following command to check if the action plan
+was created:
+
+.. code-block:: bash
+
+  $ openstack optimize actionplan list --audit a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
+  | UUID                                 | Audit                                | State       | Updated At | Global efficacy |
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
+  | 673b3fcb-8c16-4a41-9ee3-2956d9f6ca9e | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 | RECOMMENDED | None       |                 |
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+

doc/source/user/index.rst

@@ -1,4 +1,10 @@
+==========
+User Guide
+==========
+
 .. toctree::
    :maxdepth: 2
 
+   ways-to-install
    user-guide
+   event_type_audit

doc/source/user/user-guide.rst

@@ -4,8 +4,6 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-.. _user-guide:
-
 ==================
 Watcher User Guide
 ==================
@@ -60,8 +58,8 @@ plugin installation guide`_.
 .. _`OpenStack CLI`: https://docs.openstack.org/python-openstackclient/latest/cli/man/openstack.html
 .. _`Watcher CLI`: https://docs.openstack.org/python-watcherclient/latest/cli/index.html
 
-Seeing what the Watcher CLI can do ?
-------------------------------------
+Watcher CLI Command
+-------------------
 We can see all of the commands available with Watcher CLI by running the
 watcher binary without options.
 
@@ -69,8 +67,8 @@ watcher binary without options.
 
   $ openstack help optimize
 
-How do I run an audit of my cluster ?
--------------------------------------
+Running an audit of the cluster
+-------------------------------
 
 First, you need to find the :ref:`goal <goal_definition>` you want to achieve:
 

lower-constraints.txt

@@ -1,165 +0,0 @@
-alabaster==0.7.10
-alembic==0.9.8
-amqp==2.2.2
-appdirs==1.4.3
-APScheduler==3.5.1
-asn1crypto==0.24.0
-automaton==1.14.0
-Babel==2.5.3
-beautifulsoup4==4.6.0
-cachetools==2.0.1
-certifi==2018.1.18
-cffi==1.11.5
-chardet==3.0.4
-cliff==2.11.0
-cmd2==0.8.1
-contextlib2==0.5.5
-coverage==4.5.1
-croniter==0.3.20
-cryptography==2.1.4
-debtcollector==1.19.0
-decorator==4.2.1
-deprecation==2.0
-doc8==0.8.0
-docutils==0.14
-dogpile.cache==0.6.5
-dulwich==0.19.0
-enum34==1.1.6
-enum-compat==0.0.2
-eventlet==0.20.0
-extras==1.0.0
-fasteners==0.14.1
-fixtures==3.0.0
-flake8==2.5.5
-freezegun==0.3.10
-future==0.16.0
-futurist==1.8.0
-gitdb2==2.0.3
-GitPython==2.1.8
-gnocchiclient==7.0.1
-greenlet==0.4.13
-hacking==0.12.0
-idna==2.6
-imagesize==1.0.0
-iso8601==0.1.12
-Jinja2==2.10
-jmespath==0.9.3
-jsonpatch==1.21
-jsonpointer==2.0
-jsonschema==2.6.0
-keystoneauth1==3.4.0
-keystonemiddleware==4.21.0
-kombu==4.1.0
-linecache2==1.0.0
-logutils==0.3.5
-lxml==4.1.1
-Mako==1.0.7
-MarkupSafe==1.0
-mccabe==0.2.1
-microversion_parse==0.2.1
-mock==2.0.0
-monotonic==1.4
-mox3==0.25.0
-msgpack==0.5.6
-munch==2.2.0
-netaddr==0.7.19
-netifaces==0.10.6
-networkx==1.11
-openstackdocstheme==1.20.0
-openstacksdk==0.12.0
-os-api-ref===1.4.0
-os-client-config==1.29.0
-os-service-types==1.2.0
-os-testr==1.0.0
-osc-lib==1.10.0
-os-resource-classes==0.4.0
-oslo.cache==1.29.0
-oslo.concurrency==3.26.0
-oslo.config==5.2.0
-oslo.context==2.21.0
-oslo.db==4.35.0
-oslo.i18n==3.20.0
-oslo.log==3.37.0
-oslo.messaging==8.1.2
-oslo.middleware==3.35.0
-oslo.policy==1.34.0
-oslo.reports==1.27.0
-oslo.serialization==2.25.0
-oslo.service==1.30.0
-oslo.upgradecheck==0.1.0
-oslo.utils==3.36.0
-oslo.versionedobjects==1.32.0
-oslotest==3.3.0
-packaging==17.1
-Paste==2.0.3
-PasteDeploy==1.5.2
-pbr==3.1.1
-pecan==1.3.2
-pep8==1.5.7
-pika==0.10.0
-pika-pool==0.1.3
-prettytable==0.7.2
-psutil==5.4.3
-pycadf==2.7.0
-pycparser==2.18
-pyflakes==0.8.1
-Pygments==2.2.0
-pyinotify==0.9.6
-pyOpenSSL==17.5.0
-pyparsing==2.2.0
-pyperclip==1.6.0
-python-ceilometerclient==2.9.0
-python-cinderclient==3.5.0
-python-dateutil==2.7.0
-python-editor==1.0.3
-python-glanceclient==2.9.1
-python-ironicclient==2.5.0
-python-keystoneclient==3.15.0
-python-mimeparse==1.6.0
-python-monascaclient==1.12.0
-python-neutronclient==6.7.0
-python-novaclient==14.1.0
-python-openstackclient==3.14.0
-python-subunit==1.2.0
-pytz==2018.3
-PyYAML==3.12
-reno==2.7.0
-repoze.lru==0.7
-requests==2.18.4
-requestsexceptions==1.4.0
-restructuredtext-lint==1.1.3
-rfc3986==1.1.0
-Routes==2.4.1
-simplegeneric==0.8.1
-simplejson==3.13.2
-six==1.11.0
-smmap2==2.0.3
-snowballstemmer==1.2.1
-Sphinx==1.6.5
-sphinxcontrib-httpdomain==1.6.1
-sphinxcontrib-pecanwsme==0.8.0
-sphinxcontrib-websupport==1.0.1
-SQLAlchemy==1.2.5
-sqlalchemy-migrate==0.11.0
-sqlparse==0.2.4
-statsd==3.2.2
-stestr==2.0.0
-stevedore==1.28.0
-taskflow==3.1.0
-Tempita==0.5.2
-tenacity==4.9.0
-testresources==2.0.1
-testscenarios==0.5.0
-testtools==2.3.0
-traceback2==1.4.0
-tzlocal==1.5.1
-ujson==1.35
-unittest2==1.1.0
-urllib3==1.22
-vine==1.1.4
-waitress==1.1.0
-warlock==1.3.0
-WebOb==1.8.5
-WebTest==2.0.29
-wrapt==1.10.11
-WSME==0.9.2

playbooks/legacy/grenade-devstack-watcher/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/grenade-devstack-watcher/run.yaml

@@ -1,60 +0,0 @@
-- hosts: all
-  name: legacy-grenade-dsvm-watcher
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-
-          export PROJECTS="openstack/grenade $PROJECTS"
-          export PROJECTS="openstack/watcher $PROJECTS"
-          export PROJECTS="openstack/watcher-tempest-plugin $PROJECTS"
-          export PROJECTS="openstack/python-watcherclient $PROJECTS"
-          export DEVSTACK_PROJECT_FROM_GIT="python-watcherclient $DEVSTACK_PROJECT_FROM_GIT"
-
-          export GRENADE_PLUGINRC="enable_grenade_plugin watcher https://opendev.org/openstack/watcher"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"export TEMPEST_PLUGINS='/opt/stack/new/watcher-tempest-plugin'"
-
-          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
-          export DEVSTACK_GATE_GRENADE=pullup
-
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-          # Add configuration values for enabling security features in local.conf
-          function pre_test_hook {
-              if [ -f /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh ] ; then
-                  . /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh
-              fi
-          }
-          export -f pre_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

releasenotes/notes/deprecate-json-formatted-policy-file-3a92379e9f5dd203.yaml

@@ -0,0 +1,20 @@
+---
+upgrade:
+  - |
+    The default value of ``[oslo_policy] policy_file`` config option has
+    been changed from ``policy.json`` to ``policy.yaml``.
+    Operators who are utilizing customized or previously generated
+    static policy JSON files (which are not needed by default), should
+    generate new policy files or convert them in YAML format. Use the
+    `oslopolicy-convert-json-to-yaml
+    <https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_
+    tool to convert a JSON to YAML formatted policy file in
+    backward compatible way.
+deprecations:
+  - |
+    Use of JSON policy files was deprecated by the ``oslo.policy`` library
+    during the Victoria development cycle. As a result, this deprecation is
+    being noted in the Wallaby cycle with an anticipated future removal of support
+    by ``oslo.policy``. As such operators will need to convert to YAML policy
+    files. Please see the upgrade notes for details on migration of any
+    custom policy files.

releasenotes/notes/drop-py-2-7-54f8e806d71f19a7.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Python 2.7 support has been dropped. Last release of Watcher
+    to support py2.7 is OpenStack Train. The minimum version of Python now
+    supported by Watcher is Python 3.6.

releasenotes/notes/event-driven-optimization-based-4870f112bef8a560.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add a new webhook API and a new audit type EVENT, the microversion is 1.4.
+    Now Watcher user can create audit with EVENT type and the audit will be
+    triggered by webhook API.
+    The user guide is available online:
+    https://docs.openstack.org/watcher/latest/user/event_type_audit.html

releasenotes/notes/general-purpose-decision-engine-threadpool-0711b23abfc9d409.yaml

@@ -0,0 +1,20 @@
+---
+prelude: >
+    Many operations in the decision engine will block on I/O. Such I/O
+    operations can stall the execution of a sequential application
+    significantly. To reduce the potential bottleneck of many operations
+    the general purpose decision engine threadpool is introduced.
+features:
+  - |
+    A new threadpool for the decision engine that contributors can use to
+    improve the performance of many operations, primarily I/O bound onces.
+    The amount of workers used by the decision engine threadpool can be
+    configured to scale according to the available infrastructure using
+    the `watcher_decision_engine.max_general_workers` config option.
+    Documentation for contributors to effectively use this threadpool is
+    available online:
+    https://docs.openstack.org/watcher/latest/contributor/concurrency.html
+  - |
+    The building of the compute (Nova) data model will be done using the
+    decision engine threadpool, thereby, significantly reducing the total
+    time required to build it.

releasenotes/source/conf.py

@@ -53,7 +53,6 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'watcher'
 copyright = u'2016, Watcher developers'
 
 # Release notes are version independent
@@ -91,11 +90,15 @@ exclude_patterns = ['_build']
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []
 
+# openstackdocstheme options
+openstackdocs_repo_name = 'openstack/watcher'
+openstackdocs_bug_project = 'watcher'
+openstackdocs_bug_tag = ''
 
 # -- Options for HTML output --------------------------------------------------
 

releasenotes/source/index.rst

@@ -21,6 +21,9 @@ Contents:
    :maxdepth: 1
 
    unreleased
+   victoria
+   ussuri
+   train
    stein
    rocky
    queens

releasenotes/source/locale/fr/LC_MESSAGES/releasenotes.po

@@ -0,0 +1,33 @@
+# Gérald LONLAS <g.lonlas@gmail.com>, 2016. #zanata
+msgid ""
+msgstr ""
+"Project-Id-Version: python-watcher\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2019-03-22 02:21+0000\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"PO-Revision-Date: 2016-10-22 06:44+0000\n"
+"Last-Translator: Gérald LONLAS <g.lonlas@gmail.com>\n"
+"Language-Team: French\n"
+"Language: fr\n"
+"X-Generator: Zanata 4.3.3\n"
+"Plural-Forms: nplurals=2; plural=(n > 1)\n"
+
+msgid "0.29.0"
+msgstr "0.29.0"
+
+msgid "Contents:"
+msgstr "Contenu :"
+
+msgid "Current Series Release Notes"
+msgstr "Note de la release actuelle"
+
+msgid "New Features"
+msgstr "Nouvelles fonctionnalités"
+
+msgid "Newton Series Release Notes"
+msgstr "Note de release pour Newton"
+
+msgid "Welcome to watcher's Release Notes documentation!"
+msgstr "Bienvenue dans la documentation de la note de Release de Watcher"

releasenotes/source/train.rst

@@ -0,0 +1,6 @@
+==========================
+Train Series Release Notes
+==========================
+
+.. release-notes::
+   :branch: stable/train

releasenotes/source/ussuri.rst

@@ -0,0 +1,6 @@
+===========================
+Ussuri Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/ussuri

releasenotes/source/victoria.rst

@@ -0,0 +1,6 @@
+=============================
+Victoria Series Release Notes
+=============================
+
+.. release-notes::
+   :branch: stable/victoria

requirements.txt

@@ -1,35 +1,34 @@
-# The order of packages is significant, because pip processes them in the order
+#  The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
 apscheduler>=3.5.1 # MIT License
-enum34>=1.1.6;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
 jsonpatch>=1.21 # BSD
 keystoneauth1>=3.4.0 # Apache-2.0
-jsonschema>=2.6.0 # MIT
+jsonschema>=3.2.0 # MIT
 keystonemiddleware>=4.21.0 # Apache-2.0
-lxml>=4.1.1 # BSD
+lxml>=4.5.1 # BSD
 croniter>=0.3.20 # MIT License
 os-resource-classes>=0.4.0
 oslo.concurrency>=3.26.0 # Apache-2.0
 oslo.cache>=1.29.0 # Apache-2.0
-oslo.config>=5.2.0 # Apache-2.0
+oslo.config>=6.8.0 # Apache-2.0
 oslo.context>=2.21.0 # Apache-2.0
-oslo.db>=4.35.0 # Apache-2.0
+oslo.db>=4.44.0 # Apache-2.0
 oslo.i18n>=3.20.0 # Apache-2.0
 oslo.log>=3.37.0 # Apache-2.0
 oslo.messaging>=8.1.2 # Apache-2.0
-oslo.policy>=1.34.0 # Apache-2.0
+oslo.policy>=3.6.0 # Apache-2.0
 oslo.reports>=1.27.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
 oslo.service>=1.30.0 # Apache-2.0
-oslo.upgradecheck>=0.1.0 # Apache-2.0
+oslo.upgradecheck>=1.3.0 # Apache-2.0
 oslo.utils>=3.36.0 # Apache-2.0
 oslo.versionedobjects>=1.32.0 # Apache-2.0
 PasteDeploy>=1.5.2 # MIT
 pbr>=3.1.1 # Apache-2.0
 pecan>=1.3.2 # BSD
-PrettyTable<0.8,>=0.7.2 # BSD
+PrettyTable>=0.7.2 # BSD
 gnocchiclient>=7.0.1 # Apache-2.0
 python-ceilometerclient>=2.9.0 # Apache-2.0
 python-cinderclient>=3.5.0 # Apache-2.0
@@ -40,14 +39,11 @@ python-neutronclient>=6.7.0 # Apache-2.0
 python-novaclient>=14.1.0 # Apache-2.0
 python-openstackclient>=3.14.0 # Apache-2.0
 python-ironicclient>=2.5.0 # Apache-2.0
-six>=1.11.0 # MIT
 SQLAlchemy>=1.2.5 # MIT
 stevedore>=1.28.0 # Apache-2.0
-taskflow>=3.1.0 # Apache-2.0
+taskflow>=3.8.0 # Apache-2.0
 WebOb>=1.8.5 # MIT
 WSME>=0.9.2 # MIT
-# NOTE(fdegir): NetworkX 2.3 dropped support for Python 2
-networkx>=1.11,<2.3;python_version<'3.0'  # BSD
-networkx>=1.11;python_version>='3.4' # BSD
+networkx>=2.4 # BSD
 microversion_parse>=0.2.1 # Apache-2.0
 futurist>=1.8.0 # Apache-2.0

setup.cfg

@@ -6,6 +6,7 @@ description-file =
 author = OpenStack
 author-email = openstack-discuss@lists.openstack.org
 home-page = https://docs.openstack.org/watcher/latest/
+python-requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -13,11 +14,12 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
 
 [files]
 packages =
@@ -25,10 +27,6 @@ packages =
 data_files =
     etc/ = etc/*
 
-[global]
-setup-hooks =
-    pbr.hooks.setup_hook
-
 [entry_points]
 oslo.config.opts =
     watcher = watcher.conf.opts:list_opts
@@ -110,18 +108,3 @@ watcher_cluster_data_model_collectors =
     compute = watcher.decision_engine.model.collector.nova:NovaClusterDataModelCollector
     storage = watcher.decision_engine.model.collector.cinder:CinderClusterDataModelCollector
     baremetal = watcher.decision_engine.model.collector.ironic:BaremetalClusterDataModelCollector
-
-
-[compile_catalog]
-directory = watcher/locale
-domain = watcher
-
-[update_catalog]
-domain = watcher
-output_dir = watcher/locale
-input_file = watcher/locale/watcher.pot
-
-[extract_messages]
-keywords = _ gettext ngettext l_ lazy_gettext _LI _LW _LE _LC
-mapping_file = babel.cfg
-output_file = watcher/locale/watcher.pot

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
     pbr=True)

test-requirements.txt

@@ -5,8 +5,7 @@
 coverage>=4.5.1 # Apache-2.0
 doc8>=0.8.0 # Apache-2.0
 freezegun>=0.3.10 # Apache-2.0
-hacking>=1.1.0,<1.2.0 # Apache-2.0
-mock>=2.0.0 # BSD
+hacking>=3.0.1,<3.1.0 # Apache-2.0
 oslotest>=3.3.0 # Apache-2.0
 os-testr>=1.0.0 # Apache-2.0
 testscenarios>=0.5.0 # Apache-2.0/BSD

tox.ini

@@ -1,9 +1,11 @@
 [tox]
 minversion = 2.0
-envlist = py36,py37,py27,pep8
+envlist = py38,pep8
 skipsdist = True
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 usedevelop = True
 whitelist_externals = find
                       rm
@@ -11,7 +13,7 @@ install_command = pip install {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train}
+    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/wallaby}
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
 commands =
@@ -21,24 +23,21 @@ commands =
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
 
 [testenv:pep8]
-basepython = python3
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
     bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:venv]
-basepython = python3
 setenv = PYTHONHASHSEED=0
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train}
+    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/wallaby}
     -r{toxinidir}/doc/requirements.txt
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
 commands = {posargs}
 
 [testenv:cover]
-basepython = python3
 setenv =
     PYTHON=coverage run --source watcher --parallel-mode
 commands =
@@ -49,51 +48,68 @@ commands =
     coverage report
 
 [testenv:docs]
-basepython = python3
 setenv = PYTHONHASHSEED=0
-deps = -r{toxinidir}/doc/requirements.txt
+deps =
+  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/doc/requirements.txt
 commands =
   rm -fr doc/build doc/source/api/ .autogenerated
-  sphinx-build -W -b html doc/source doc/build/html
+  sphinx-build -W --keep-going -b html doc/source doc/build/html
 
 [testenv:api-ref]
-basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt
 whitelist_externals = bash
 commands =
   bash -c 'rm -rf api-ref/build'
-  sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
+  sphinx-build -W --keep-going -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
 
 [testenv:debug]
-basepython = python3
 commands = oslo_debug_helper -t watcher/tests {posargs}
 
 [testenv:genconfig]
-basepython = python3
 sitepackages = False
 commands =
     oslo-config-generator --config-file etc/watcher/oslo-config-generator/watcher.conf
 
 [testenv:genpolicy]
-basepython = python3
 commands =
     oslopolicy-sample-generator --config-file etc/watcher/oslo-policy-generator/watcher-policy-generator.conf
 
 [flake8]
 filename = *.py,app.wsgi
 show-source=True
-ignore= H105,E123,E226,N320,H202
+# W504 line break after binary operator
+ignore= H105,E123,E226,N320,H202,W504
 builtins= _
 enable-extensions = H106,H203,H904
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*,demo/,releasenotes
 
 [testenv:wheel]
-basepython = python3
 commands = python setup.py bdist_wheel
 
 [hacking]
 import_exceptions = watcher._i18n
-local-check-factory = watcher.hacking.checks.factory
+
+[flake8:local-plugins]
+extension =
+   N319 = checks:no_translate_debug_logs
+   N321 = checks:use_jsonutils
+   N322 = checks:check_assert_called_once_with
+   N325 = checks:check_python3_xrange
+   N326 = checks:check_no_basestring
+   N327 = checks:check_python3_no_iteritems
+   N328 = checks:check_asserttrue
+   N329 = checks:check_assertfalse
+   N330 = checks:check_assertempty
+   N331 = checks:check_assertisinstance
+   N332 = checks:check_assertequal_for_httpcode
+   N333 = checks:check_log_warn_deprecated
+   N340 = checks:check_oslo_i18n_wrapper
+   N341 = checks:check_builtins_gettext
+   N342 = checks:no_redundant_import_alias
+   N366 = checks:import_stock_mock
+paths = ./watcher/hacking
+
 
 [doc8]
 extension=.rst
@@ -101,7 +117,6 @@ extension=.rst
 ignore-path=doc/source/image_src,doc/source/man,doc/source/api
 
 [testenv:pdf-docs]
-basepython = python3
 envdir = {toxworkdir}/docs
 deps = {[testenv:docs]deps}
 whitelist_externals =
@@ -109,22 +124,13 @@ whitelist_externals =
   make
 commands =
   rm -rf doc/build/pdf
-  sphinx-build -W -b latex doc/source doc/build/pdf
+  sphinx-build -W --keep-going -b latex doc/source doc/build/pdf
   make -C doc/build/pdf
 
 [testenv:releasenotes]
-basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
 
 [testenv:bandit]
-basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands = bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
-
-[testenv:lower-constraints]
-basepython = python3
-deps =
-  -c{toxinidir}/lower-constraints.txt
-  -r{toxinidir}/test-requirements.txt
-  -r{toxinidir}/requirements.txt

watcher/api/acl.py

@@ -37,5 +37,5 @@ def install(app, conf, public_routes):
     if not CONF.get('enable_authentication'):
         return app
     return auth_token.AuthTokenMiddleware(app,
-                                          conf=dict(conf),
+                                          conf=dict(conf.keystone_authtoken),
                                           public_api_routes=public_routes)

watcher/api/config.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import unicode_literals
-
 from oslo_config import cfg
 from watcher.api import hooks
 
@@ -27,6 +25,10 @@ server = {
 
 # Pecan Application Configurations
 # See https://pecan.readthedocs.org/en/latest/configuration.html#application-configuration # noqa
+acl_public_routes = ['/']
+if not cfg.CONF.api.get("enable_webhooks_auth"):
+    acl_public_routes.append('/v1/webhooks/.*')
+
 app = {
     'root': 'watcher.api.controllers.root.RootController',
     'modules': ['watcher.api'],
@@ -36,9 +38,7 @@ app = {
     ],
     'static_root': '%(confdir)s/public',
     'enable_acl': True,
-    'acl_public_routes': [
-        '/',
-    ],
+    'acl_public_routes': acl_public_routes,
 }
 
 # WSME Configurations

watcher/api/controllers/link.py

@@ -23,7 +23,7 @@ from watcher.api.controllers import base
 
 def build_url(resource, resource_args, bookmark=False, base_url=None):
     if base_url is None:
-        base_url = pecan.request.host_url
+        base_url = pecan.request.application_url
 
     template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s'
     # FIXME(lucasagomes): I'm getting a 404 when doing a GET on

watcher/api/controllers/rest_api_version_history.rst

@@ -30,3 +30,12 @@ audits.
 ---
 Added ``force`` into create audit request. If ``force`` is true,
 audit will be executed despite of ongoing actionplan.
+
+1.3
+---
+Added list data model API.
+
+1.4
+---
+Added Watcher webhook API. It can be used to trigger audit
+with ``event`` type.

watcher/api/controllers/root.py

@@ -59,7 +59,8 @@ class Version(base.APIBase):
         version.status = status
         version.max_version = v.max_version_string()
         version.min_version = v.min_version_string()
-        version.links = [link.Link.make_link('self', pecan.request.host_url,
+        version.links = [link.Link.make_link('self',
+                                             pecan.request.application_url,
                                              id, '', bookmark=True)]
         return version
 

watcher/api/controllers/v1/__init__.py

@@ -40,7 +40,9 @@ from watcher.api.controllers.v1 import goal
 from watcher.api.controllers.v1 import scoring_engine
 from watcher.api.controllers.v1 import service
 from watcher.api.controllers.v1 import strategy
+from watcher.api.controllers.v1 import utils
 from watcher.api.controllers.v1 import versions
+from watcher.api.controllers.v1 import webhooks
 
 
 def min_version():
@@ -130,6 +132,9 @@ class V1(APIBase):
     services = [link.Link]
     """Links to the services resource"""
 
+    webhooks = [link.Link]
+    """Links to the webhooks resource"""
+
     links = [link.Link]
     """Links that point to a specific URL for this version and documentation"""
 
@@ -137,7 +142,8 @@ class V1(APIBase):
     def convert():
         v1 = V1()
         v1.id = "v1"
-        v1.links = [link.Link.make_link('self', pecan.request.host_url,
+        base_url = pecan.request.application_url
+        v1.links = [link.Link.make_link('self', base_url,
                                         'v1', '', bookmark=True),
                     link.Link.make_link('describedby',
                                         'http://docs.openstack.org',
@@ -148,57 +154,66 @@ class V1(APIBase):
         v1.media_types = [MediaType('application/json',
                           'application/vnd.openstack.watcher.v1+json')]
         v1.audit_templates = [link.Link.make_link('self',
-                                                  pecan.request.host_url,
+                                                  base_url,
                                                   'audit_templates', ''),
                               link.Link.make_link('bookmark',
-                                                  pecan.request.host_url,
+                                                  base_url,
                                                   'audit_templates', '',
                                                   bookmark=True)
                               ]
-        v1.audits = [link.Link.make_link('self', pecan.request.host_url,
+        v1.audits = [link.Link.make_link('self', base_url,
                                          'audits', ''),
                      link.Link.make_link('bookmark',
-                                         pecan.request.host_url,
+                                         base_url,
                                          'audits', '',
                                          bookmark=True)
                      ]
-        v1.data_model = [link.Link.make_link('self', pecan.request.host_url,
-                                             'data_model', ''),
-                         link.Link.make_link('bookmark',
-                                             pecan.request.host_url,
-                                             'data_model', '',
-                                             bookmark=True)
-                         ]
-        v1.actions = [link.Link.make_link('self', pecan.request.host_url,
+        if utils.allow_list_datamodel():
+            v1.data_model = [link.Link.make_link('self', base_url,
+                                                 'data_model', ''),
+                             link.Link.make_link('bookmark',
+                                                 base_url,
+                                                 'data_model', '',
+                                                 bookmark=True)
+                             ]
+        v1.actions = [link.Link.make_link('self', base_url,
                                           'actions', ''),
                       link.Link.make_link('bookmark',
-                                          pecan.request.host_url,
+                                          base_url,
                                           'actions', '',
                                           bookmark=True)
                       ]
         v1.action_plans = [link.Link.make_link(
-            'self', pecan.request.host_url, 'action_plans', ''),
+            'self', base_url, 'action_plans', ''),
             link.Link.make_link('bookmark',
-                                pecan.request.host_url,
+                                base_url,
                                 'action_plans', '',
                                 bookmark=True)
             ]
 
         v1.scoring_engines = [link.Link.make_link(
-            'self', pecan.request.host_url, 'scoring_engines', ''),
+            'self', base_url, 'scoring_engines', ''),
             link.Link.make_link('bookmark',
-                                pecan.request.host_url,
+                                base_url,
                                 'scoring_engines', '',
                                 bookmark=True)
             ]
 
         v1.services = [link.Link.make_link(
-            'self', pecan.request.host_url, 'services', ''),
+            'self', base_url, 'services', ''),
             link.Link.make_link('bookmark',
-                                pecan.request.host_url,
+                                base_url,
                                 'services', '',
                                 bookmark=True)
             ]
+        if utils.allow_webhook_api():
+            v1.webhooks = [link.Link.make_link(
+                'self', base_url, 'webhooks', ''),
+                link.Link.make_link('bookmark',
+                                    base_url,
+                                    'webhooks', '',
+                                    bookmark=True)
+                ]
         return v1
 
 
@@ -214,6 +229,7 @@ class Controller(rest.RestController):
     services = service.ServicesController()
     strategies = strategy.StrategiesController()
     data_model = data_model.DataModelController()
+    webhooks = webhooks.WebhookController()
 
     @wsme_pecan.wsexpose(V1)
     def get(self):

watcher/api/controllers/v1/action_plan.py

@@ -165,7 +165,7 @@ class ActionPlan(base.APIBase):
                         name=indicator.name,
                         description=indicator.description,
                         unit=indicator.unit,
-                        value=indicator.value,
+                        value=float(indicator.value),
                     )
                     efficacy_indicators.append(efficacy_indicator.as_dict())
                 self._efficacy_indicators = efficacy_indicators

watcher/api/controllers/v1/data_model.py

@@ -24,6 +24,7 @@ from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
 from watcher.api.controllers.v1 import types
+from watcher.api.controllers.v1 import utils
 from watcher.common import exception
 from watcher.common import policy
 from watcher.decision_engine import rpcapi
@@ -49,6 +50,8 @@ class DataModelController(rest.RestController):
         :param audit_uuid: The UUID of the audit,  used to filter data model
                            by the scope in audit.
         """
+        if not utils.allow_list_datamodel():
+            raise exception.NotAcceptable
         if self.from_data_model:
             raise exception.OperationNotPermitted
         allowed_data_model_type = [

watcher/api/controllers/v1/service.py

@@ -19,8 +19,6 @@ Service mechanism provides ability to monitor Watcher services state.
 """
 
 import datetime
-import six
-
 from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import timeutils
@@ -70,7 +68,7 @@ class Service(base.APIBase):
         service = objects.Service.get(pecan.request.context, id)
         last_heartbeat = (service.last_seen_up or service.updated_at or
                           service.created_at)
-        if isinstance(last_heartbeat, six.string_types):
+        if isinstance(last_heartbeat, str):
             # NOTE(russellb) If this service came in over rpc via
             # conductor, then the timestamp will be a string and needs to be
             # converted back to a datetime.

watcher/api/controllers/v1/types.py

@@ -15,7 +15,6 @@
 
 from oslo_serialization import jsonutils
 from oslo_utils import strutils
-import six
 import wsme
 from wsme import types as wtypes
 
@@ -132,7 +131,7 @@ class JsonType(wtypes.UserType):
 
     def __str__(self):
         # These are the json serializable native types
-        return ' | '.join(map(str, (wtypes.text, six.integer_types, float,
+        return ' | '.join(map(str, (wtypes.text, int, float,
                                     BooleanType, list, dict, None)))
 
     @staticmethod
@@ -184,7 +183,7 @@ class MultiType(wtypes.UserType):
 class JsonPatchType(wtypes.Base):
     """A complex type that represents a single json-patch operation."""
 
-    path = wtypes.wsattr(wtypes.StringType(pattern='^(/[\w-]+)+$'),
+    path = wtypes.wsattr(wtypes.StringType(pattern=r'^(/[\w-]+)+$'),
                          mandatory=True)
     op = wtypes.wsattr(wtypes.Enum(str, 'add', 'replace', 'remove'),
                        mandatory=True)

watcher/api/controllers/v1/utils.py

@@ -164,7 +164,8 @@ def allow_start_end_audit_time():
     Version 1.1 of the API added support for start and end time of continuous
     audits.
     """
-    return pecan.request.version.minor >= versions.MINOR_1_START_END_TIMING
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_1_START_END_TIMING.value)
 
 
 def allow_force():
@@ -173,4 +174,23 @@ def allow_force():
     Version 1.2 of the API added support for forced audits that allows to
     launch audit when other action plan is ongoing.
     """
-    return pecan.request.version.minor >= versions.MINOR_2_FORCE
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_2_FORCE.value)
+
+
+def allow_list_datamodel():
+    """Check if we should support list data model API.
+
+    Version 1.3 of the API added support to list data model.
+    """
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_3_DATAMODEL.value)
+
+
+def allow_webhook_api():
+    """Check if we should support webhook API.
+
+    Version 1.4 of the API added support to trigger webhook.
+    """
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_4_WEBHOOK_API.value)

watcher/api/controllers/v1/versions.py

@@ -14,25 +14,25 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import enum
+
+
+class VERSIONS(enum.Enum):
+    MINOR_0_ROCKY = 0  # v1.0: corresponds to Rocky API
+    MINOR_1_START_END_TIMING = 1  # v1.1: Add start/end timei for audit
+    MINOR_2_FORCE = 2  # v1.2: Add force field to audit
+    MINOR_3_DATAMODEL = 3  # v1.3: Add list datamodel API
+    MINOR_4_WEBHOOK_API = 4  # v1.4: Add webhook trigger API
+    MINOR_MAX_VERSION = 4
+
 
 # This is the version 1 API
 BASE_VERSION = 1
-
-# Here goes a short log of changes in every version.
-#
-# v1.0: corresponds to Rocky API
-# v1.1: Add start/end time for continuous audit
-# v1.2: Add force field to audit
-
-MINOR_0_ROCKY = 0
-MINOR_1_START_END_TIMING = 1
-MINOR_2_FORCE = 2
-
-MINOR_MAX_VERSION = MINOR_2_FORCE
-
 # String representations of the minor and maximum versions
-_MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_0_ROCKY)
-_MAX_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_MAX_VERSION)
+_MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION,
+                                     VERSIONS.MINOR_0_ROCKY.value)
+_MAX_VERSION_STRING = '{}.{}'.format(BASE_VERSION,
+                                     VERSIONS.MINOR_MAX_VERSION.value)
 
 
 def service_type_string():

watcher/api/controllers/v1/webhooks.py

@@ -0,0 +1,62 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Webhook endpoint for Watcher v1 REST API.
+"""
+
+from oslo_log import log
+import pecan
+from pecan import rest
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from watcher.api.controllers.v1 import types
+from watcher.api.controllers.v1 import utils
+from watcher.common import exception
+from watcher.decision_engine import rpcapi
+from watcher import objects
+
+LOG = log.getLogger(__name__)
+
+
+class WebhookController(rest.RestController):
+    """REST controller for webhooks resource."""
+    def __init__(self):
+        super(WebhookController, self).__init__()
+        self.dc_client = rpcapi.DecisionEngineAPI()
+
+    @wsme_pecan.wsexpose(None, wtypes.text, body=types.jsontype,
+                         status_code=202)
+    def post(self, audit_ident, body):
+        """Trigger the given audit.
+
+        :param audit_ident: UUID or name of an audit.
+        """
+
+        LOG.debug("Webhook trigger Audit: %s.", audit_ident)
+
+        context = pecan.request.context
+        audit = utils.get_resource('Audit', audit_ident)
+        if audit is None:
+            raise exception.AuditNotFound(audit=audit_ident)
+        if audit.audit_type != objects.audit.AuditType.EVENT.value:
+            raise exception.AuditTypeNotAllowed(audit_type=audit.audit_type)
+        allowed_state = (
+            objects.audit.State.PENDING,
+            objects.audit.State.SUCCEEDED,
+            )
+        if audit.state not in allowed_state:
+            raise exception.AuditStateNotAllowed(state=audit.state)
+
+        # trigger decision-engine to run the audit
+        self.dc_client.trigger_audit(context, audit.uuid)

watcher/api/hooks.py

@@ -15,9 +15,9 @@
 # under the License.
 
 
+from http import client as http_client
 from oslo_config import cfg
 from pecan import hooks
-from six.moves import http_client
 
 from watcher.common import context
 

watcher/api/middleware/auth_token.py

@@ -34,7 +34,7 @@ class AuthTokenMiddleware(auth_token.AuthProtocol):
 
     """
     def __init__(self, app, conf, public_api_routes=()):
-        route_pattern_tpl = '%s(\.json|\.xml)?$'
+        route_pattern_tpl = r'%s(\.json|\.xml)?$'
 
         try:
             self.public_api_routes = [re.compile(route_pattern_tpl % route_tpl)

watcher/api/middleware/parsable_error.py

@@ -24,7 +24,6 @@ from xml import etree as et
 
 from oslo_log import log
 from oslo_serialization import jsonutils
-import six
 import webob
 
 from watcher._i18n import _
@@ -84,12 +83,10 @@ class ParsableErrorMiddleware(object):
                             '</error_message>' % state['status_code']]
                 state['headers'].append(('Content-Type', 'application/xml'))
             else:
-                if six.PY3:
-                    app_iter = [i.decode('utf-8') for i in app_iter]
+                app_iter = [i.decode('utf-8') for i in app_iter]
                 body = [jsonutils.dumps(
                     {'error_message': '\n'.join(app_iter)})]
-                if six.PY3:
-                    body = [item.encode('utf-8') for item in body]
+                body = [item.encode('utf-8') for item in body]
                 state['headers'].append(('Content-Type', 'application/json'))
             state['headers'].append(('Content-Length', str(len(body[0]))))
         else:

watcher/api/scheduling.py

@@ -20,7 +20,6 @@ import itertools
 from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import timeutils
-import six
 
 from watcher.common import context as watcher_context
 from watcher.common import scheduling
@@ -83,7 +82,7 @@ class APISchedulingService(scheduling.BackgroundSchedulerService):
         service = objects.Service.get(context, service_id)
         last_heartbeat = (service.last_seen_up or service.updated_at or
                           service.created_at)
-        if isinstance(last_heartbeat, six.string_types):
+        if isinstance(last_heartbeat, str):
             # NOTE(russellb) If this service came in over rpc via
             # conductor, then the timestamp will be a string and needs to be
             # converted back to a datetime.

watcher/applier/action_plan/base.py

@@ -18,11 +18,9 @@
 #
 
 import abc
-import six
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BaseActionPlanHandler(object):
+class BaseActionPlanHandler(object, metaclass=abc.ABCMeta):
     @abc.abstractmethod
     def execute(self):
         raise NotImplementedError()

watcher/applier/actions/base.py

@@ -19,14 +19,12 @@
 import abc
 
 import jsonschema
-import six
 
 from watcher.common import clients
 from watcher.common.loader import loadable
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BaseAction(loadable.Loadable):
+class BaseAction(loadable.Loadable, metaclass=abc.ABCMeta):
     # NOTE(jed): by convention we decided
     # that the attribute "resource_id" is the unique id of
     # the resource to which the Action applies to allow us to use it in the
@@ -140,7 +138,7 @@ class BaseAction(loadable.Loadable):
         raise NotImplementedError()
 
     def check_abort(self):
-        if self.__class__.__name__ is 'Migrate':
+        if self.__class__.__name__ == 'Migrate':
             if self.migration_type == self.LIVE_MIGRATION:
                 return True
             else:

watcher/applier/actions/factory.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import unicode_literals
-
 from oslo_log import log
 
 from watcher.applier.loading import default

watcher/applier/actions/migration.py

@@ -186,7 +186,7 @@ class Migrate(base.BaseAction):
         return self.migrate(destination=self.destination_node)
 
     def revert(self):
-        LOG.info('Migrate action do not revert!')
+        return self.migrate(destination=self.source_node)
 
     def abort(self):
         nova = nova_helper.NovaHelper(osc=self.osc)

watcher/applier/actions/resize.py

@@ -47,24 +47,24 @@ class Resize(base.BaseAction):
 
     @property
     def schema(self):
-            return {
-                'type': 'object',
-                'properties': {
-                    'resource_id': {
-                        'type': 'string',
-                        'minlength': 1,
-                        'pattern': ('^([a-fA-F0-9]){8}-([a-fA-F0-9]){4}-'
-                                    '([a-fA-F0-9]){4}-([a-fA-F0-9]){4}-'
-                                    '([a-fA-F0-9]){12}$')
-                    },
-                    'flavor': {
-                        'type': 'string',
-                        'minlength': 1,
-                    },
+        return {
+            'type': 'object',
+            'properties': {
+                'resource_id': {
+                    'type': 'string',
+                    'minlength': 1,
+                    'pattern': ('^([a-fA-F0-9]){8}-([a-fA-F0-9]){4}-'
+                                '([a-fA-F0-9]){4}-([a-fA-F0-9]){4}-'
+                                '([a-fA-F0-9]){12}$')
                 },
-                'required': ['resource_id', 'flavor'],
-                'additionalProperties': False,
-            }
+                'flavor': {
+                    'type': 'string',
+                    'minlength': 1,
+                },
+            },
+            'required': ['resource_id', 'flavor'],
+            'additionalProperties': False,
+        }
 
     @property
     def instance_uuid(self):
@@ -95,7 +95,7 @@ class Resize(base.BaseAction):
         return self.resize()
 
     def revert(self):
-        return self.migrate(destination=self.source_node)
+        LOG.warning("revert not supported")
 
     def pre_condition(self):
         # TODO(jed): check if the instance exists / check if the instance is on

watcher/applier/base.py

@@ -26,11 +26,9 @@ See: :doc:`../architecture` for more details on this component.
 """
 
 import abc
-import six
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BaseApplier(object):
+class BaseApplier(object, metaclass=abc.ABCMeta):
     @abc.abstractmethod
     def execute(self, action_plan_uuid):
         raise NotImplementedError()

watcher/applier/loading/default.py

@@ -11,9 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import unicode_literals
-
-
 from watcher.common.loader import default
 
 

watcher/applier/workflow_engine/base.py

@@ -17,7 +17,6 @@
 #
 
 import abc
-import six
 import time
 
 import eventlet
@@ -40,8 +39,7 @@ CANCEL_STATE = [objects.action_plan.State.CANCELLING,
                 objects.action_plan.State.CANCELLED]
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BaseWorkFlowEngine(loadable.Loadable):
+class BaseWorkFlowEngine(loadable.Loadable, metaclass=abc.ABCMeta):
 
     def __init__(self, config, context=None, applier_manager=None):
         """Constructor

watcher/applier/workflow_engine/default.py

@@ -25,8 +25,11 @@ from taskflow import task as flow_task
 
 from watcher.applier.workflow_engine import base
 from watcher.common import exception
+from watcher import conf
 from watcher import objects
 
+CONF = conf.CONF
+
 LOG = log.getLogger(__name__)
 
 
@@ -112,7 +115,7 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
 
             return flow
 
-        except exception.ActionPlanCancelled as e:
+        except exception.ActionPlanCancelled:
             raise
 
         except tf_exception.WrappedFailure as e:
@@ -127,9 +130,11 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
 
 class TaskFlowActionContainer(base.BaseTaskFlowActionContainer):
     def __init__(self, db_action, engine):
-        name = "action_type:{0} uuid:{1}".format(db_action.action_type,
-                                                 db_action.uuid)
-        super(TaskFlowActionContainer, self).__init__(name, db_action, engine)
+        self.name = "action_type:{0} uuid:{1}".format(db_action.action_type,
+                                                      db_action.uuid)
+        super(TaskFlowActionContainer, self).__init__(self.name,
+                                                      db_action,
+                                                      engine)
 
     def do_pre_execute(self):
         db_action = self.engine.notify(self._db_action,
@@ -158,6 +163,12 @@ class TaskFlowActionContainer(base.BaseTaskFlowActionContainer):
         self.action.post_condition()
 
     def do_revert(self, *args, **kwargs):
+        # NOTE: Not rollback action plan
+        if not CONF.watcher_applier.rollback_when_actionplan_failed:
+            LOG.info("Failed actionplan rollback option is turned off, and "
+                     "the following action will be skipped: %s", self.name)
+            return
+
         LOG.warning("Revert action: %s", self.name)
         try:
             # TODO(jed): do we need to update the states in case of failure?

watcher/cmd/__init__.py

@@ -18,3 +18,10 @@
 import eventlet
 
 eventlet.monkey_patch()
+
+# Monkey patch the original current_thread to use the up-to-date _active
+# global variable. See https://bugs.launchpad.net/bugs/1863021 and
+# https://github.com/eventlet/eventlet/issues/592
+import __original_module_threading as orig_threading  # noqa
+import threading  # noqa
+orig_threading.current_thread.__globals__['_active'] = threading._active

watcher/cmd/status.py

@@ -14,8 +14,8 @@
 
 import sys
 
+from oslo_upgradecheck import common_checks
 from oslo_upgradecheck import upgradecheck
-import six
 
 from watcher._i18n import _
 from watcher.common import clients
@@ -38,12 +38,16 @@ class Checks(upgradecheck.UpgradeCommands):
             clients.check_min_nova_api_version(CONF.nova_client.api_version)
         except ValueError as e:
             return upgradecheck.Result(
-                upgradecheck.Code.FAILURE, six.text_type(e))
+                upgradecheck.Code.FAILURE, str(e))
         return upgradecheck.Result(upgradecheck.Code.SUCCESS)
 
     _upgrade_checks = (
         # Added in Train.
         (_('Minimum Nova API Version'), _minimum_nova_api_version),
+        # Added in Wallaby.
+        (_("Policy File JSON to YAML Migration"),
+         (common_checks.check_policy_json, {'conf': CONF})),
+
     )
 
 

watcher/common/context.py

@@ -13,7 +13,6 @@
 from oslo_context import context
 from oslo_log import log
 from oslo_utils import timeutils
-import six
 
 LOG = log.getLogger(__name__)
 
@@ -69,7 +68,7 @@ class RequestContext(context.RequestContext):
         self.project_id = project_id
         if not timestamp:
             timestamp = timeutils.utcnow()
-        if isinstance(timestamp, six.string_types):
+        if isinstance(timestamp, str):
             timestamp = timeutils.parse_isotime(timestamp)
         self.timestamp = timestamp
         self.user_name = user_name

watcher/common/exception.py

@@ -28,7 +28,6 @@ import sys
 from keystoneclient import exceptions as keystone_exceptions
 from oslo_config import cfg
 from oslo_log import log
-import six
 
 from watcher._i18n import _
 
@@ -97,19 +96,16 @@ class WatcherException(Exception):
 
     def __str__(self):
         """Encode to utf-8 then wsme api can consume it as well"""
-        if not six.PY3:
-            return six.text_type(self.args[0]).encode('utf-8')
-        else:
-            return self.args[0]
+        return self.args[0]
 
     def __unicode__(self):
-        return six.text_type(self.args[0])
+        return str(self.args[0])
 
     def format_message(self):
         if self.__class__.__name__.endswith('_Remote'):
             return self.args[0]
         else:
-            return six.text_type(self)
+            return str(self)
 
 
 class UnsupportedError(WatcherException):
@@ -243,6 +239,14 @@ class AuditTypeNotFound(Invalid):
     msg_fmt = _("Audit type %(audit_type)s could not be found")
 
 
+class AuditTypeNotAllowed(Invalid):
+    msg_fmt = _("Audit type %(audit_type)s is disallowed.")
+
+
+class AuditStateNotAllowed(Invalid):
+    msg_fmt = _("Audit state %(state)s is disallowed.")
+
+
 class AuditParameterNotAllowed(Invalid):
     msg_fmt = _("Audit parameter %(parameter)s are not allowed")
 

watcher/common/loader/base.py

@@ -14,14 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import unicode_literals
-
 import abc
-import six
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BaseLoader(object):
+class BaseLoader(object, metaclass=abc.ABCMeta):
 
     @abc.abstractmethod
     def list_available(self):

watcher/common/loader/default.py

@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import unicode_literals
-
 from oslo_config import cfg
 from oslo_log import log
 from stevedore import driver as drivermanager

watcher/common/loader/loadable.py

@@ -16,13 +16,10 @@
 
 import abc
 
-import six
-
 from watcher.common import service
 
 
-@six.add_metaclass(abc.ABCMeta)
-class Loadable(object):
+class Loadable(object, metaclass=abc.ABCMeta):
     """Generic interface for dynamically loading a driver/entry point.
 
     This defines the contract in order to let the loader manager inject
@@ -48,8 +45,7 @@ LoadableSingletonMeta = type(
     "LoadableSingletonMeta", (abc.ABCMeta, service.Singleton), {})
 
 
-@six.add_metaclass(LoadableSingletonMeta)
-class LoadableSingleton(object):
+class LoadableSingleton(object, metaclass=LoadableSingletonMeta):
     """Generic interface for dynamically loading a driver as a singleton.
 
     This defines the contract in order to let the loader manager inject

watcher/common/policy.py

@@ -18,6 +18,7 @@
 import sys
 
 from oslo_config import cfg
+from oslo_policy import opts
 from oslo_policy import policy
 
 from watcher.common import exception
@@ -26,6 +27,12 @@ from watcher.common import policies
 _ENFORCER = None
 CONF = cfg.CONF
 
+# TODO(gmann): Remove setting the default value of config policy_file
+# once oslo_policy change the default value to 'policy.yaml'.
+# https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49
+DEFAULT_POLICY_FILE = 'policy.yaml'
+opts.set_defaults(CONF, DEFAULT_POLICY_FILE)
+
 
 # we can get a policy enforcer by this init.
 # oslo policy support change policy rule dynamically.

watcher/common/rpc.py

@@ -121,22 +121,40 @@ class RequestContextSerializer(messaging.Serializer):
 def get_client(target, version_cap=None, serializer=None):
     assert TRANSPORT is not None
     serializer = RequestContextSerializer(serializer)
-    return messaging.RPCClient(TRANSPORT,
-                               target,
-                               version_cap=version_cap,
-                               serializer=serializer)
+    return messaging.RPCClient(
+        TRANSPORT,
+        target,
+        version_cap=version_cap,
+        serializer=serializer
+    )
 
 
 def get_server(target, endpoints, serializer=None):
     assert TRANSPORT is not None
     access_policy = dispatcher.DefaultRPCAccessPolicy
     serializer = RequestContextSerializer(serializer)
-    return messaging.get_rpc_server(TRANSPORT,
-                                    target,
-                                    endpoints,
-                                    executor='eventlet',
-                                    serializer=serializer,
-                                    access_policy=access_policy)
+    return messaging.get_rpc_server(
+        TRANSPORT,
+        target,
+        endpoints,
+        executor='eventlet',
+        serializer=serializer,
+        access_policy=access_policy
+    )
+
+
+def get_notification_listener(targets, endpoints, serializer=None, pool=None):
+    assert NOTIFICATION_TRANSPORT is not None
+    serializer = RequestContextSerializer(serializer)
+    return messaging.get_notification_listener(
+        NOTIFICATION_TRANSPORT,
+        targets,
+        endpoints,
+        allow_requeue=False,
+        executor='eventlet',
+        pool=pool,
+        serializer=serializer
+    )
 
 
 def get_notifier(publisher_id):

watcher/common/scheduling.py

@@ -37,6 +37,7 @@ class GreenThreadPoolExecutor(BasePoolExecutor):
         pool = futurist.GreenThreadPoolExecutor(int(max_workers))
         super(GreenThreadPoolExecutor, self).__init__(pool)
 
+
 executors = {
     'default': GreenThreadPoolExecutor(),
 }

watcher/common/service.py

@@ -21,14 +21,12 @@ from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import _options
 from oslo_log import log
-import oslo_messaging as om
+import oslo_messaging as messaging
 from oslo_reports import guru_meditation_report as gmr
 from oslo_reports import opts as gmr_opts
 from oslo_service import service
 from oslo_service import wsgi
 
-from oslo_messaging.rpc import dispatcher
-
 from watcher._i18n import _
 from watcher.api import app
 from watcher.common import config
@@ -183,11 +181,6 @@ class Service(service.ServiceBase):
         ]
         self.notification_endpoints = self.manager.notification_endpoints
 
-        self.serializer = rpc.RequestContextSerializer(
-            base.WatcherObjectSerializer())
-
-        self._transport = None
-        self._notification_transport = None
         self._conductor_client = None
 
         self.conductor_topic_handler = None
@@ -201,27 +194,17 @@ class Service(service.ServiceBase):
                 self.notification_topics, self.notification_endpoints
             )
 
-    @property
-    def transport(self):
-        if self._transport is None:
-            self._transport = om.get_rpc_transport(CONF)
-        return self._transport
-
-    @property
-    def notification_transport(self):
-        if self._notification_transport is None:
-            self._notification_transport = om.get_notification_transport(CONF)
-        return self._notification_transport
-
     @property
     def conductor_client(self):
         if self._conductor_client is None:
-            target = om.Target(
+            target = messaging.Target(
                 topic=self.conductor_topic,
                 version=self.API_VERSION,
             )
-            self._conductor_client = om.RPCClient(
-                self.transport, target, serializer=self.serializer)
+            self._conductor_client = rpc.get_client(
+                target,
+                serializer=base.WatcherObjectSerializer()
+            )
         return self._conductor_client
 
     @conductor_client.setter
@@ -229,21 +212,18 @@ class Service(service.ServiceBase):
         self.conductor_client = c
 
     def build_topic_handler(self, topic_name, endpoints=()):
-        access_policy = dispatcher.DefaultRPCAccessPolicy
-        serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
-        target = om.Target(
+        target = messaging.Target(
             topic=topic_name,
             # For compatibility, we can override it with 'host' opt
             server=CONF.host or socket.gethostname(),
             version=self.api_version,
         )
-        return om.get_rpc_server(
-            self.transport, target, endpoints,
-            executor='eventlet', serializer=serializer,
-            access_policy=access_policy)
+        return rpc.get_server(
+            target, endpoints,
+            serializer=rpc.JsonPayloadSerializer()
+        )
 
     def build_notification_handler(self, topic_names, endpoints=()):
-        serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
         targets = []
         for topic in topic_names:
             kwargs = {}
@@ -251,11 +231,13 @@ class Service(service.ServiceBase):
                 exchange, topic = topic.split('.')
                 kwargs['exchange'] = exchange
             kwargs['topic'] = topic
-            targets.append(om.Target(**kwargs))
-        return om.get_notification_listener(
-            self.notification_transport, targets, endpoints,
-            executor='eventlet', serializer=serializer,
-            allow_requeue=False, pool=CONF.host)
+            targets.append(messaging.Target(**kwargs))
+
+        return rpc.get_notification_listener(
+            targets, endpoints,
+            serializer=rpc.JsonPayloadSerializer(),
+            pool=CONF.host
+        )
 
     def start(self):
         LOG.debug("Connecting to '%s'", CONF.transport_url)

watcher/common/service_manager.py

@@ -15,11 +15,9 @@
 # under the License.
 
 import abc
-import six
 
 
-@six.add_metaclass(abc.ABCMeta)
-class ServiceManager(object):
+class ServiceManager(object, metaclass=abc.ABCMeta):
 
     @abc.abstractproperty
     def service_name(self):

watcher/common/utils.py

@@ -28,7 +28,6 @@ from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import strutils
 from oslo_utils import uuidutils
-import six
 
 from watcher.common import exception
 
@@ -82,7 +81,7 @@ def safe_rstrip(value, chars=None):
     :return: Stripped value.
 
     """
-    if not isinstance(value, six.string_types):
+    if not isinstance(value, str):
         LOG.warning(
             "Failed to remove trailing character. Returning original object."
             "Supplied object is not a string: %s,", value)
@@ -104,7 +103,7 @@ def is_hostname_safe(hostname):
 
     """
     m = r'^[a-z0-9]([a-z0-9\-]{0,61}[a-z0-9])?$'
-    return (isinstance(hostname, six.string_types) and
+    return (isinstance(hostname, str) and
             (re.match(m, hostname) is not None))
 
 
@@ -153,6 +152,7 @@ def extend_with_strict_schema(validator_class):
 
     return validators.extend(validator_class, {"properties": strict_schema})
 
+
 StrictDefaultValidatingDraft4Validator = extend_with_default(
     extend_with_strict_schema(validators.Draft4Validator))
 

watcher/conf/api.py

@@ -55,6 +55,11 @@ API_SERVICE_OPTS = [
                      "the service, this option should be False; note, you "
                      "will want to change public API endpoint to represent "
                      "SSL termination URL with 'public_endpoint' option."),
+
+    cfg.BoolOpt('enable_webhooks_auth',
+                default=True,
+                help='This option enables or disables webhook request '
+                     'authentication via keystone. Default value is True.'),
 ]
 
 

watcher/conf/applier.py

@@ -43,11 +43,20 @@ APPLIER_MANAGER_OPTS = [
                help='Select the engine to use to execute the workflow'),
 ]
 
+APPLIER_OPTS = [
+    cfg.BoolOpt('rollback_when_actionplan_failed',
+                default=False,
+                help='If set True, the failed actionplan will rollback '
+                     'when executing. Defaule value is False.'),
+]
+
 
 def register_opts(conf):
     conf.register_group(watcher_applier)
     conf.register_opts(APPLIER_MANAGER_OPTS, group=watcher_applier)
+    conf.register_opts(APPLIER_OPTS, group=watcher_applier)
 
 
 def list_opts():
-    return [(watcher_applier, APPLIER_MANAGER_OPTS)]
+    return [(watcher_applier, APPLIER_MANAGER_OPTS),
+            (watcher_applier, APPLIER_OPTS)]

watcher/conf/decision_engine.py

@@ -40,11 +40,18 @@ WATCHER_DECISION_ENGINE_OPTS = [
                default='watcher.decision.api',
                help='The identifier used by the Watcher '
                     'module on the message broker'),
-    cfg.IntOpt('max_workers',
+    cfg.IntOpt('max_audit_workers',
                default=2,
                required=True,
                help='The maximum number of threads that can be used to '
-                    'execute strategies'),
+                    'execute audits in parallel.'),
+    cfg.IntOpt('max_general_workers',
+               default=4,
+               required=True,
+               help='The maximum number of threads that can be used to '
+                    'execute general tasks in parallel. The number of general '
+                    'workers will not increase depending on the number of '
+                    'audit workers!'),
     cfg.IntOpt('action_plan_expiry',
                default=24,
                mutable=True,

watcher/db/api.py

@@ -18,7 +18,6 @@ Base classes for storage engines
 import abc
 from oslo_config import cfg
 from oslo_db import api as db_api
-import six
 
 _BACKEND_MAPPING = {'sqlalchemy': 'watcher.db.sqlalchemy.api'}
 IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING,
@@ -30,8 +29,7 @@ def get_instance():
     return IMPL
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BaseConnection(object):
+class BaseConnection(object, metaclass=abc.ABCMeta):
     """Base class for storage system connections."""
 
     @abc.abstractmethod

watcher/db/purge.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-from __future__ import print_function
-
 import collections
 import datetime
 import itertools
@@ -25,7 +23,6 @@ import sys
 from oslo_log import log
 from oslo_utils import strutils
 import prettytable as ptable
-from six.moves import input
 
 from watcher._i18n import _
 from watcher._i18n import lazy_translation_enabled

watcher/db/sqlalchemy/api.py

@@ -278,7 +278,7 @@ class Connection(api.BaseConnection):
             query = model_query(model, session=session)
             query = add_identity_filter(query, id_)
             try:
-                ref = query.with_lockmode('update').one()
+                ref = query.with_for_update().one()
             except exc.NoResultFound:
                 raise exception.ResourceNotFound(name=model.__name__, id=id_)
 
@@ -815,7 +815,7 @@ class Connection(api.BaseConnection):
             query = model_query(models.Action, session=session)
             query = add_identity_filter(query, action_id)
             try:
-                ref = query.with_lockmode('update').one()
+                ref = query.with_for_update().one()
             except exc.NoResultFound:
                 raise exception.ActionNotFound(action=action_id)
 
@@ -900,7 +900,7 @@ class Connection(api.BaseConnection):
             query = model_query(models.ActionPlan, session=session)
             query = add_identity_filter(query, action_plan_id)
             try:
-                ref = query.with_lockmode('update').one()
+                ref = query.with_for_update().one()
             except exc.NoResultFound:
                 raise exception.ActionPlanNotFound(action_plan=action_plan_id)
 
@@ -1125,8 +1125,8 @@ class Connection(api.BaseConnection):
 
     def get_action_description_by_id(self, context,
                                      action_id, eager=False):
-                return self._get_action_description(
-                    context, fieldname="id", value=action_id, eager=eager)
+        return self._get_action_description(
+            context, fieldname="id", value=action_id, eager=eager)
 
     def get_action_description_by_type(self, context,
                                        action_type, eager=False):

watcher/db/sqlalchemy/models.py

@@ -18,7 +18,6 @@ SQLAlchemy models for watcher service
 
 from oslo_db.sqlalchemy import models
 from oslo_serialization import jsonutils
-import six.moves.urllib.parse as urlparse
 from sqlalchemy import Boolean
 from sqlalchemy import Column
 from sqlalchemy import DateTime
@@ -33,7 +32,7 @@ from sqlalchemy import String
 from sqlalchemy import Text
 from sqlalchemy.types import TypeDecorator, TEXT
 from sqlalchemy import UniqueConstraint
-
+import urllib.parse as urlparse
 from watcher import conf
 
 CONF = conf.CONF

watcher/decision_engine/audit/base.py

@@ -18,7 +18,6 @@
 # limitations under the License.
 #
 import abc
-import six
 
 from oslo_config import cfg
 from oslo_log import log
@@ -36,9 +35,11 @@ CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
 
-@six.add_metaclass(abc.ABCMeta)
-@six.add_metaclass(service.Singleton)
-class BaseAuditHandler(object):
+class BaseMetaClass(service.Singleton, abc.ABCMeta):
+    pass
+
+
+class BaseAuditHandler(object, metaclass=BaseMetaClass):
 
     @abc.abstractmethod
     def execute(self, audit, request_context):
@@ -57,8 +58,7 @@ class BaseAuditHandler(object):
         raise NotImplementedError()
 
 
-@six.add_metaclass(abc.ABCMeta)
-class AuditHandler(BaseAuditHandler):
+class AuditHandler(BaseAuditHandler, metaclass=abc.ABCMeta):
 
     def __init__(self):
         super(AuditHandler, self).__init__()

watcher/decision_engine/audit/event.py

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 ZTE Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from watcher.decision_engine.audit import base
+from watcher import objects
+
+
+class EventAuditHandler(base.AuditHandler):
+
+    def post_execute(self, audit, solution, request_context):
+        super(EventAuditHandler, self).post_execute(audit, solution,
+                                                    request_context)
+        # change state of the audit to SUCCEEDED
+        self.update_audit_state(audit, objects.audit.State.SUCCEEDED)

watcher/decision_engine/datasources/base.py

@@ -19,6 +19,8 @@ import time
 from oslo_config import cfg
 from oslo_log import log
 
+from watcher.common import exception
+
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
@@ -54,6 +56,13 @@ class DataSourceBase(object):
                       instance_root_disk_size=None,
                       )
 
+    def _get_meter(self, meter_name):
+        """Retrieve the meter from the metric map or raise error"""
+        meter = self.METRIC_MAP.get(meter_name)
+        if meter is None:
+            raise exception.MetricNotAvailable(metric=meter_name)
+        return meter
+
     def query_retry(self, f, *args, **kwargs):
         """Attempts to retrieve metrics from the external service
 
@@ -122,6 +131,30 @@ class DataSourceBase(object):
 
         pass
 
+    @abc.abstractmethod
+    def statistic_series(self, resource=None, resource_type=None,
+                         meter_name=None, start_time=None, end_time=None,
+                         granularity=300):
+        """Retrieves metrics based on the specified parameters over a period
+
+        :param resource: Resource object as defined in watcher models such as
+                         ComputeNode and Instance
+        :param resource_type: Indicates which type of object is supplied
+                              to the resource parameter
+        :param meter_name: The desired metric to retrieve as key from
+                           METRIC_MAP
+        :param start_time: The datetime to start retrieving metrics for
+        :type start_time: datetime.datetime
+        :param end_time: The datetime to limit the retrieval of metrics to
+        :type end_time: datetime.datetime
+        :param granularity: Interval between samples in measurements in
+                            seconds
+        :return: Dictionary of key value pairs with timestamps and metric
+                 values
+        """
+
+        pass
+
     @abc.abstractmethod
     def get_host_cpu_usage(self, resource, period, aggregate,
                            granularity=None):

watcher/decision_engine/datasources/ceilometer.py

@@ -161,9 +161,7 @@ class CeilometerHelper(base.DataSourceBase):
         end_time = datetime.datetime.utcnow()
         start_time = end_time - datetime.timedelta(seconds=int(period))
 
-        meter = self.METRIC_MAP.get(meter_name)
-        if meter is None:
-            raise exception.MetricNotAvailable(metric=meter_name)
+        meter = self._get_meter(meter_name)
 
         if aggregate == 'mean':
             aggregate = 'avg'
@@ -188,12 +186,18 @@ class CeilometerHelper(base.DataSourceBase):
         item_value = None
         if statistic:
             item_value = statistic[-1]._info.get('aggregate').get(aggregate)
-            if meter_name is 'host_airflow':
+            if meter_name == 'host_airflow':
                 # Airflow from hardware.ipmi.node.airflow is reported as
                 # 1/10 th of actual CFM
                 item_value *= 10
         return item_value
 
+    def statistic_series(self, resource=None, resource_type=None,
+                         meter_name=None, start_time=None, end_time=None,
+                         granularity=300):
+        raise NotImplementedError(
+            _('Ceilometer helper does not support statistic series method'))
+
     def get_host_cpu_usage(self, resource, period,
                            aggregate, granularity=None):
 

watcher/decision_engine/datasources/gnocchi.py

@@ -23,7 +23,6 @@ from oslo_config import cfg
 from oslo_log import log
 
 from watcher.common import clients
-from watcher.common import exception
 from watcher.decision_engine.datasources import base
 
 CONF = cfg.CONF
@@ -72,9 +71,7 @@ class GnocchiHelper(base.DataSourceBase):
         stop_time = datetime.utcnow()
         start_time = stop_time - timedelta(seconds=(int(period)))
 
-        meter = self.METRIC_MAP.get(meter_name)
-        if meter is None:
-            raise exception.MetricNotAvailable(metric=meter_name)
+        meter = self._get_meter(meter_name)
 
         if aggregate == 'count':
             aggregate = 'mean'
@@ -116,13 +113,59 @@ class GnocchiHelper(base.DataSourceBase):
             # measure has structure [time, granularity, value]
             return_value = statistics[-1][2]
 
-            if meter_name is 'host_airflow':
+            if meter_name == 'host_airflow':
                 # Airflow from hardware.ipmi.node.airflow is reported as
                 # 1/10 th of actual CFM
                 return_value *= 10
 
         return return_value
 
+    def statistic_series(self, resource=None, resource_type=None,
+                         meter_name=None, start_time=None, end_time=None,
+                         granularity=300):
+
+        meter = self._get_meter(meter_name)
+
+        resource_id = resource.uuid
+        if resource_type == 'compute_node':
+            resource_id = "%s_%s" % (resource.hostname, resource.hostname)
+            kwargs = dict(query={"=": {"original_resource_id": resource_id}},
+                          limit=1)
+            resources = self.query_retry(
+                f=self.gnocchi.resource.search, **kwargs)
+
+            if not resources:
+                LOG.warning("The {0} resource {1} could not be "
+                            "found".format(self.NAME, resource_id))
+                return
+
+            resource_id = resources[0]['id']
+
+        raw_kwargs = dict(
+            metric=meter,
+            start=start_time,
+            stop=end_time,
+            resource_id=resource_id,
+            granularity=granularity,
+        )
+
+        kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
+
+        statistics = self.query_retry(
+            f=self.gnocchi.metric.get_measures, **kwargs)
+
+        return_value = None
+        if statistics:
+            # measure has structure [time, granularity, value]
+            if meter_name == 'host_airflow':
+                # Airflow from hardware.ipmi.node.airflow is reported as
+                # 1/10 th of actual CFM
+                return_value = {s[0]: s[2]*10 for s in statistics}
+            else:
+                return_value = {s[0]: s[2] for s in statistics}
+
+        return return_value
+
     def get_host_cpu_usage(self, resource, period, aggregate,
                            granularity=300):
 

watcher/decision_engine/datasources/grafana.py

@@ -16,10 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from urllib import parse as urlparse
+
 from oslo_config import cfg
 from oslo_log import log
-import six.moves.urllib.parse as urlparse
 
+from watcher._i18n import _
 from watcher.common import clients
 from watcher.common import exception
 from watcher.decision_engine.datasources import base
@@ -72,7 +74,7 @@ class GrafanaHelper(base.DataSourceBase):
 
         # Very basic url parsing
         parse = urlparse.urlparse(self._base_url)
-        if parse.scheme is '' or parse.netloc is '' or parse.path is '':
+        if parse.scheme == '' or parse.netloc == '' or parse.path == '':
             LOG.critical("GrafanaHelper url not properly configured, "
                          "check base_url and project_id")
             return
@@ -187,6 +189,12 @@ class GrafanaHelper(base.DataSourceBase):
 
         return result
 
+    def statistic_series(self, resource=None, resource_type=None,
+                         meter_name=None, start_time=None, end_time=None,
+                         granularity=300):
+        raise NotImplementedError(
+            _('Grafana helper does not support statistic series method'))
+
     def get_host_cpu_usage(self, resource, period=300,
                            aggregate="mean", granularity=None):
         return self.statistic_aggregation(

watcher/decision_engine/datasources/manager.py

@@ -112,10 +112,10 @@ class DataSourceManager(object):
         datasource is attempted.
         """
 
-        if not self.datasources or len(self.datasources) is 0:
+        if not self.datasources or len(self.datasources) == 0:
             raise exception.NoDatasourceAvailable
 
-        if not metrics or len(metrics) is 0:
+        if not metrics or len(metrics) == 0:
             LOG.critical("Can not retrieve datasource without specifying "
                          "list of required metrics.")
             raise exception.InvalidParameter(parameter='metrics',
@@ -125,11 +125,11 @@ class DataSourceManager(object):
             no_metric = False
             for metric in metrics:
                 if (metric not in self.metric_map[datasource] or
-                   self.metric_map[datasource].get(metric) is None):
-                        no_metric = True
-                        LOG.warning("Datasource: {0} could not be used due to "
-                                    "metric: {1}".format(datasource, metric))
-                        break
+                        self.metric_map[datasource].get(metric) is None):
+                    no_metric = True
+                    LOG.warning("Datasource: {0} could not be used due to "
+                                "metric: {1}".format(datasource, metric))
+                    break
             if not no_metric:
                 # Try to use a specific datasource but attempt additional
                 # datasources upon exceptions (if config has more datasources)