Native Zuul v3 watcher-grenade job + some cleanup

Create a native Zuul v3 grenade job. It matches the existing job,
even though it doesn't call any local hook as the current legacy
job does (because no local hook exists and it should be rewritten
as zuul configuration if it did).

The new job reuses the variable definition of the devstack watcher
job, so clean up that job as well:
- do not depend on devstack-gate, which is not needed and will be
  deprecated soon anyway;
- use the new way (tempest_plugins) to define which tempest plugin
  should be installed;
- remove the definition of USE_PYTHON3: true and simply inherit
  the value set by devstack;
- remove the definition of PYTHONUNBUFFERED, not really set
  anywhere else and only useful back in the days in Jenkins.

Change-Id: Ib0ed3c0f395e1b85b8f25f6e438c414165baab32
(cherry picked from commit ed59145354)

.gitreview

@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
-defaultbranch=stable/train
+defaultbranch=stable/ussuri

.zuul.yaml

@@ -3,8 +3,7 @@
       - check-requirements
       - openstack-cover-jobs
       - openstack-lower-constraints-jobs
-      - openstack-python-jobs
+      - openstack-python3-ussuri-jobs
-      - openstack-python3-train-jobs
       - publish-openstack-docs-pti
       - release-notes-jobs-python3
     check:
@@ -161,7 +160,6 @@
     timeout: 7200
     required-projects: &base_required_projects
       - openstack/ceilometer
-      - openstack/devstack-gate
       - openstack/python-openstackclient
       - openstack/python-watcherclient
       - openstack/watcher
@@ -180,13 +178,10 @@
         s-container: false
         s-object: false
         s-proxy: false
-      devstack_localrc:
+      tempest_plugins:
-        TEMPEST_PLUGINS: /opt/stack/watcher-tempest-plugin
+        - watcher-tempest-plugin
       tempest_test_regex: watcher_tempest_plugin.tests.api
       tox_envlist: all
-      tox_environment:
-        # Do we really need to set this? It's cargo culted
-        PYTHONUNBUFFERED: 'true'
       zuul_copy_output:
         /etc/hosts: logs
 
@@ -200,10 +195,12 @@
 
 - job:
     name: watcher-grenade
-    parent: legacy-dsvm-base
+    parent: grenade
-    timeout: 10800
+    required-projects:
-    run: playbooks/legacy/grenade-devstack-watcher/run.yaml
+      - openstack/watcher
-    post-run: playbooks/legacy/grenade-devstack-watcher/post.yaml
+      - openstack/python-watcherclient
+      - openstack/watcher-tempest-plugin
+    vars: *base_vars
     irrelevant-files:
       - ^(test-|)requirements.txt$
       - ^.*\.rst$
@@ -215,12 +212,6 @@
       - ^setup.cfg$
       - ^tools/.*$
       - ^tox.ini$
-    required-projects:
-      - openstack/grenade
-      - openstack/devstack-gate
-      - openstack/watcher
-      - openstack/python-watcherclient
-      - openstack/watcher-tempest-plugin
 
 - job:
     # This job is used in python-watcherclient repo

README.rst

@@ -1,6 +1,6 @@
-========================
+=======
-Team and repository tags
+Watcher
-========================
+=======
 
 .. image:: https://governance.openstack.org/tc/badges/watcher.svg
     :target: https://governance.openstack.org/tc/reference/tags/index.html
@@ -13,10 +13,6 @@ Team and repository tags
 
           https://creativecommons.org/licenses/by/3.0/
 
-=======
-Watcher
-=======
-
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
 Watcher provides a robust framework to realize a wide range of cloud

api-ref/source/index.rst

@@ -16,3 +16,4 @@ Watcher API
 .. include:: watcher-api-v1-services.inc
 .. include:: watcher-api-v1-scoring_engines.inc
 .. include:: watcher-api-v1-datamodel.inc
+.. include:: watcher-api-v1-webhooks.inc

api-ref/source/watcher-api-v1-datamodel.inc

@@ -4,6 +4,8 @@
 Data Model
 ==========
 
+.. versionadded:: 1.3
+
 ``Data Model`` is very important for Watcher to generate resource
 optimization solutions. Users can easily view the data model by the
 API.
@@ -18,7 +20,7 @@ Returns the information about Data Model.
 
 Normal response codes: 200
 
-Error codes: 400,401
+Error codes: 400,401,406
 
 Request
 -------

api-ref/source/watcher-api-v1-webhooks.inc

@@ -0,0 +1,26 @@
+.. -*- rst -*-
+
+========
+Webhooks
+========
+
+.. versionadded:: 1.4
+
+Triggers an event based Audit.
+
+
+Trigger EVENT Audit
+===================
+
+.. rest_method::  POST /v1/webhooks/{audit_ident}
+
+Normal response codes: 202
+
+Error codes: 400,404
+
+Request
+-------
+
+.. rest_parameters:: parameters.yaml
+
+   - audit_ident: audit_ident

devstack/lib/watcher

@@ -298,7 +298,7 @@ function start_watcher_api {
         service_protocol="http"
     fi
     if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
-        run_process "watcher-api" "$WATCHER_BIN_DIR/uwsgi --ini $WATCHER_UWSGI_CONF"
+        run_process "watcher-api" "$WATCHER_BIN_DIR/uwsgi --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
         watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
     else
         watcher_url=$service_protocol://$SERVICE_HOST:$service_port

doc/requirements.txt

@@ -2,8 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 openstackdocstheme>=1.20.0 # Apache-2.0
-sphinx>=1.6.5,!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7' # BSD
+sphinx>=1.8.0,!=2.1.0,!=3.0.0 # BSD
-sphinx>=1.6.5,!=1.6.6,!=1.6.7,!=2.1.0;python_version>='3.4' # BSD
 sphinxcontrib-pecanwsme>=0.8.0 # Apache-2.0
 sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD
 reno>=2.7.0 # Apache-2.0

doc/source/admin/index.rst

@@ -8,6 +8,7 @@ Administrator Guide
    apache-mod-wsgi
    gmr
    policy
-   ways-to-install
    ../strategies/index
    ../datasources/index
+   ../contributor/notifications
+   ../contributor/concurrency

doc/source/architecture.rst

@@ -281,11 +281,13 @@ previously created :ref:`Audit template <audit_template_definition>`:
    :width: 100%
 
 The :ref:`Administrator <administrator_definition>` also can specify type of
-Audit and interval (in case of CONTINUOUS type). There is two types of Audit:
+Audit and interval (in case of CONTINUOUS type). There is three types of Audit:
-ONESHOT and CONTINUOUS. Oneshot Audit is launched once and if it succeeded
+ONESHOT, CONTINUOUS and EVENT. ONESHOT Audit is launched once and if it
-executed new action plan list will be provided. Continuous Audit creates
+succeeded executed new action plan list will be provided; CONTINUOUS Audit
-action plans with specified interval (in seconds); if action plan
+creates action plans with specified interval (in seconds or cron format, cron
-has been created, all previous action plans get CANCELLED state.
+inteval can be used like: `*/5 * * * *`), if action plan
+has been created, all previous action plans get CANCELLED state;
+EVENT audit is launched when receiving webhooks API.
 
 A message is sent on the :ref:`AMQP bus <amqp_bus_definition>` which triggers
 the Audit in the

doc/source/conf.py

@@ -36,7 +36,6 @@ extensions = [
     'sphinxcontrib.httpdomain',
     'sphinxcontrib.pecanwsme.rest',
     'stevedore.sphinxext',
-    'wsmeext.sphinxext',
     'ext.term',
     'ext.versioned_notifications',
     'oslo_config.sphinxconfiggen',

doc/source/contributor/concurrency.rst

@@ -0,0 +1,248 @@
+===========
+Concurrency
+===========
+
+Introduction
+************
+
+Modern processors typically contain multiple cores all capable of executing
+instructions in parallel. Ensuring applications can fully utilize modern
+underlying hardware requires developing with these concepts in mind. The
+OpenStack foundation maintains a number of libraries to facilitate this
+utilization, combined with constructs like CPython's GIL_ the proper use of
+these concepts becomes more straightforward compared to other programming
+languages.
+
+The primary libraries maintained by OpenStack to facilitate concurrency are
+futurist_ and taskflow_. Here futurist is a more straightforward and
+lightweight library while taskflow is more advanced supporting features like
+rollback mechanisms. Within Watcher both libraries are used to facilitate
+concurrency.
+
+.. _GIL: https://wiki.python.org/moin/GlobalInterpreterLock
+.. _futurist: https://docs.openstack.org/futurist/latest/
+.. _taskflow: https://docs.openstack.org/taskflow/latest/
+
+Threadpool
+**********
+
+A threadpool is a collection of one or more threads typically called *workers*
+to which tasks can be submitted. These submitted tasks will be scheduled by a
+threadpool and subsequently executed. In the case of Python tasks typically are
+bounded or unbounded methods while other programming languages like Java
+require implementing an interface.
+
+The order and amount of concurrency with which these tasks are executed is up
+to the threadpool to decide. Some libraries like taskflow allow for either
+strong or loose ordering of tasks while others like futurist might only support
+loose ordering. Taskflow supports building tree-based hierarchies of dependent
+tasks for example.
+
+Upon submission of a task to a threadpool a so called future_ is returned.
+These objects allow to determine information about the task such as if it is
+currently being executed or if it has finished execution. When the task has
+finished execution the future can also be used to retrieve what was returned by
+the method.
+
+Some libraries like futurist provide synchronization primitives for collections
+of futures such as wait_for_any_. The following sections will cover different
+types of concurrency used in various services of Watcher.
+
+.. _future: https://docs.python.org/3/library/concurrent.futures.html
+.. _wait_for_any: https://docs.openstack.org/futurist/latest/reference/index.html#waiters
+
+
+Decision engine concurrency
+***************************
+
+The concurrency in the decision engine is governed by two independent
+threadpools. Both of these threadpools are GreenThreadPoolExecutor_ from the
+futurist_ library. One of these is used automatically and most contributors
+will not interact with it while developing new features. The other threadpool
+can frequently be used while developing new features or updating existing ones.
+It is known as the DecisionEngineThreadpool and allows to achieve performance
+improvements in network or I/O bound operations.
+
+.. _GreenThreadPoolExecutor: https://docs.openstack.org/futurist/latest/reference/index.html#executors
+
+AuditEndpoint
+#############
+
+The first threadpool is used to allow multiple audits to be run in parallel.
+In practice, however, only one audit can be run in parallel. This is due to
+the data model used by audits being a singleton. To prevent audits destroying
+each others data model one must wait for the other to complete before being
+allowed to access this data model. A performance improvement could be achieved
+by being more intelligent in the use, caching and construction of these
+data models.
+
+DecisionEngineThreadPool
+########################
+
+The second threadpool is used for generic tasks, typically networking and I/O
+could benefit the most of this threadpool. Upon execution of an audit this
+threadpool can be utilized to retrieve information from the Nova compute
+service for instance. This second threadpool is a singleton and is shared
+amongst concurrently running audits as a result the amount of workers is static
+and independent from the amount of workers in the first threadpool. The use of
+the :class:`~.DecisionEngineThreadpool` while building the Nova compute data
+model is demonstrated to show how it can effectively be used.
+
+In the following example a reference to the
+:class:`~.DecisionEngineThreadpool` is stored in ``self.executor``. Here two
+tasks are submitted one with function ``self._collect_aggregates`` and the
+other function ``self._collect_zones``. With both ``self.executor.submit``
+calls subsequent arguments are passed to the function. All subsequent arguments
+are passed to the function being submitted as task following the common
+``(fn, *args, **kwargs)`` signature. One of the original signatures would be
+``def _collect_aggregates(host_aggregates, compute_nodes)`` for example.
+
+.. code-block:: python
+
+    zone_aggregate_futures = {
+        self.executor.submit(
+            self._collect_aggregates, host_aggregates, compute_nodes),
+        self.executor.submit(
+            self._collect_zones, availability_zones, compute_nodes)
+    }
+    waiters.wait_for_all(zone_aggregate_futures)
+
+The last statement of the example above waits on all futures to complete.
+Similarly, ``waiters.wait_for_any`` will wait for any future of the specified
+collection to complete. To simplify the usage of ``wait_for_any`` the
+:class:`~.DecisiongEngineThreadpool` defines a ``do_while_futures`` method.
+This method will iterate in a do_while loop over a collection of futures until
+all of them have completed. The advantage of ``do_while_futures`` is that it
+allows to immediately call a method as soon as a future finishes. The arguments
+for this callback method can be supplied when calling ``do_while_futures``,
+however, the first argument to the callback is always the future itself! If
+the collection of futures can safely be modified ``do_while_futures_modify``
+can be used and should have slightly better performance. The following example
+will show how ``do_while_futures`` is used in the decision engine.
+
+.. code-block:: python
+
+    # For every compute node from compute_nodes submit a task to gather the node it's information.
+    # List comprehension is used to store all the futures of the submitted tasks in node_futures.
+    node_futures = [self.executor.submit(
+        self.nova_helper.get_compute_node_by_name,
+        node, servers=True, detailed=True)
+        for node in compute_nodes]
+    LOG.debug("submitted {0} jobs".format(len(compute_nodes)))
+
+    future_instances = []
+    # do_while iterate over node_futures and upon completion of a future call
+    # self._compute_node_future with the future and future_instances as arguments.
+    self.executor.do_while_futures_modify(
+        node_futures, self._compute_node_future, future_instances)
+
+    # Wait for all instance jobs to finish
+    waiters.wait_for_all(future_instances)
+
+Finally, let's demonstrate how powerful this ``do_while_futures`` can be by
+showing what the ``compute_node_future`` callback does. First, it retrieves the
+result from the future and adds the compute node to the data model. Afterwards,
+it checks if the compute node has any associated instances and if so it submits
+an additional task to the :class:`~.DecisionEngineThreadpool`. The future is
+appended to the ``future_instances`` so ``waiters.wait_for_all`` can be called
+on this list. This is important as otherwise the building of the data model
+might return before all tasks for instances have finished.
+
+.. code-block:: python
+
+    # Get the result from the future.
+    node_info = future.result()[0]
+
+    # Filter out baremetal nodes.
+    if node_info.hypervisor_type == 'ironic':
+        LOG.debug("filtering out baremetal node: %s", node_info)
+        return
+
+    # Add the compute node to the data model.
+    self.add_compute_node(node_info)
+    # Get the instances from the compute node.
+    instances = getattr(node_info, "servers", None)
+    # Do not submit job if there are no instances on compute node.
+    if instances is None:
+        LOG.info("No instances on compute_node: {0}".format(node_info))
+        return
+    # Submit a job to retrieve detailed information about the instances.
+    future_instances.append(
+        self.executor.submit(
+            self.add_instance_node, node_info, instances)
+    )
+
+Without ``do_while_futures`` an additional ``waiters.wait_for_all`` would be
+required in between the compute node tasks and the instance tasks. This would
+cause the progress of the decision engine to stall as less and less tasks
+remain active before the instance tasks could be submitted. This demonstrates
+how ``do_while_futures`` can be used to achieve more constant utilization of
+the underlying hardware.
+
+Applier concurrency
+*******************
+
+The applier does not use the futurist_ GreenThreadPoolExecutor_ directly but
+instead uses taskflow_. However, taskflow still utilizes a greenthreadpool.
+This threadpool is initialized in the workflow engine called
+:class:`~.DefaultWorkFlowEngine`. Currently Watcher supports one workflow
+engine but the base class allows contributors to develop other workflow engines
+as well. In taskflow tasks are created using different types of flows such as a
+linear, unordered or a graph flow. The linear and graph flow allow for strong
+ordering between individual tasks and it is for this reason that the workflow
+engine utilizes a graph flow. The creation of tasks, subsequently linking them
+into a graph like structure and submitting them is shown below.
+
+.. code-block:: python
+
+    self.execution_rule = self.get_execution_rule(actions)
+    flow = gf.Flow("watcher_flow")
+    actions_uuid = {}
+    for a in actions:
+        task = TaskFlowActionContainer(a, self)
+        flow.add(task)
+        actions_uuid[a.uuid] = task
+
+    for a in actions:
+        for parent_id in a.parents:
+            flow.link(actions_uuid[parent_id], actions_uuid[a.uuid],
+                      decider=self.decider)
+
+    e = engines.load(
+        flow, executor='greenthreaded', engine='parallel',
+        max_workers=self.config.max_workers)
+    e.run()
+
+    return flow
+
+In the applier tasks are contained in a :class:`~.TaskFlowActionContainer`
+which allows them to trigger events in the workflow engine. This way the
+workflow engine can halt or take other actions while the action plan is being
+executed based on the success or failure of individual actions. However, the
+base workflow engine simply uses these notifies to store the result of
+individual actions in the database. Additionally, since taskflow uses a graph
+flow if any of the tasks would fail all childs of this tasks not be executed
+while ``do_revert`` will be triggered for all parents.
+
+.. code-block:: python
+
+    class TaskFlowActionContainer(...):
+        ...
+        def do_execute(self, *args, **kwargs):
+            ...
+            result = self.action.execute()
+            if result is True:
+                return self.engine.notify(self._db_action,
+                                          objects.action.State.SUCCEEDED)
+            else:
+                self.engine.notify(self._db_action,
+                                   objects.action.State.FAILED)
+
+    class BaseWorkFlowEngine(...):
+        ...
+        def notify(self, action, state):
+            db_action = objects.Action.get_by_uuid(self.context, action.uuid,
+                                                   eager=True)
+            db_action.state = state
+            db_action.save()
+            return db_action

doc/source/contributor/contributing.rst

@@ -1,71 +1,111 @@
-..
+============================
-      Except where otherwise noted, this document is licensed under Creative
+So You Want to Contribute...
-      Commons Attribution 3.0 License.  You can view the license at:
+============================
 
-          https://creativecommons.org/licenses/by/3.0/
+For general information on contributing to OpenStack, please check out the
+`contributor guide <https://docs.openstack.org/contributors/>`_ to get started.
+It covers all the basics that are common to all OpenStack projects:
+the accounts you need, the basics of interacting with our Gerrit review system,
+how we communicate as a community, etc.
 
-.. _contributing:
+Below will cover the more project specific information you need to get started
+with Watcher.
 
-=======================
+Communication
-Contributing to Watcher
+~~~~~~~~~~~~~~
-=======================
+.. This would be a good place to put the channel you chat in as a project; when/
-
+   where your meeting is, the tags you prepend to your ML threads, etc.
-If you're interested in contributing to the Watcher project,
-the following will help get you started.
-
-Contributor License Agreement
------------------------------
-
-.. index::
-   single: license; agreement
-
-In order to contribute to the Watcher project, you need to have
-signed OpenStack's contributor's agreement.
-
-.. seealso::
-
-   * https://docs.openstack.org/infra/manual/developers.html
-   * https://wiki.openstack.org/CLA
-
-LaunchPad Project
------------------
-
-Most of the tools used for OpenStack depend on a launchpad.net ID for
-authentication. After signing up for a launchpad account, join the
-"openstack" team to have access to the mailing list and receive
-notifications of important events.
-
-.. seealso::
-
-   * https://launchpad.net
-   * https://launchpad.net/watcher
-   * https://launchpad.net/openstack
-
-
-Project Hosting Details
------------------------
-
-Bug tracker
-    https://launchpad.net/watcher
-
-Mailing list (prefix subjects with ``[watcher]`` for faster responses)
-    http://lists.openstack.org/pipermail/openstack-discuss/
-
-Wiki
-    https://wiki.openstack.org/Watcher
-
-Code Hosting
-    https://opendev.org/openstack/watcher
-
-Code Review
-    https://review.opendev.org/#/q/status:open+project:openstack/watcher,n,z
 
 IRC Channel
     ``#openstack-watcher`` (changelog_)
 
+Mailing list(prefix subjects with ``[watcher]``)
+    http://lists.openstack.org/pipermail/openstack-discuss/
+
 Weekly Meetings
     Bi-weekly, on Wednesdays at 08:00 UTC on odd weeks in the
     ``#openstack-meeting-alt`` IRC channel (`meetings logs`_)
 
+Meeting Agenda
+    https://wiki.openstack.org/wiki/Watcher_Meeting_Agenda
+
 .. _changelog: http://eavesdrop.openstack.org/irclogs/%23openstack-watcher/
 .. _meetings logs:  http://eavesdrop.openstack.org/meetings/watcher/
+
+Contacting the Core Team
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. This section should list the core team, their irc nicks, emails, timezones etc.
+   If all this info is maintained elsewhere (i.e. a wiki), you can link to that
+   instead of enumerating everyone here.
+
++--------------------+---------------+------------------------------------+
+| Name               | IRC           | Email                              |
++====================+===============+====================================+
+| `Li Canwei`_       | licanwei      | li.canwei2@zte.com.cn              |
++--------------------+---------------+------------------------------------+
+| `chen ke`_         | chenke        | chen.ke14@zte.com.cn               |
++--------------------+---------------+------------------------------------+
+| `Corne Lukken`_    | dantalion     | info@dantalion.nl                  |
++--------------------+---------------+------------------------------------+
+| `su zhengwei`_     | suzhengwei    | sugar-2008@163.com                 |
++--------------------+---------------+------------------------------------+
+| `Yumeng Bao`_      | Yumeng        | yumeng_bao@yahoo.com               |
++--------------------+---------------+------------------------------------+
+
+.. _Corne Lukken: https://launchpad.net/~dantalion
+.. _Li Canwei: https://launchpad.net/~li-canwei2
+.. _su zhengwei: https://launchpad.net/~sue.sam
+.. _Yumeng Bao: https://launchpad.net/~yumeng-bao
+.. _chen ke: https://launchpad.net/~chenker
+
+New Feature Planning
+~~~~~~~~~~~~~~~~~~~~
+.. This section is for talking about the process to get a new feature in. Some
+   projects use blueprints, some want specs, some want both! Some projects
+   stick to a strict schedule when selecting what new features will be reviewed
+   for a release.
+
+New feature will be discussed via IRC or ML (with [Watcher] prefix).
+Watcher team uses blueprints in `Launchpad`_ to manage the new features.
+
+.. _Launchpad: https://launchpad.net/watcher
+
+Task Tracking
+~~~~~~~~~~~~~~
+.. This section is about where you track tasks- launchpad? storyboard?
+   is there more than one launchpad project? what's the name of the project
+   group in storyboard?
+
+We track our tasks in Launchpad.
+If you're looking for some smaller, easier work item to pick up and get started
+on, search for the 'low-hanging-fruit' tag.
+
+.. NOTE: If your tag is not 'low-hanging-fruit' please change the text above.
+
+Reporting a Bug
+~~~~~~~~~~~~~~~
+.. Pretty self explanatory section, link directly to where people should report bugs for
+   your project.
+
+You found an issue and want to make sure we are aware of it? You can do so
+`HERE`_.
+
+.. _HERE: https://bugs.launchpad.net/watcher
+
+Getting Your Patch Merged
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. This section should have info about what it takes to get something merged.
+   Do you require one or two +2's before +W? Do some of your repos require
+   unit test changes with all patches? etc.
+
+Due to the small number of core reviewers of the Watcher project,
+we only need one +2 before +W (merge). All patches excepting for documentation
+or typos fixes must have unit test.
+
+Project Team Lead Duties
+------------------------
+.. this section is where you can put PTL specific duties not already listed in
+   the common PTL guide (linked below)  or if you already have them written
+   up elsewhere, you can link to that doc here.
+
+All common PTL duties are enumerated here in the `PTL guide <https://docs.openstack.org/project-team-guide/ptl.html>`_.

doc/source/contributor/index.rst

@@ -1,8 +1,12 @@
-.. toctree::
+==================
-   :maxdepth: 1
+Contribution Guide
+==================
 
+.. toctree::
+   :maxdepth: 2
+
+   contributing
    environment
    devstack
-   notifications
    testing
    rally_link

doc/source/contributor/plugin/index.rst

@@ -1,3 +1,7 @@
+============
+Plugin Guide
+============
+
 .. toctree::
    :maxdepth: 1
 

doc/source/contributor/testing.rst

@@ -4,9 +4,9 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-=======
+=================
-Testing
+Developer Testing
-=======
+=================
 
 .. _unit_tests:
 
@@ -15,7 +15,7 @@ Unit tests
 
 All unit tests should be run using `tox`_. Before running the unit tests, you
 should download the latest `watcher`_ from the github. To run the same unit
-tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
+tests that are executing onto `Gerrit`_ which includes ``py36``, ``py37`` and
 ``pep8``, you can issue the following command::
 
     $ git clone https://opendev.org/openstack/watcher
@@ -26,8 +26,8 @@ tests that are executing onto `Gerrit`_ which includes ``py35``, ``py27`` and
 If you only want to run one of the aforementioned, you can then issue one of
 the following::
 
-    $ tox -e py35
+    $ tox -e py36
-    $ tox -e py27
+    $ tox -e py37
     $ tox -e pep8
 
 .. _tox: https://tox.readthedocs.org/
@@ -38,7 +38,7 @@ If you only want to run specific unit test code and don't like to waste time
 waiting for all unit tests to execute, you can add parameters ``--`` followed
 by a regex string::
 
-    $ tox -e py27 -- watcher.tests.api
+    $ tox -e py37 -- watcher.tests.api
 
 .. _tempest_tests:
 

doc/source/index.rst

@@ -32,91 +32,21 @@ specific prior release.
 .. _python-watcherclient: https://opendev.org/openstack/python-watcherclient/
 .. _watcher-dashboard: https://opendev.org/openstack/watcher-dashboard/
 
-Developer Guide
-===============
-
-Introduction
-------------
-
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
-  glossary
   architecture
-  contributor/contributing
-
-
-Getting Started
----------------
-
-.. toctree::
-  :maxdepth: 1
-
   contributor/index
-
-Installation
-============
-.. toctree::
-  :maxdepth: 2
-
   install/index
-
-Admin Guide
-===========
-
-.. toctree::
-  :maxdepth: 2
-
   admin/index
-
-User Guide
-==========
-
-.. toctree::
-  :maxdepth: 2
-
   user/index
-
+  configuration/index
-API References
+  contributor/plugin/index
-==============
+  man/index
 
 .. toctree::
   :maxdepth: 1
 
   API Reference <https://docs.openstack.org/api-ref/resource-optimization/>
   Watcher API Microversion History </contributor/api_microversion_history>
-
+  glossary
-Plugins
--------
-
-.. toctree::
-  :maxdepth: 1
-
-  contributor/plugin/index
-
-Watcher Configuration Options
-=============================
-
-.. toctree::
-  :maxdepth: 2
-
-  configuration/index
-
-Watcher Manual Pages
-====================
-
-.. toctree::
-   :glob:
-   :maxdepth: 1
-
-   man/index
-
-
-.. only:: html
-
-Indices and tables
-==================
-
-   * :ref:`genindex`
-   * :ref:`modindex`
-   * :ref:`search`

doc/source/install/index.rst

@@ -1,6 +1,6 @@
-===================================
+=============
-Infrastructure Optimization service
+Install Guide
-===================================
+=============
 
 .. toctree::
    :maxdepth: 2

doc/source/man/index.rst

@@ -1,3 +1,7 @@
+====================
+Watcher Manual Pages
+====================
+
 .. toctree::
    :glob:
    :maxdepth: 1

doc/source/user/event_type_audit.rst

@@ -0,0 +1,195 @@
+..
+  Licensed under the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License. You may obtain
+  a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+
+
+======================
+Audit using Aodh alarm
+======================
+
+Audit with EVENT type can be triggered by special alarm. This guide walks
+you through the steps to build an event-driven optimization solution by
+integrating Watcher with Ceilometer/Aodh.
+
+Step 1: Create an audit with EVENT type
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first step is to create an audit with EVENT type,
+you can create an audit template firstly:
+
+.. code-block:: bash
+
+  $ openstack optimize audittemplate create your_template_name <your_goal> \
+    --strategy <your_strategy>
+
+or create an audit directly with special goal and strategy:
+
+.. code-block:: bash
+
+  $ openstack optimize audit create --goal <your_goal> \
+    --strategy <your_strategy> --audit_type EVENT
+
+This is an example for creating an audit with dummy strategy:
+
+.. code-block:: bash
+
+  $ openstack optimize audit create --goal dummy \
+    --strategy dummy --audit_type EVENT
+  +---------------+--------------------------------------+
+  | Field         | Value                                |
+  +---------------+--------------------------------------+
+  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
+  | Name          | dummy-2020-01-14T03:21:19.168467     |
+  | Created At    | 2020-01-14T03:21:19.200279+00:00     |
+  | Updated At    | None                                 |
+  | Deleted At    | None                                 |
+  | State         | PENDING                              |
+  | Audit Type    | EVENT                                |
+  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
+  | Interval      | None                                 |
+  | Goal          | dummy                                |
+  | Strategy      | dummy                                |
+  | Audit Scope   | []                                   |
+  | Auto Trigger  | False                                |
+  | Next Run Time | None                                 |
+  | Hostname      | None                                 |
+  | Start Time    | None                                 |
+  | End Time      | None                                 |
+  | Force         | False                                |
+  +---------------+--------------------------------------+
+
+We need to build Aodh action url using Watcher webhook API.
+For convenience we export the url into an environment variable:
+
+.. code-block:: bash
+
+  $ export AUDIT_UUID=a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  $ export ALARM_URL="trust+http://localhost/infra-optim/v1/webhooks/$AUDIT_UUID"
+
+Step 2: Create Aodh Alarm
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Once we have the audit created, we can continue to create Aodh alarm and
+set the alarm action to Watcher webhook API. The alarm type can be event(
+i.e. ``compute.instance.create.end``) or gnocchi_resources_threshold(i.e.
+``cpu_util``), more info refer to alarm-creation_
+
+For example:
+
+.. code-block:: bash
+
+  $ openstack alarm create \
+    --type event --name instance_create \
+    --event-type "compute.instance.create.end" \
+    --enable True --repeat-actions False \
+    --alarm-action $ALARM_URL
+  +---------------------------+------------------------------------------------------------------------------------------+
+  | Field                     | Value                                                                                    |
+  +---------------------------+------------------------------------------------------------------------------------------+
+  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5'] |
+  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                     |
+  | description               | Alarm when compute.instance.create.end event occurred.                                   |
+  | enabled                   | True                                                                                     |
+  | event_type                | compute.instance.create.end                                                              |
+  | insufficient_data_actions | []                                                                                       |
+  | name                      | instance_create                                                                          |
+  | ok_actions                | []                                                                                       |
+  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                         |
+  | query                     |                                                                                          |
+  | repeat_actions            | False                                                                                    |
+  | severity                  | low                                                                                      |
+  | state                     | insufficient data                                                                        |
+  | state_reason              | Not evaluated yet                                                                        |
+  | state_timestamp           | 2020-01-14T03:56:26.894416                                                               |
+  | time_constraints          | []                                                                                       |
+  | timestamp                 | 2020-01-14T03:56:26.894416                                                               |
+  | type                      | event                                                                                    |
+  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                         |
+  +---------------------------+------------------------------------------------------------------------------------------+
+
+.. _alarm-creation: https://docs.openstack.org/aodh/latest/admin/telemetry-alarms.html#alarm-creation
+
+Step 3: Trigger the alarm
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this example, you can create a new instance to trigger the alarm.
+The alarm state will translate from ``insufficient data`` to ``alarm``.
+
+.. code-block:: bash
+
+  $ openstack alarm show b9e381fc-8e3e-4943-82ee-647e7a2ef644
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+  | Field                     | Value                                                                                                             |
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+  | alarm_actions             | [u'trust+http://localhost/infra-optim/v1/webhooks/a3326a6a-c18e-4e8e-adba-d0c61ad404c5']                          |
+  | alarm_id                  | b9e381fc-8e3e-4943-82ee-647e7a2ef644                                                                              |
+  | description               | Alarm when compute.instance.create.end event occurred.                                                            |
+  | enabled                   | True                                                                                                              |
+  | event_type                | compute.instance.create.end                                                                                       |
+  | insufficient_data_actions | []                                                                                                                |
+  | name                      | instance_create                                                                                                   |
+  | ok_actions                | []                                                                                                                |
+  | project_id                | 728d66e18c914af1a41e2a585cf766af                                                                                  |
+  | query                     |                                                                                                                   |
+  | repeat_actions            | False                                                                                                             |
+  | severity                  | low                                                                                                               |
+  | state                     | alarm                                                                                                             |
+  | state_reason              | Event <id=67dd0afa-2082-45a4-8825-9573b2cc60e5,event_type=compute.instance.create.end> hits the query <query=[]>. |
+  | state_timestamp           | 2020-01-14T03:56:26.894416                                                                                        |
+  | time_constraints          | []                                                                                                                |
+  | timestamp                 | 2020-01-14T06:17:40.350649                                                                                        |
+  | type                      | event                                                                                                             |
+  | user_id                   | 88c40156af7445cc80580a1e7e3ba308                                                                                  |
+  +---------------------------+-------------------------------------------------------------------------------------------------------------------+
+
+Step 4: Verify the audit
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can be verified to check if the audit state was ``SUCCEEDED``:
+
+.. code-block:: bash
+
+  $ openstack optimize audit show a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  +---------------+--------------------------------------+
+  | Field         | Value                                |
+  +---------------+--------------------------------------+
+  | UUID          | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 |
+  | Name          | dummy-2020-01-14T03:21:19.168467     |
+  | Created At    | 2020-01-14T03:21:19+00:00            |
+  | Updated At    | 2020-01-14T06:26:40+00:00            |
+  | Deleted At    | None                                 |
+  | State         | SUCCEEDED                            |
+  | Audit Type    | EVENT                                |
+  | Parameters    | {u'para2': u'hello', u'para1': 3.2}  |
+  | Interval      | None                                 |
+  | Goal          | dummy                                |
+  | Strategy      | dummy                                |
+  | Audit Scope   | []                                   |
+  | Auto Trigger  | False                                |
+  | Next Run Time | None                                 |
+  | Hostname      | ubuntudbs                            |
+  | Start Time    | None                                 |
+  | End Time      | None                                 |
+  | Force         | False                                |
+  +---------------+--------------------------------------+
+
+and you can use the following command to check if the action plan
+was created:
+
+.. code-block:: bash
+
+  $ openstack optimize actionplan list --audit a3326a6a-c18e-4e8e-adba-d0c61ad404c5
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
+  | UUID                                 | Audit                                | State       | Updated At | Global efficacy |
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+
+  | 673b3fcb-8c16-4a41-9ee3-2956d9f6ca9e | a3326a6a-c18e-4e8e-adba-d0c61ad404c5 | RECOMMENDED | None       |                 |
+  +--------------------------------------+--------------------------------------+-------------+------------+-----------------+

doc/source/user/index.rst

@@ -1,4 +1,10 @@
+==========
+User Guide
+==========
+
 .. toctree::
    :maxdepth: 2
 
+   ways-to-install
    user-guide
+   event_type_audit

doc/source/user/user-guide.rst

@@ -4,8 +4,6 @@
 
           https://creativecommons.org/licenses/by/3.0/
 
-.. _user-guide:
-
 ==================
 Watcher User Guide
 ==================
@@ -60,8 +58,8 @@ plugin installation guide`_.
 .. _`OpenStack CLI`: https://docs.openstack.org/python-openstackclient/latest/cli/man/openstack.html
 .. _`Watcher CLI`: https://docs.openstack.org/python-watcherclient/latest/cli/index.html
 
-Seeing what the Watcher CLI can do ?
+Watcher CLI Command
-------------------------------------
+-------------------
 We can see all of the commands available with Watcher CLI by running the
 watcher binary without options.
 
@@ -69,8 +67,8 @@ watcher binary without options.
 
   $ openstack help optimize
 
-How do I run an audit of my cluster ?
+Running an audit of the cluster
--------------------------------------
+-------------------------------
 
 First, you need to find the :ref:`goal <goal_definition>` you want to achieve:
 

lower-constraints.txt

@@ -30,7 +30,6 @@ eventlet==0.20.0
 extras==1.0.0
 fasteners==0.14.1
 fixtures==3.0.0
-flake8==2.5.5
 freezegun==0.3.10
 future==0.16.0
 futurist==1.8.0
@@ -38,7 +37,6 @@ gitdb2==2.0.3
 GitPython==2.1.8
 gnocchiclient==7.0.1
 greenlet==0.4.13
-hacking==0.12.0
 idna==2.6
 imagesize==1.0.0
 iso8601==0.1.12
@@ -64,7 +62,7 @@ msgpack==0.5.6
 munch==2.2.0
 netaddr==0.7.19
 netifaces==0.10.6
-networkx==1.11
+networkx==2.2
 openstackdocstheme==1.20.0
 openstacksdk==0.12.0
 os-api-ref===1.4.0
@@ -95,14 +93,12 @@ Paste==2.0.3
 PasteDeploy==1.5.2
 pbr==3.1.1
 pecan==1.3.2
-pep8==1.5.7
 pika==0.10.0
 pika-pool==0.1.3
 prettytable==0.7.2
 psutil==5.4.3
 pycadf==2.7.0
 pycparser==2.18
-pyflakes==0.8.1
 Pygments==2.2.0
 pyinotify==0.9.6
 pyOpenSSL==17.5.0
@@ -145,7 +141,7 @@ sqlparse==0.2.4
 statsd==3.2.2
 stestr==2.0.0
 stevedore==1.28.0
-taskflow==3.1.0
+taskflow==3.7.1
 Tempita==0.5.2
 tenacity==4.9.0
 testresources==2.0.1

playbooks/legacy/grenade-devstack-watcher/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/grenade-devstack-watcher/run.yaml

@@ -1,60 +0,0 @@
-- hosts: all
-  name: legacy-grenade-dsvm-watcher
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-
-          export PROJECTS="openstack/grenade $PROJECTS"
-          export PROJECTS="openstack/watcher $PROJECTS"
-          export PROJECTS="openstack/watcher-tempest-plugin $PROJECTS"
-          export PROJECTS="openstack/python-watcherclient $PROJECTS"
-          export DEVSTACK_PROJECT_FROM_GIT="python-watcherclient $DEVSTACK_PROJECT_FROM_GIT"
-
-          export GRENADE_PLUGINRC="enable_grenade_plugin watcher https://opendev.org/openstack/watcher"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"export TEMPEST_PLUGINS='/opt/stack/new/watcher-tempest-plugin'"
-
-          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
-          export DEVSTACK_GATE_GRENADE=pullup
-
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-          # Add configuration values for enabling security features in local.conf
-          function pre_test_hook {
-              if [ -f /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh ] ; then
-                  . /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh
-              fi
-          }
-          export -f pre_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

releasenotes/notes/drop-py-2-7-54f8e806d71f19a7.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Python 2.7 support has been dropped. Last release of Watcher
+    to support py2.7 is OpenStack Train. The minimum version of Python now
+    supported by Watcher is Python 3.6.

releasenotes/notes/event-driven-optimization-based-4870f112bef8a560.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add a new webhook API and a new audit type EVENT, the microversion is 1.4.
+    Now Watcher user can create audit with EVENT type and the audit will be
+    triggered by webhook API.
+    The user guide is available online:
+    https://docs.openstack.org/watcher/latest/user/event_type_audit.html

releasenotes/notes/general-purpose-decision-engine-threadpool-0711b23abfc9d409.yaml

@@ -0,0 +1,20 @@
+---
+prelude: >
+    Many operations in the decision engine will block on I/O. Such I/O
+    operations can stall the execution of a sequential application
+    significantly. To reduce the potential bottleneck of many operations
+    the general purpose decision engine threadpool is introduced.
+features:
+  - |
+    A new threadpool for the decision engine that contributors can use to
+    improve the performance of many operations, primarily I/O bound onces.
+    The amount of workers used by the decision engine threadpool can be
+    configured to scale according to the available infrastructure using
+    the `watcher_decision_engine.max_general_workers` config option.
+    Documentation for contributors to effectively use this threadpool is
+    available online:
+    https://docs.openstack.org/watcher/latest/contributor/concurrency.html
+  - |
+    The building of the compute (Nova) data model will be done using the
+    decision engine threadpool, thereby, significantly reducing the total
+    time required to build it.

releasenotes/source/index.rst

@@ -21,6 +21,7 @@ Contents:
    :maxdepth: 1
 
    unreleased
+   train
    stein
    rocky
    queens

releasenotes/source/train.rst

@@ -0,0 +1,6 @@
+==========================
+Train Series Release Notes
+==========================
+
+.. release-notes::
+   :branch: stable/train

requirements.txt

@@ -3,7 +3,6 @@
 # process, which may cause wedges in the gate later.
 
 apscheduler>=3.5.1 # MIT License
-enum34>=1.1.6;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
 jsonpatch>=1.21 # BSD
 keystoneauth1>=3.4.0 # Apache-2.0
 jsonschema>=2.6.0 # MIT
@@ -43,11 +42,9 @@ python-ironicclient>=2.5.0 # Apache-2.0
 six>=1.11.0 # MIT
 SQLAlchemy>=1.2.5 # MIT
 stevedore>=1.28.0 # Apache-2.0
-taskflow>=3.1.0 # Apache-2.0
+taskflow>=3.7.1 # Apache-2.0
 WebOb>=1.8.5 # MIT
 WSME>=0.9.2 # MIT
-# NOTE(fdegir): NetworkX 2.3 dropped support for Python 2
+networkx>=2.2;python_version>='3.4' # BSD
-networkx>=1.11,<2.3;python_version<'3.0'  # BSD
-networkx>=1.11;python_version>='3.4' # BSD
 microversion_parse>=0.2.1 # Apache-2.0
 futurist>=1.8.0 # Apache-2.0

setup.cfg

@@ -6,6 +6,7 @@ description-file =
 author = OpenStack
 author-email = openstack-discuss@lists.openstack.org
 home-page = https://docs.openstack.org/watcher/latest/
+python-requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -13,8 +14,8 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
+    Programming Language :: Python :: Implementation :: CPython
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
@@ -25,10 +26,6 @@ packages =
 data_files =
     etc/ = etc/*
 
-[global]
-setup-hooks =
-    pbr.hooks.setup_hook
-
 [entry_points]
 oslo.config.opts =
     watcher = watcher.conf.opts:list_opts

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
     pbr=True)

test-requirements.txt

@@ -5,7 +5,7 @@
 coverage>=4.5.1 # Apache-2.0
 doc8>=0.8.0 # Apache-2.0
 freezegun>=0.3.10 # Apache-2.0
-hacking>=1.1.0,<1.2.0 # Apache-2.0
+hacking>=3.0,<3.1.0 # Apache-2.0
 mock>=2.0.0 # BSD
 oslotest>=3.3.0 # Apache-2.0
 os-testr>=1.0.0 # Apache-2.0

tox.ini

@@ -1,9 +1,11 @@
 [tox]
 minversion = 2.0
-envlist = py36,py37,py27,pep8
+envlist = py36,py37,pep8
 skipsdist = True
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 usedevelop = True
 whitelist_externals = find
                       rm
@@ -11,7 +13,7 @@ install_command = pip install {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train}
+    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/ussuri}
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
 commands =
@@ -21,24 +23,21 @@ commands =
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
 
 [testenv:pep8]
-basepython = python3
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
     bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:venv]
-basepython = python3
 setenv = PYTHONHASHSEED=0
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train}
+    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/ussuri}
     -r{toxinidir}/doc/requirements.txt
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
 commands = {posargs}
 
 [testenv:cover]
-basepython = python3
 setenv =
     PYTHON=coverage run --source watcher --parallel-mode
 commands =
@@ -49,51 +48,65 @@ commands =
     coverage report
 
 [testenv:docs]
-basepython = python3
 setenv = PYTHONHASHSEED=0
 deps = -r{toxinidir}/doc/requirements.txt
 commands =
   rm -fr doc/build doc/source/api/ .autogenerated
-  sphinx-build -W -b html doc/source doc/build/html
+  sphinx-build -W --keep-going -b html doc/source doc/build/html
 
 [testenv:api-ref]
-basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt
 whitelist_externals = bash
 commands =
   bash -c 'rm -rf api-ref/build'
-  sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
+  sphinx-build -W --keep-going -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
 
 [testenv:debug]
-basepython = python3
 commands = oslo_debug_helper -t watcher/tests {posargs}
 
 [testenv:genconfig]
-basepython = python3
 sitepackages = False
 commands =
     oslo-config-generator --config-file etc/watcher/oslo-config-generator/watcher.conf
 
 [testenv:genpolicy]
-basepython = python3
 commands =
     oslopolicy-sample-generator --config-file etc/watcher/oslo-policy-generator/watcher-policy-generator.conf
 
 [flake8]
 filename = *.py,app.wsgi
 show-source=True
-ignore= H105,E123,E226,N320,H202
+# W504 line break after binary operator
+ignore= H105,E123,E226,N320,H202,W504
 builtins= _
 enable-extensions = H106,H203,H904
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*,demo/,releasenotes
 
 [testenv:wheel]
-basepython = python3
 commands = python setup.py bdist_wheel
 
 [hacking]
 import_exceptions = watcher._i18n
-local-check-factory = watcher.hacking.checks.factory
+
+[flake8:local-plugins]
+extension =
+   N319 = checks:no_translate_debug_logs
+   N321 = checks:use_jsonutils
+   N322 = checks:check_assert_called_once_with
+   N325 = checks:check_python3_xrange
+   N326 = checks:check_no_basestring
+   N327 = checks:check_python3_no_iteritems
+   N328 = checks:check_asserttrue
+   N329 = checks:check_assertfalse
+   N330 = checks:check_assertempty
+   N331 = checks:check_assertisinstance
+   N332 = checks:check_assertequal_for_httpcode
+   N333 = checks:check_log_warn_deprecated
+   N340 = checks:check_oslo_i18n_wrapper
+   N341 = checks:check_builtins_gettext
+   N342 = checks:no_redundant_import_alias
+paths = ./watcher/hacking
+
 
 [doc8]
 extension=.rst
@@ -101,7 +114,6 @@ extension=.rst
 ignore-path=doc/source/image_src,doc/source/man,doc/source/api
 
 [testenv:pdf-docs]
-basepython = python3
 envdir = {toxworkdir}/docs
 deps = {[testenv:docs]deps}
 whitelist_externals =
@@ -109,21 +121,18 @@ whitelist_externals =
   make
 commands =
   rm -rf doc/build/pdf
-  sphinx-build -W -b latex doc/source doc/build/pdf
+  sphinx-build -W --keep-going -b latex doc/source doc/build/pdf
   make -C doc/build/pdf
 
 [testenv:releasenotes]
-basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
 
 [testenv:bandit]
-basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands = bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:lower-constraints]
-basepython = python3
 deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt

watcher/api/acl.py

@@ -37,5 +37,5 @@ def install(app, conf, public_routes):
     if not CONF.get('enable_authentication'):
         return app
     return auth_token.AuthTokenMiddleware(app,
-                                          conf=dict(conf),
+                                          conf=dict(conf.keystone_authtoken),
                                           public_api_routes=public_routes)

watcher/api/config.py

@@ -27,6 +27,10 @@ server = {
 
 # Pecan Application Configurations
 # See https://pecan.readthedocs.org/en/latest/configuration.html#application-configuration # noqa
+acl_public_routes = ['/']
+if not cfg.CONF.api.get("enable_webhooks_auth"):
+    acl_public_routes.append('/v1/webhooks/.*')
+
 app = {
     'root': 'watcher.api.controllers.root.RootController',
     'modules': ['watcher.api'],
@@ -36,9 +40,7 @@ app = {
     ],
     'static_root': '%(confdir)s/public',
     'enable_acl': True,
-    'acl_public_routes': [
+    'acl_public_routes': acl_public_routes,
-        '/',
-    ],
 }
 
 # WSME Configurations

watcher/api/controllers/link.py

@@ -23,7 +23,7 @@ from watcher.api.controllers import base
 
 def build_url(resource, resource_args, bookmark=False, base_url=None):
     if base_url is None:
-        base_url = pecan.request.host_url
+        base_url = pecan.request.application_url
 
     template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s'
     # FIXME(lucasagomes): I'm getting a 404 when doing a GET on

watcher/api/controllers/rest_api_version_history.rst

@@ -30,3 +30,12 @@ audits.
 ---
 Added ``force`` into create audit request. If ``force`` is true,
 audit will be executed despite of ongoing actionplan.
+
+1.3
+---
+Added list data model API.
+
+1.4
+---
+Added Watcher webhook API. It can be used to trigger audit
+with ``event`` type.

watcher/api/controllers/root.py

@@ -59,7 +59,8 @@ class Version(base.APIBase):
         version.status = status
         version.max_version = v.max_version_string()
         version.min_version = v.min_version_string()
-        version.links = [link.Link.make_link('self', pecan.request.host_url,
+        version.links = [link.Link.make_link('self',
+                                             pecan.request.application_url,
                                              id, '', bookmark=True)]
         return version
 

watcher/api/controllers/v1/__init__.py

@@ -40,7 +40,9 @@ from watcher.api.controllers.v1 import goal
 from watcher.api.controllers.v1 import scoring_engine
 from watcher.api.controllers.v1 import service
 from watcher.api.controllers.v1 import strategy
+from watcher.api.controllers.v1 import utils
 from watcher.api.controllers.v1 import versions
+from watcher.api.controllers.v1 import webhooks
 
 
 def min_version():
@@ -130,6 +132,9 @@ class V1(APIBase):
     services = [link.Link]
     """Links to the services resource"""
 
+    webhooks = [link.Link]
+    """Links to the webhooks resource"""
+
     links = [link.Link]
     """Links that point to a specific URL for this version and documentation"""
 
@@ -137,7 +142,8 @@ class V1(APIBase):
     def convert():
         v1 = V1()
         v1.id = "v1"
-        v1.links = [link.Link.make_link('self', pecan.request.host_url,
+        base_url = pecan.request.application_url
+        v1.links = [link.Link.make_link('self', base_url,
                                         'v1', '', bookmark=True),
                     link.Link.make_link('describedby',
                                         'http://docs.openstack.org',
@@ -148,57 +154,66 @@ class V1(APIBase):
         v1.media_types = [MediaType('application/json',
                           'application/vnd.openstack.watcher.v1+json')]
         v1.audit_templates = [link.Link.make_link('self',
-                                                  pecan.request.host_url,
+                                                  base_url,
                                                   'audit_templates', ''),
                               link.Link.make_link('bookmark',
-                                                  pecan.request.host_url,
+                                                  base_url,
                                                   'audit_templates', '',
                                                   bookmark=True)
                               ]
-        v1.audits = [link.Link.make_link('self', pecan.request.host_url,
+        v1.audits = [link.Link.make_link('self', base_url,
                                          'audits', ''),
                      link.Link.make_link('bookmark',
-                                         pecan.request.host_url,
+                                         base_url,
                                          'audits', '',
                                          bookmark=True)
                      ]
-        v1.data_model = [link.Link.make_link('self', pecan.request.host_url,
+        if utils.allow_list_datamodel():
-                                             'data_model', ''),
+            v1.data_model = [link.Link.make_link('self', base_url,
-                         link.Link.make_link('bookmark',
+                                                 'data_model', ''),
-                                             pecan.request.host_url,
+                             link.Link.make_link('bookmark',
-                                             'data_model', '',
+                                                 base_url,
-                                             bookmark=True)
+                                                 'data_model', '',
-                         ]
+                                                 bookmark=True)
-        v1.actions = [link.Link.make_link('self', pecan.request.host_url,
+                             ]
+        v1.actions = [link.Link.make_link('self', base_url,
                                           'actions', ''),
                       link.Link.make_link('bookmark',
-                                          pecan.request.host_url,
+                                          base_url,
                                           'actions', '',
                                           bookmark=True)
                       ]
         v1.action_plans = [link.Link.make_link(
-            'self', pecan.request.host_url, 'action_plans', ''),
+            'self', base_url, 'action_plans', ''),
             link.Link.make_link('bookmark',
-                                pecan.request.host_url,
+                                base_url,
                                 'action_plans', '',
                                 bookmark=True)
             ]
 
         v1.scoring_engines = [link.Link.make_link(
-            'self', pecan.request.host_url, 'scoring_engines', ''),
+            'self', base_url, 'scoring_engines', ''),
             link.Link.make_link('bookmark',
-                                pecan.request.host_url,
+                                base_url,
                                 'scoring_engines', '',
                                 bookmark=True)
             ]
 
         v1.services = [link.Link.make_link(
-            'self', pecan.request.host_url, 'services', ''),
+            'self', base_url, 'services', ''),
             link.Link.make_link('bookmark',
-                                pecan.request.host_url,
+                                base_url,
                                 'services', '',
                                 bookmark=True)
             ]
+        if utils.allow_webhook_api():
+            v1.webhooks = [link.Link.make_link(
+                'self', base_url, 'webhooks', ''),
+                link.Link.make_link('bookmark',
+                                    base_url,
+                                    'webhooks', '',
+                                    bookmark=True)
+                ]
         return v1
 
 
@@ -214,6 +229,7 @@ class Controller(rest.RestController):
     services = service.ServicesController()
     strategies = strategy.StrategiesController()
     data_model = data_model.DataModelController()
+    webhooks = webhooks.WebhookController()
 
     @wsme_pecan.wsexpose(V1)
     def get(self):

watcher/api/controllers/v1/action_plan.py

@@ -165,7 +165,7 @@ class ActionPlan(base.APIBase):
                         name=indicator.name,
                         description=indicator.description,
                         unit=indicator.unit,
-                        value=indicator.value,
+                        value=float(indicator.value),
                     )
                     efficacy_indicators.append(efficacy_indicator.as_dict())
                 self._efficacy_indicators = efficacy_indicators

watcher/api/controllers/v1/audit_template.py

@@ -138,6 +138,9 @@ class AuditTemplatePostType(wtypes.Base):
             raise exception.InvalidGoal(goal=audit_template.goal)
 
         if audit_template.scope:
+            keys = [list(s)[0] for s in audit_template.scope]
+            if keys[0] not in ('compute', 'storage'):
+                audit_template.scope = [dict(compute=audit_template.scope)]
             common_utils.Draft4Validator(
                 AuditTemplatePostType._build_schema()
                 ).validate(audit_template.scope)
@@ -158,18 +161,23 @@ class AuditTemplatePostType(wtypes.Base):
                         "included and excluded together"))
 
         if audit_template.strategy:
-            available_strategies = objects.Strategy.list(
+            try:
-                AuditTemplatePostType._ctx)
+                if (common_utils.is_uuid_like(audit_template.strategy) or
-            available_strategies_map = {
+                        common_utils.is_int_like(audit_template.strategy)):
-                s.uuid: s for s in available_strategies}
+                    strategy = objects.Strategy.get(
-            if audit_template.strategy not in available_strategies_map:
+                        AuditTemplatePostType._ctx, audit_template.strategy)
+                else:
+                    strategy = objects.Strategy.get_by_name(
+                        AuditTemplatePostType._ctx, audit_template.strategy)
+            except Exception:
                 raise exception.InvalidStrategy(
                     strategy=audit_template.strategy)
 
-            strategy = available_strategies_map[audit_template.strategy]
             # Check that the strategy we indicate is actually related to the
             # specified goal
             if strategy.goal_id != goal.id:
+                available_strategies = objects.Strategy.list(
+                    AuditTemplatePostType._ctx)
                 choices = ["'%s' (%s)" % (s.uuid, s.name)
                            for s in available_strategies]
                 raise exception.InvalidStrategy(

watcher/api/controllers/v1/data_model.py

@@ -24,6 +24,7 @@ from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
 from watcher.api.controllers.v1 import types
+from watcher.api.controllers.v1 import utils
 from watcher.common import exception
 from watcher.common import policy
 from watcher.decision_engine import rpcapi
@@ -49,6 +50,8 @@ class DataModelController(rest.RestController):
         :param audit_uuid: The UUID of the audit,  used to filter data model
                            by the scope in audit.
         """
+        if not utils.allow_list_datamodel():
+            raise exception.NotAcceptable
         if self.from_data_model:
             raise exception.OperationNotPermitted
         allowed_data_model_type = [

watcher/api/controllers/v1/types.py

@@ -184,7 +184,7 @@ class MultiType(wtypes.UserType):
 class JsonPatchType(wtypes.Base):
     """A complex type that represents a single json-patch operation."""
 
-    path = wtypes.wsattr(wtypes.StringType(pattern='^(/[\w-]+)+$'),
+    path = wtypes.wsattr(wtypes.StringType(pattern=r'^(/[\w-]+)+$'),
                          mandatory=True)
     op = wtypes.wsattr(wtypes.Enum(str, 'add', 'replace', 'remove'),
                        mandatory=True)

watcher/api/controllers/v1/utils.py

@@ -164,7 +164,8 @@ def allow_start_end_audit_time():
     Version 1.1 of the API added support for start and end time of continuous
     audits.
     """
-    return pecan.request.version.minor >= versions.MINOR_1_START_END_TIMING
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_1_START_END_TIMING.value)
 
 
 def allow_force():
@@ -173,4 +174,23 @@ def allow_force():
     Version 1.2 of the API added support for forced audits that allows to
     launch audit when other action plan is ongoing.
     """
-    return pecan.request.version.minor >= versions.MINOR_2_FORCE
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_2_FORCE.value)
+
+
+def allow_list_datamodel():
+    """Check if we should support list data model API.
+
+    Version 1.3 of the API added support to list data model.
+    """
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_3_DATAMODEL.value)
+
+
+def allow_webhook_api():
+    """Check if we should support webhook API.
+
+    Version 1.4 of the API added support to trigger webhook.
+    """
+    return pecan.request.version.minor >= (
+        versions.VERSIONS.MINOR_4_WEBHOOK_API.value)

watcher/api/controllers/v1/versions.py

@@ -14,25 +14,25 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import enum
+
+
+class VERSIONS(enum.Enum):
+    MINOR_0_ROCKY = 0  # v1.0: corresponds to Rocky API
+    MINOR_1_START_END_TIMING = 1  # v1.1: Add start/end timei for audit
+    MINOR_2_FORCE = 2  # v1.2: Add force field to audit
+    MINOR_3_DATAMODEL = 3  # v1.3: Add list datamodel API
+    MINOR_4_WEBHOOK_API = 4  # v1.4: Add webhook trigger API
+    MINOR_MAX_VERSION = 4
+
 
 # This is the version 1 API
 BASE_VERSION = 1
-
-# Here goes a short log of changes in every version.
-#
-# v1.0: corresponds to Rocky API
-# v1.1: Add start/end time for continuous audit
-# v1.2: Add force field to audit
-
-MINOR_0_ROCKY = 0
-MINOR_1_START_END_TIMING = 1
-MINOR_2_FORCE = 2
-
-MINOR_MAX_VERSION = MINOR_2_FORCE
-
 # String representations of the minor and maximum versions
-_MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_0_ROCKY)
+_MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION,
-_MAX_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_MAX_VERSION)
+                                     VERSIONS.MINOR_0_ROCKY.value)
+_MAX_VERSION_STRING = '{}.{}'.format(BASE_VERSION,
+                                     VERSIONS.MINOR_MAX_VERSION.value)
 
 
 def service_type_string():

watcher/api/controllers/v1/webhooks.py

@@ -0,0 +1,62 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Webhook endpoint for Watcher v1 REST API.
+"""
+
+from oslo_log import log
+import pecan
+from pecan import rest
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from watcher.api.controllers.v1 import types
+from watcher.api.controllers.v1 import utils
+from watcher.common import exception
+from watcher.decision_engine import rpcapi
+from watcher import objects
+
+LOG = log.getLogger(__name__)
+
+
+class WebhookController(rest.RestController):
+    """REST controller for webhooks resource."""
+    def __init__(self):
+        super(WebhookController, self).__init__()
+        self.dc_client = rpcapi.DecisionEngineAPI()
+
+    @wsme_pecan.wsexpose(None, wtypes.text, body=types.jsontype,
+                         status_code=202)
+    def post(self, audit_ident, body):
+        """Trigger the given audit.
+
+        :param audit_ident: UUID or name of an audit.
+        """
+
+        LOG.debug("Webhook trigger Audit: %s.", audit_ident)
+
+        context = pecan.request.context
+        audit = utils.get_resource('Audit', audit_ident)
+        if audit is None:
+            raise exception.AuditNotFound(audit=audit_ident)
+        if audit.audit_type != objects.audit.AuditType.EVENT.value:
+            raise exception.AuditTypeNotAllowed(audit_type=audit.audit_type)
+        allowed_state = (
+            objects.audit.State.PENDING,
+            objects.audit.State.SUCCEEDED,
+            )
+        if audit.state not in allowed_state:
+            raise exception.AuditStateNotAllowed(state=audit.state)
+
+        # trigger decision-engine to run the audit
+        self.dc_client.trigger_audit(context, audit.uuid)

watcher/api/middleware/auth_token.py

@@ -34,7 +34,7 @@ class AuthTokenMiddleware(auth_token.AuthProtocol):
 
     """
     def __init__(self, app, conf, public_api_routes=()):
-        route_pattern_tpl = '%s(\.json|\.xml)?$'
+        route_pattern_tpl = r'%s(\.json|\.xml)?$'
 
         try:
             self.public_api_routes = [re.compile(route_pattern_tpl % route_tpl)

watcher/applier/actions/base.py

@@ -140,7 +140,7 @@ class BaseAction(loadable.Loadable):
         raise NotImplementedError()
 
     def check_abort(self):
-        if self.__class__.__name__ is 'Migrate':
+        if self.__class__.__name__ == 'Migrate':
             if self.migration_type == self.LIVE_MIGRATION:
                 return True
             else:

watcher/applier/actions/resize.py

@@ -47,24 +47,24 @@ class Resize(base.BaseAction):
 
     @property
     def schema(self):
-            return {
+        return {
-                'type': 'object',
+            'type': 'object',
-                'properties': {
+            'properties': {
-                    'resource_id': {
+                'resource_id': {
-                        'type': 'string',
+                    'type': 'string',
-                        'minlength': 1,
+                    'minlength': 1,
-                        'pattern': ('^([a-fA-F0-9]){8}-([a-fA-F0-9]){4}-'
+                    'pattern': ('^([a-fA-F0-9]){8}-([a-fA-F0-9]){4}-'
-                                    '([a-fA-F0-9]){4}-([a-fA-F0-9]){4}-'
+                                '([a-fA-F0-9]){4}-([a-fA-F0-9]){4}-'
-                                    '([a-fA-F0-9]){12}$')
+                                '([a-fA-F0-9]){12}$')
-                    },
-                    'flavor': {
-                        'type': 'string',
-                        'minlength': 1,
-                    },
                 },
-                'required': ['resource_id', 'flavor'],
+                'flavor': {
-                'additionalProperties': False,
+                    'type': 'string',
-            }
+                    'minlength': 1,
+                },
+            },
+            'required': ['resource_id', 'flavor'],
+            'additionalProperties': False,
+        }
 
     @property
     def instance_uuid(self):

watcher/applier/workflow_engine/default.py

@@ -112,7 +112,7 @@ class DefaultWorkFlowEngine(base.BaseWorkFlowEngine):
 
             return flow
 
-        except exception.ActionPlanCancelled as e:
+        except exception.ActionPlanCancelled:
             raise
 
         except tf_exception.WrappedFailure as e:

watcher/cmd/__init__.py

@@ -18,3 +18,10 @@
 import eventlet
 
 eventlet.monkey_patch()
+
+# Monkey patch the original current_thread to use the up-to-date _active
+# global variable. See https://bugs.launchpad.net/bugs/1863021 and
+# https://github.com/eventlet/eventlet/issues/592
+import __original_module_threading as orig_threading  # noqa
+import threading  # noqa
+orig_threading.current_thread.__globals__['_active'] = threading._active

watcher/cmd/status.py

@@ -15,7 +15,6 @@
 import sys
 
 from oslo_upgradecheck import upgradecheck
-import six
 
 from watcher._i18n import _
 from watcher.common import clients
@@ -38,7 +37,7 @@ class Checks(upgradecheck.UpgradeCommands):
             clients.check_min_nova_api_version(CONF.nova_client.api_version)
         except ValueError as e:
             return upgradecheck.Result(
-                upgradecheck.Code.FAILURE, six.text_type(e))
+                upgradecheck.Code.FAILURE, str(e))
         return upgradecheck.Result(upgradecheck.Code.SUCCESS)
 
     _upgrade_checks = (

watcher/common/exception.py

@@ -28,7 +28,6 @@ import sys
 from keystoneclient import exceptions as keystone_exceptions
 from oslo_config import cfg
 from oslo_log import log
-import six
 
 from watcher._i18n import _
 
@@ -97,19 +96,16 @@ class WatcherException(Exception):
 
     def __str__(self):
         """Encode to utf-8 then wsme api can consume it as well"""
-        if not six.PY3:
+        return self.args[0]
-            return six.text_type(self.args[0]).encode('utf-8')
-        else:
-            return self.args[0]
 
     def __unicode__(self):
-        return six.text_type(self.args[0])
+        return str(self.args[0])
 
     def format_message(self):
         if self.__class__.__name__.endswith('_Remote'):
             return self.args[0]
         else:
-            return six.text_type(self)
+            return str(self)
 
 
 class UnsupportedError(WatcherException):
@@ -243,6 +239,14 @@ class AuditTypeNotFound(Invalid):
     msg_fmt = _("Audit type %(audit_type)s could not be found")
 
 
+class AuditTypeNotAllowed(Invalid):
+    msg_fmt = _("Audit type %(audit_type)s is disallowed.")
+
+
+class AuditStateNotAllowed(Invalid):
+    msg_fmt = _("Audit state %(state)s is disallowed.")
+
+
 class AuditParameterNotAllowed(Invalid):
     msg_fmt = _("Audit parameter %(parameter)s are not allowed")
 

watcher/common/scheduling.py

@@ -37,6 +37,7 @@ class GreenThreadPoolExecutor(BasePoolExecutor):
         pool = futurist.GreenThreadPoolExecutor(int(max_workers))
         super(GreenThreadPoolExecutor, self).__init__(pool)
 
+
 executors = {
     'default': GreenThreadPoolExecutor(),
 }

watcher/common/utils.py

@@ -153,6 +153,7 @@ def extend_with_strict_schema(validator_class):
 
     return validators.extend(validator_class, {"properties": strict_schema})
 
+
 StrictDefaultValidatingDraft4Validator = extend_with_default(
     extend_with_strict_schema(validators.Draft4Validator))
 

watcher/conf/api.py

@@ -55,6 +55,11 @@ API_SERVICE_OPTS = [
                      "the service, this option should be False; note, you "
                      "will want to change public API endpoint to represent "
                      "SSL termination URL with 'public_endpoint' option."),
+
+    cfg.BoolOpt('enable_webhooks_auth',
+                default=True,
+                help='This option enables or disables webhook request '
+                     'authentication via keystone. Default value is True.'),
 ]
 
 

watcher/conf/decision_engine.py

@@ -40,11 +40,18 @@ WATCHER_DECISION_ENGINE_OPTS = [
                default='watcher.decision.api',
                help='The identifier used by the Watcher '
                     'module on the message broker'),
-    cfg.IntOpt('max_workers',
+    cfg.IntOpt('max_audit_workers',
                default=2,
                required=True,
                help='The maximum number of threads that can be used to '
-                    'execute strategies'),
+                    'execute audits in parallel.'),
+    cfg.IntOpt('max_general_workers',
+               default=4,
+               required=True,
+               help='The maximum number of threads that can be used to '
+                    'execute general tasks in parallel. The number of general '
+                    'workers will not increase depending on the number of '
+                    'audit workers!'),
     cfg.IntOpt('action_plan_expiry',
                default=24,
                mutable=True,

watcher/db/sqlalchemy/api.py

@@ -1125,8 +1125,8 @@ class Connection(api.BaseConnection):
 
     def get_action_description_by_id(self, context,
                                      action_id, eager=False):
-                return self._get_action_description(
+        return self._get_action_description(
-                    context, fieldname="id", value=action_id, eager=eager)
+            context, fieldname="id", value=action_id, eager=eager)
 
     def get_action_description_by_type(self, context,
                                        action_type, eager=False):

watcher/decision_engine/audit/event.py

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 ZTE Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from watcher.decision_engine.audit import base
+from watcher import objects
+
+
+class EventAuditHandler(base.AuditHandler):
+
+    def post_execute(self, audit, solution, request_context):
+        super(EventAuditHandler, self).post_execute(audit, solution,
+                                                    request_context)
+        # change state of the audit to SUCCEEDED
+        self.update_audit_state(audit, objects.audit.State.SUCCEEDED)

watcher/decision_engine/datasources/base.py

@@ -19,8 +19,6 @@ import time
 from oslo_config import cfg
 from oslo_log import log
 
-from watcher.common import exception
-
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
@@ -79,7 +77,6 @@ class DataSourceBase(object):
                 LOG.warning("Retry {0} of {1} while retrieving metrics retry "
                             "in {2} seconds".format(i+1, num_retries, timeout))
                 time.sleep(timeout)
-        raise exception.DataSourceNotAvailable(datasource=self.NAME)
 
     @abc.abstractmethod
     def query_retry_reset(self, exception_instance):

watcher/decision_engine/datasources/ceilometer.py

@@ -136,19 +136,18 @@ class CeilometerHelper(base.DataSourceBase):
 
     def list_metrics(self):
         """List the user's meters."""
-        try:
+        meters = self.query_retry(f=self.ceilometer.meters.list)
-            meters = self.query_retry(f=self.ceilometer.meters.list)
+        if not meters:
-        except Exception:
             return set()
         else:
             return meters
 
     def check_availability(self):
-        try:
+        status = self.query_retry(self.ceilometer.resources.list)
-            self.query_retry(self.ceilometer.resources.list)
+        if status:
-        except Exception:
+            return 'available'
+        else:
             return 'not available'
-        return 'available'
 
     def query_sample(self, meter_name, query, limit=1):
         return self.query_retry(f=self.ceilometer.samples.list,
@@ -189,7 +188,7 @@ class CeilometerHelper(base.DataSourceBase):
         item_value = None
         if statistic:
             item_value = statistic[-1]._info.get('aggregate').get(aggregate)
-            if meter_name is 'host_airflow':
+            if meter_name == 'host_airflow':
                 # Airflow from hardware.ipmi.node.airflow is reported as
                 # 1/10 th of actual CFM
                 item_value *= 10

watcher/decision_engine/datasources/gnocchi.py

@@ -52,17 +52,16 @@ class GnocchiHelper(base.DataSourceBase):
         self.gnocchi = self.osc.gnocchi()
 
     def check_availability(self):
-        try:
+        status = self.query_retry(self.gnocchi.status.get)
-            self.query_retry(self.gnocchi.status.get)
+        if status:
-        except Exception:
+            return 'available'
+        else:
             return 'not available'
-        return 'available'
 
     def list_metrics(self):
         """List the user's meters."""
-        try:
+        response = self.query_retry(f=self.gnocchi.metric.list)
-            response = self.query_retry(f=self.gnocchi.metric.list)
+        if not response:
-        except Exception:
             return set()
         else:
             return set([metric['name'] for metric in response])
@@ -91,8 +90,9 @@ class GnocchiHelper(base.DataSourceBase):
                 f=self.gnocchi.resource.search, **kwargs)
 
             if not resources:
-                raise exception.ResourceNotFound(name='gnocchi',
+                LOG.warning("The {0} resource {1} could not be "
-                                                 id=resource_id)
+                            "found".format(self.NAME, resource_id))
+                return
 
             resource_id = resources[0]['id']
 
@@ -110,17 +110,18 @@ class GnocchiHelper(base.DataSourceBase):
         statistics = self.query_retry(
             f=self.gnocchi.metric.get_measures, **kwargs)
 
+        return_value = None
         if statistics:
             # return value of latest measure
             # measure has structure [time, granularity, value]
             return_value = statistics[-1][2]
 
-            if meter_name is 'host_airflow':
+            if meter_name == 'host_airflow':
                 # Airflow from hardware.ipmi.node.airflow is reported as
                 # 1/10 th of actual CFM
                 return_value *= 10
 
-            return return_value
+        return return_value
 
     def get_host_cpu_usage(self, resource, period, aggregate,
                            granularity=300):

watcher/decision_engine/datasources/grafana.py

@@ -72,7 +72,7 @@ class GrafanaHelper(base.DataSourceBase):
 
         # Very basic url parsing
         parse = urlparse.urlparse(self._base_url)
-        if parse.scheme is '' or parse.netloc is '' or parse.path is '':
+        if parse.scheme == '' or parse.netloc == '' or parse.path == '':
             LOG.critical("GrafanaHelper url not properly configured, "
                          "check base_url and project_id")
             return
@@ -179,6 +179,9 @@ class GrafanaHelper(base.DataSourceBase):
         kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
 
         resp = self.query_retry(self._request, **kwargs)
+        if not resp:
+            LOG.warning("Datasource {0} is not available.".format(self.NAME))
+            return
 
         result = translator.extract_result(resp.content)
 

watcher/decision_engine/datasources/manager.py

@@ -112,10 +112,10 @@ class DataSourceManager(object):
         datasource is attempted.
         """
 
-        if not self.datasources or len(self.datasources) is 0:
+        if not self.datasources or len(self.datasources) == 0:
             raise exception.NoDatasourceAvailable
 
-        if not metrics or len(metrics) is 0:
+        if not metrics or len(metrics) == 0:
             LOG.critical("Can not retrieve datasource without specifying "
                          "list of required metrics.")
             raise exception.InvalidParameter(parameter='metrics',
@@ -125,11 +125,11 @@ class DataSourceManager(object):
             no_metric = False
             for metric in metrics:
                 if (metric not in self.metric_map[datasource] or
-                   self.metric_map[datasource].get(metric) is None):
+                        self.metric_map[datasource].get(metric) is None):
-                        no_metric = True
+                    no_metric = True
-                        LOG.warning("Datasource: {0} could not be used due to "
+                    LOG.warning("Datasource: {0} could not be used due to "
-                                    "metric: {1}".format(datasource, metric))
+                                "metric: {1}".format(datasource, metric))
-                        break
+                    break
             if not no_metric:
                 # Try to use a specific datasource but attempt additional
                 # datasources upon exceptions (if config has more datasources)

watcher/decision_engine/datasources/monasca.py

@@ -73,11 +73,11 @@ class MonascaHelper(base.DataSourceBase):
             self.monasca = self.osc.monasca()
 
     def check_availability(self):
-        try:
+        result = self.query_retry(self.monasca.metrics.list)
-            self.query_retry(self.monasca.metrics.list)
+        if result:
-        except Exception:
+            return 'available'
+        else:
             return 'not available'
-        return 'available'
 
     def list_metrics(self):
         # TODO(alexchadin): this method should be implemented in accordance to

watcher/decision_engine/messaging/audit_endpoint.py

@@ -22,6 +22,7 @@ from oslo_config import cfg
 from oslo_log import log
 
 from watcher.decision_engine.audit import continuous as c_handler
+from watcher.decision_engine.audit import event as e_handler
 from watcher.decision_engine.audit import oneshot as o_handler
 
 from watcher import objects
@@ -35,9 +36,10 @@ class AuditEndpoint(object):
     def __init__(self, messaging):
         self._messaging = messaging
         self._executor = futurist.GreenThreadPoolExecutor(
-            max_workers=CONF.watcher_decision_engine.max_workers)
+            max_workers=CONF.watcher_decision_engine.max_audit_workers)
         self._oneshot_handler = o_handler.OneShotAuditHandler()
         self._continuous_handler = c_handler.ContinuousAuditHandler().start()
+        self._event_handler = e_handler.EventAuditHandler()
 
     @property
     def executor(self):
@@ -45,7 +47,10 @@ class AuditEndpoint(object):
 
     def do_trigger_audit(self, context, audit_uuid):
         audit = objects.Audit.get_by_uuid(context, audit_uuid, eager=True)
-        self._oneshot_handler.execute(audit, context)
+        if audit.audit_type == objects.audit.AuditType.ONESHOT.value:
+            self._oneshot_handler.execute(audit, context)
+        if audit.audit_type == objects.audit.AuditType.EVENT.value:
+            self._event_handler.execute(audit, context)
 
     def trigger_audit(self, context, audit_uuid):
         LOG.debug("Trigger audit %s", audit_uuid)

watcher/decision_engine/model/collector/cinder.py

@@ -152,6 +152,9 @@ class CinderClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build storage data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build storage data model")
+            return
 
         builder = CinderModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/decision_engine/model/collector/ironic.py

@@ -63,6 +63,9 @@ class BaremetalClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build Baremetal data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build Baremetal data model")
+            return
 
         builder = BareMetalModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)

watcher/decision_engine/model/collector/nova.py

@@ -16,6 +16,8 @@
 import os_resource_classes as orc
 from oslo_log import log
 
+from futurist import waiters
+
 from watcher.common import nova_helper
 from watcher.common import placement_helper
 from watcher.decision_engine.model.collector import base
@@ -23,6 +25,7 @@ from watcher.decision_engine.model import element
 from watcher.decision_engine.model import model_root
 from watcher.decision_engine.model.notification import nova
 from watcher.decision_engine.scope import compute as compute_scope
+from watcher.decision_engine import threading
 
 LOG = log.getLogger(__name__)
 
@@ -181,6 +184,9 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
         if self._audit_scope_handler is None:
             LOG.debug("No audit, Don't Build compute data model")
             return
+        if self._data_model_scope is None:
+            LOG.debug("No audit scope, Don't Build compute data model")
+            return
 
         builder = NovaModelBuilder(self.osc)
         return builder.execute(self._data_model_scope)
@@ -212,8 +218,12 @@ class NovaModelBuilder(base.BaseModelBuilder):
         self.nova = osc.nova()
         self.nova_helper = nova_helper.NovaHelper(osc=self.osc)
         self.placement_helper = placement_helper.PlacementHelper(osc=self.osc)
+        self.executor = threading.DecisionEngineThreadPool()
 
     def _collect_aggregates(self, host_aggregates, _nodes):
+        if not host_aggregates:
+            return
+
         aggregate_list = self.call_retry(f=self.nova_helper.get_aggregate_list)
         aggregate_ids = [aggregate['id'] for aggregate
                          in host_aggregates if 'id' in aggregate]
@@ -229,6 +239,9 @@ class NovaModelBuilder(base.BaseModelBuilder):
                 _nodes.update(aggregate.hosts)
 
     def _collect_zones(self, availability_zones, _nodes):
+        if not availability_zones:
+            return
+
         service_list = self.call_retry(f=self.nova_helper.get_service_list)
         zone_names = [zone['name'] for zone
                       in availability_zones]
@@ -239,20 +252,71 @@ class NovaModelBuilder(base.BaseModelBuilder):
             if service.zone in zone_names or include_all_nodes:
                 _nodes.add(service.host)
 
-    def _add_physical_layer(self):
+    def _compute_node_future(self, future, future_instances):
-        """Add the physical layer of the graph.
+        """Add compute node information to model and schedule instance info job
 
-        This includes components which represent actual infrastructure
+        :param future: The future from the finished execution
-        hardware.
+        :rtype future: :py:class:`futurist.GreenFuture`
+        :param future_instances: list of futures for instance jobs
+        :rtype future_instances:  list :py:class:`futurist.GreenFuture`
         """
+        try:
+            node_info = future.result()[0]
+
+            # filter out baremetal node
+            if node_info.hypervisor_type == 'ironic':
+                LOG.debug("filtering out baremetal node: %s", node_info)
+                return
+            self.add_compute_node(node_info)
+            # node.servers is a list of server objects
+            # New in nova version 2.53
+            instances = getattr(node_info, "servers", None)
+            # Do not submit job if there are no instances on compute node
+            if instances is None:
+                LOG.info("No instances on compute_node: {0}".format(node_info))
+                return
+            future_instances.append(
+                self.executor.submit(
+                    self.add_instance_node, node_info, instances)
+            )
+        except Exception:
+            LOG.error("compute node from aggregate / "
+                      "availability_zone could not be found")
+
+    def _add_physical_layer(self):
+        """Collects all information on compute nodes and instances
+
+        Will collect all required compute node and instance information based
+        on the host aggregates and availability zones. If aggregates and zones
+        do not specify any compute nodes all nodes are retrieved instead.
+
+        The collection of information happens concurrently using the
+        DecisionEngineThreadpool. The collection is parallelized in three steps
+        first information about aggregates and zones is gathered. Secondly,
+        for each of the compute nodes a tasks is submitted to get detailed
+        information about the compute node. Finally, Each of these submitted
+        tasks will submit an additional task if the compute node contains
+        instances. Before returning from this function all instance tasks are
+        waited upon to complete.
+        """
+
         compute_nodes = set()
         host_aggregates = self.model_scope.get("host_aggregates")
         availability_zones = self.model_scope.get("availability_zones")
-        if host_aggregates:
-            self._collect_aggregates(host_aggregates, compute_nodes)
-        if availability_zones:
-            self._collect_zones(availability_zones, compute_nodes)
 
+        """Submit tasks to gather compute nodes from availability zones and
+        host aggregates. Each task adds compute nodes to the set, this set is
+        threadsafe under the assumption that CPython is used with the GIL
+        enabled."""
+        zone_aggregate_futures = {
+            self.executor.submit(
+                self._collect_aggregates, host_aggregates, compute_nodes),
+            self.executor.submit(
+                self._collect_zones, availability_zones, compute_nodes)
+        }
+        waiters.wait_for_all(zone_aggregate_futures)
+
+        # if zones and aggregates did not contain any nodes get every node.
         if not compute_nodes:
             self.no_model_scope_flag = True
             all_nodes = self.call_retry(
@@ -260,24 +324,20 @@ class NovaModelBuilder(base.BaseModelBuilder):
             compute_nodes = set(
                 [node.hypervisor_hostname for node in all_nodes])
         LOG.debug("compute nodes: %s", compute_nodes)
-        for node_name in compute_nodes:
+
-            cnode = self.call_retry(
+        node_futures = [self.executor.submit(
-                self.nova_helper.get_compute_node_by_name,
+            self.nova_helper.get_compute_node_by_name,
-                node_name, servers=True, detailed=True)
+            node, servers=True, detailed=True)
-            if cnode:
+            for node in compute_nodes]
-                node_info = cnode[0]
+        LOG.debug("submitted {0} jobs".format(len(compute_nodes)))
-                # filter out baremetal node
+
-                if node_info.hypervisor_type == 'ironic':
+        # Futures will concurrently be added, only safe with CPython GIL
-                    LOG.debug("filtering out baremetal node: %s", node_name)
+        future_instances = []
-                    continue
+        self.executor.do_while_futures_modify(
-                self.add_compute_node(node_info)
+            node_futures, self._compute_node_future, future_instances)
-                # node.servers is a list of server objects
+
-                # New in nova version 2.53
+        # Wait for all instance jobs to finish
-                instances = getattr(node_info, "servers", None)
+        waiters.wait_for_all(future_instances)
-                self.add_instance_node(node_info, instances)
-            else:
-                LOG.error("compute_node from aggregate / availability_zone "
-                          "could not be found: {0}".format(node_name))
 
     def add_compute_node(self, node):
         # Build and add base node.

watcher/decision_engine/model/model_root.py

@@ -171,7 +171,7 @@ class ModelRoot(nx.DiGraph, base.Model):
 
     def _get_by_uuid(self, uuid):
         try:
-            return self.node[uuid]['attr']
+            return self.nodes[uuid]['attr']
         except Exception as exc:
             LOG.exception(exc)
             raise exception.ComputeResourceNotFound(name=uuid)
@@ -466,14 +466,14 @@ class StorageModelRoot(nx.DiGraph, base.Model):
 
     def _get_by_uuid(self, uuid):
         try:
-            return self.node[uuid]['attr']
+            return self.nodes[uuid]['attr']
         except Exception as exc:
             LOG.exception(exc)
             raise exception.StorageResourceNotFound(name=uuid)
 
     def _get_by_name(self, name):
         try:
-            return self.node[name]['attr']
+            return self.nodes[name]['attr']
         except Exception as exc:
             LOG.exception(exc)
             raise exception.StorageResourceNotFound(name=name)
@@ -648,7 +648,7 @@ class BaremetalModelRoot(nx.DiGraph, base.Model):
 
     def _get_by_uuid(self, uuid):
         try:
-            return self.node[uuid]['attr']
+            return self.nodes[uuid]['attr']
         except Exception as exc:
             LOG.exception(exc)
         raise exception.BaremetalResourceNotFound(name=uuid)

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -201,19 +201,13 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
         LOG.debug('Migrate instance %s from %s to  %s',
                   instance_to_migrate, source_node, destination_node)
 
-        total_cores = 0
+        used_resources = self.compute_model.get_node_used_resources(
-        total_disk = 0
+            destination_node)
-        total_mem = 0
-        for instance in self.compute_model.get_node_instances(
-                destination_node):
-            total_cores += instance.vcpus
-            total_disk += instance.disk
-            total_mem += instance.memory
 
         # capacity requested by the compute node
-        total_cores += instance_to_migrate.vcpus
+        total_cores = used_resources['vcpu'] + instance_to_migrate.vcpus
-        total_disk += instance_to_migrate.disk
+        total_disk = used_resources['disk'] + instance_to_migrate.disk
-        total_mem += instance_to_migrate.memory
+        total_mem = used_resources['memory'] + instance_to_migrate.memory
 
         return self.check_threshold(destination_node, total_cores, total_disk,
                                     total_mem)
@@ -407,7 +401,7 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
         self._pre_execute()
 
         # backwards compatibility for node parameter.
-        if self.aggregation_method['node'] is not '':
+        if self.aggregation_method['node'] != '':
             LOG.warning('Parameter node has been renamed to compute_node and '
                         'will be removed in next release.')
             self.aggregation_method['compute_node'] = \

watcher/decision_engine/strategy/strategies/host_maintenance.py

@@ -137,37 +137,6 @@ class HostMaintenance(base.HostMaintenanceBaseStrategy):
                     ram=node.memory_mb_capacity,
                     disk=node.disk_gb_capacity)
 
-    def get_node_used(self, node):
-        """Collect cpu, ram and disk used of a node.
-
-        :param node: node object
-        :return: dict(cpu(cores), ram(MB), disk(B))
-        """
-        vcpus_used = 0
-        memory_used = 0
-        disk_used = 0
-        for instance in self.compute_model.get_node_instances(node):
-            vcpus_used += instance.vcpus
-            memory_used += instance.memory
-            disk_used += instance.disk
-
-        return dict(cpu=vcpus_used,
-                    ram=memory_used,
-                    disk=disk_used)
-
-    def get_node_free(self, node):
-        """Collect cpu, ram and disk free of a node.
-
-        :param node: node object
-        :return: dict(cpu(cores), ram(MB), disk(B))
-        """
-        node_capacity = self.get_node_capacity(node)
-        node_used = self.get_node_used(node)
-        return dict(cpu=node_capacity['cpu']-node_used['cpu'],
-                    ram=node_capacity['ram']-node_used['ram'],
-                    disk=node_capacity['disk']-node_used['disk'],
-                    )
-
     def host_fits(self, source_node, destination_node):
         """check host fits
 
@@ -175,9 +144,11 @@ class HostMaintenance(base.HostMaintenanceBaseStrategy):
         from source_node to destination_node.
         """
 
-        source_node_used = self.get_node_used(source_node)
+        source_node_used = self.compute_model.get_node_used_resources(
-        destination_node_free = self.get_node_free(destination_node)
+            source_node)
-        metrics = ['cpu', 'ram']
+        destination_node_free = self.compute_model.get_node_free_resources(
+            destination_node)
+        metrics = ['vcpu', 'memory']
         for m in metrics:
             if source_node_used[m] > destination_node_free[m]:
                 return False

watcher/decision_engine/strategy/strategies/noisy_neighbor.py

@@ -208,19 +208,6 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
 
         return hosts_need_release, hosts_target
 
-    def calc_used_resource(self, node):
-        """Calculate the used vcpus, memory and disk based on VM flavors"""
-        instances = self.compute_model.get_node_instances(node)
-        vcpus_used = 0
-        memory_mb_used = 0
-        disk_gb_used = 0
-        for instance in instances:
-            vcpus_used += instance.vcpus
-            memory_mb_used += instance.memory
-            disk_gb_used += instance.disk
-
-        return vcpus_used, memory_mb_used, disk_gb_used
-
     def filter_dest_servers(self, hosts, instance_to_migrate):
         required_cores = instance_to_migrate.vcpus
         required_disk = instance_to_migrate.disk
@@ -228,12 +215,9 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
 
         dest_servers = []
         for host in hosts:
-            cores_used, mem_used, disk_used = self.calc_used_resource(host)
+            free_res = self.compute_model.get_node_free_resources(host)
-            cores_available = host.vcpu_capacity - cores_used
+            if (free_res['vcpu'] >= required_cores and free_res['disk'] >=
-            disk_available = host.disk_gb_capacity - disk_used
+                    required_disk and free_res['memory'] >= required_memory):
-            mem_available = host.memory_mb_capacity - mem_used
-            if (cores_available >= required_cores and disk_available >=
-                    required_disk and mem_available >= required_memory):
                 dest_servers.append(host)
 
         return dest_servers

watcher/decision_engine/strategy/strategies/outlet_temp_control.py

@@ -140,19 +140,6 @@ class OutletTempControl(base.ThermalOptimizationBaseStrategy):
                 if cn.state == element.ServiceState.ONLINE.value and
                 cn.status in default_node_scope}
 
-    def calc_used_resource(self, node):
-        """Calculate the used vcpus, memory and disk based on VM flavors"""
-        instances = self.compute_model.get_node_instances(node)
-        vcpus_used = 0
-        memory_mb_used = 0
-        disk_gb_used = 0
-        for instance in instances:
-            vcpus_used += instance.vcpus
-            memory_mb_used += instance.memory
-            disk_gb_used += instance.disk
-
-        return vcpus_used, memory_mb_used, disk_gb_used
-
     def group_hosts_by_outlet_temp(self):
         """Group hosts based on outlet temp meters"""
         nodes = self.get_available_compute_nodes()
@@ -222,13 +209,9 @@ class OutletTempControl(base.ThermalOptimizationBaseStrategy):
         for instance_data in hosts:
             host = instance_data['compute_node']
             # available
-            cores_used, mem_used, disk_used = self.calc_used_resource(host)
+            free_res = self.compute_model.get_node_free_resources(host)
-            cores_available = host.vcpu_capacity - cores_used
+            if (free_res['vcpu'] >= required_cores and free_res['disk'] >=
-            disk_available = host.disk_gb_capacity - disk_used
+                    required_disk and free_res['memory'] >= required_memory):
-            mem_available = host.memory_mb_capacity - mem_used
-            if cores_available >= required_cores \
-                    and disk_available >= required_disk \
-                    and mem_available >= required_memory:
                 dest_servers.append(instance_data)
 
         return dest_servers

watcher/decision_engine/strategy/strategies/uniform_airflow.py

@@ -135,19 +135,12 @@ class UniformAirflow(base.BaseStrategy):
 
     def calculate_used_resource(self, node):
         """Compute the used vcpus, memory and disk based on instance flavors"""
-        instances = self.compute_model.get_node_instances(node)
+        used_res = self.compute_model.get_node_used_resources(node)
-        vcpus_used = 0
-        memory_mb_used = 0
-        disk_gb_used = 0
-        for instance in instances:
-            vcpus_used += instance.vcpus
-            memory_mb_used += instance.memory
-            disk_gb_used += instance.disk
 
-        return vcpus_used, memory_mb_used, disk_gb_used
+        return used_res['vcpu'], used_res['memory'], used_res['disk']
 
     def choose_instance_to_migrate(self, hosts):
-        """Pick up an active instance instance to migrate from provided hosts
+        """Pick up an active instance to migrate from provided hosts
 
         :param hosts: the array of dict which contains node object
         """

watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py

@@ -48,7 +48,7 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
     A capacity coefficients (cc) might be used to adjust optimization
     thresholds. Different resources may require different coefficient
     values as well as setting up different coefficient values in both
-    phases may lead to to more efficient consolidation in the end.
+    phases may lead to more efficient consolidation in the end.
     If the cc equals 1 the full resource capacity may be used, cc
     values lower than 1 will lead to resource under utilization and
     values higher than 1 will lead to resource overbooking.

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -132,21 +132,8 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
                 if cn.state == element.ServiceState.ONLINE.value and
                 cn.status in default_node_scope}
 
-    def calculate_used_resource(self, node):
-        """Calculate the used vcpus, memory and disk based on VM flavors"""
-        instances = self.compute_model.get_node_instances(node)
-        vcpus_used = 0
-        memory_mb_used = 0
-        disk_gb_used = 0
-        for instance in instances:
-            vcpus_used += instance.vcpus
-            memory_mb_used += instance.memory
-            disk_gb_used += instance.disk
-
-        return vcpus_used, memory_mb_used, disk_gb_used
-
     def choose_instance_to_migrate(self, hosts, avg_workload, workload_cache):
-        """Pick up an active instance instance to migrate from provided hosts
+        """Pick up an active instance to migrate from provided hosts
 
         :param hosts: the array of dict which contains node object
         :param avg_workload: the average workload value of all nodes
@@ -203,14 +190,10 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
             host = instance_data['compute_node']
             workload = instance_data['workload']
             # calculate the available resources
-            cores_used, mem_used, disk_used = self.calculate_used_resource(
+            free_res = self.compute_model.get_node_free_resources(host)
-                host)
+            if (free_res['vcpu'] >= required_cores and
-            cores_available = host.vcpu_capacity - cores_used
+                    free_res['memory'] >= required_mem and
-            disk_available = host.disk_gb_capacity - disk_used
+                    free_res['disk'] >= required_disk):
-            mem_available = host.memory_mb_capacity - mem_used
-            if (cores_available >= required_cores and
-                    mem_available >= required_mem and
-                    disk_available >= required_disk):
                 if (self._meter == 'instance_cpu_usage' and
                     ((src_instance_workload + workload) <
                      self.threshold / 100 * host.vcpus)):

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -514,7 +514,7 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                 self.aggregation_method['node']
 
         # backwards compatibility for node parameter with period.
-        if self.periods['node'] is not 0:
+        if self.periods['node'] != 0:
             LOG.warning('Parameter node has been renamed to compute_node and '
                         'will be removed in next release.')
             self.periods['compute_node'] = self.periods['node']

watcher/decision_engine/threading.py

@@ -0,0 +1,98 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 European Organization for Nuclear Research (CERN)
+#
+# Authors: Corne Lukken <info@dantalion.nl>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import futurist
+from futurist import waiters
+import six
+
+from oslo_config import cfg
+from oslo_log import log
+from oslo_service import service
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+@six.add_metaclass(service.Singleton)
+class DecisionEngineThreadPool(object):
+    """Singleton threadpool to submit general tasks to"""
+
+    def __init__(self):
+        self.amount_workers = CONF.watcher_decision_engine.max_general_workers
+        self._threadpool = futurist.GreenThreadPoolExecutor(
+            max_workers=self.amount_workers)
+
+    def submit(self, fn, *args, **kwargs):
+        """Will submit the job to the underlying threadpool
+
+        :param fn: function to execute in another thread
+        :param args: arguments for the function
+        :param kwargs: amount of arguments for the function
+        :return: future to monitor progress of execution
+        :rtype: :py:class"`futurist.GreenFuture`
+        """
+
+        return self._threadpool.submit(fn, *args, **kwargs)
+
+    @staticmethod
+    def do_while_futures(futures, fn, *args, **kwargs):
+        """Do while to execute a function upon completion from a collection
+
+        Will execute the specified function with its arguments when one of the
+        futures from the passed collection finishes. Additionally, the future
+        is passed as first argument to the function. Does not modify the passed
+        collection of futures.
+
+        :param futures: list, set or dictionary of futures
+        :type  futures: list :py:class:`futurist.GreenFuture`
+        :param fn:  function to execute upon the future finishing exection
+        :param args: arguments for the function
+        :param kwargs: amount of arguments for the function
+        """
+
+        # shallow copy the collection to not modify it outside of this method.
+        # shallow copy must be used because the type of collection needs to be
+        # determined at runtime (can be both list, set and dict).
+        futures = copy.copy(futures)
+
+        DecisionEngineThreadPool.do_while_futures_modify(
+            futures, fn, *args, **kwargs)
+
+    @staticmethod
+    def do_while_futures_modify(futures, fn, *args, **kwargs):
+        """Do while to execute a function upon completion from a collection
+
+        Will execute the specified function with its arguments when one of the
+        futures from the passed collection finishes. Additionally, the future
+        is passed as first argument to the function. Modifies the collection
+        by removing completed elements,
+
+        :param futures: list, set or dictionary of futures
+        :type  futures: list :py:class:`futurist.GreenFuture`
+        :param fn:  function to execute upon the future finishing exection
+        :param args: arguments for the function
+        :param kwargs: amount of arguments for the function
+        """
+
+        waits = waiters.wait_for_any(futures)
+        while len(waits[0]) > 0 or len(waits[1]) > 0:
+            for future in waiters.wait_for_any(futures)[0]:
+                fn(future, *args, **kwargs)
+                futures.remove(future)
+            waits = waiters.wait_for_any(futures)

watcher/hacking/checks.py

@@ -23,6 +23,8 @@ def flake8ext(f):
     only for unit tests to know which are watcher flake8 extensions.
     """
     f.name = __name__
+    f.version = '0.0.1'
+    f.skip_on_py3 = False
     return f
 
 
@@ -162,11 +164,11 @@ def check_asserttrue(logical_line, filename):
 def check_assertfalse(logical_line, filename):
     if 'watcher/tests/' in filename:
         if re.search(r"assertEqual\(\s*False,[^,]*(,[^,]*)?\)", logical_line):
-            msg = ("N328: Use assertFalse(observed) instead of "
+            msg = ("N329: Use assertFalse(observed) instead of "
                    "assertEqual(False, observed)")
             yield (0, msg)
         if re.search(r"assertEqual\([^,]*,\s*False(,[^,]*)?\)", logical_line):
-            msg = ("N328: Use assertFalse(observed) instead of "
+            msg = ("N329: Use assertFalse(observed) instead of "
                    "assertEqual(False, observed)")
             yield (0, msg)
 
@@ -283,21 +285,3 @@ def no_redundant_import_alias(logical_line):
     """
     if re.match(re_redundant_import_alias, logical_line):
         yield(0, "N342: No redundant import alias.")
-
-
-def factory(register):
-    register(use_jsonutils)
-    register(check_assert_called_once_with)
-    register(no_translate_debug_logs)
-    register(check_python3_xrange)
-    register(check_no_basestring)
-    register(check_python3_no_iteritems)
-    register(check_asserttrue)
-    register(check_assertfalse)
-    register(check_assertempty)
-    register(check_assertisinstance)
-    register(check_assertequal_for_httpcode)
-    register(check_log_warn_deprecated)
-    register(check_oslo_i18n_wrapper)
-    register(check_builtins_gettext)
-    register(no_redundant_import_alias)

watcher/locale/de/LC_MESSAGES/watcher.po

@@ -1,14 +1,14 @@
 # Frank Kloeker <eumel@arcor.de>, 2018. #zanata
-# Andreas Jaeger <jaegerandi@gmail.com>, 2019. #zanata
+# Andreas Jaeger <jaegerandi@gmail.com>, 2020. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: watcher VERSION\n"
 "Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
-"POT-Creation-Date: 2019-09-23 08:56+0000\n"
+"POT-Creation-Date: 2020-04-26 01:47+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2019-09-25 09:29+0000\n"
+"PO-Revision-Date: 2020-04-25 11:45+0000\n"
 "Last-Translator: Andreas Jaeger <jaegerandi@gmail.com>\n"
 "Language-Team: German\n"
 "Language: de\n"
@@ -196,10 +196,18 @@ msgstr "Prüfungsvorlagen"
 msgid "Audit parameter %(parameter)s are not allowed"
 msgstr "Prüfparameter %(parameter)s sind nicht erlaubt"
 
+#, python-format
+msgid "Audit state %(state)s is disallowed."
+msgstr "Auditstatus %(state)s ist ungültig."
+
 #, python-format
 msgid "Audit type %(audit_type)s could not be found"
 msgstr "Audit-Typ %(audit_type)s konnte nicht gefunden werden"
 
+#, python-format
+msgid "Audit type %(audit_type)s is disallowed."
+msgstr "Audit-Typ %(audit_type)s nicht erlaubt."
+
 #, python-format
 msgid "AuditTemplate %(audit_template)s could not be found"
 msgstr "AuditTemplate %(audit_template)s konnte nicht gefunden werden"
@@ -279,6 +287,10 @@ msgstr "Patch '%(patch)s' konnte nicht angewendet werden. Grund:%(reason)s"
 msgid "Couldn't delete when state is '%(state)s'."
 msgstr "Konnte nicht gelöscht werden, wenn der Status '%(state)s' ist."
 
+#, python-format
+msgid "Couldn't start when state is '%(state)s'."
+msgstr "Konnte nicht gestartet werden, wenn der Status '%(state)s' ist."
+
 #, python-format
 msgid "Datasource %(datasource)s is not available."
 msgstr "Datenquelle %(datasource)s ist nicht verfügbar."
@@ -421,6 +433,10 @@ msgstr "Ungültige Abfrage: %(start_time)s > %(end_time)s"
 msgid "Invalid sort direction: %s. Acceptable values are 'asc' or 'desc'"
 msgstr "Ungültige Sortierrichtung: %s. Akzeptable Werte sind 'asc' oder 'desc'"
 
+#, python-format
+msgid "Invalid sort key: %s"
+msgstr "Ungültiger Sortierschlüssel: %s"
+
 msgid "Invalid state for swapping volume"
 msgstr "Ungültiger Status für das Auslagern des Datenträgers"
 
@@ -440,6 +456,10 @@ msgstr "Limit sollte positiv sein"
 msgid "Maximum time since last check-in for up service."
 msgstr "Maximale Zeit seit dem letzten Check-in für den Up-Service."
 
+#, python-format
+msgid "Metric: %(metric)s not available"
+msgstr "Metrik: %(metric)s nicht verfügbar"
+
 #, python-format
 msgid "Migration of type '%(migration_type)s' is not supported."
 msgstr "Die Migration vom Typ '%(migration_type)s' wird nicht unterstützt."
@@ -535,6 +555,9 @@ msgstr ""
 "Verhältnis der freigegebenen Compute-Knoten geteilt durch die Gesamtzahl der "
 "aktivierten Compute-Knoten."
 
+msgid "Request not acceptable."
+msgstr "Anforderung nicht zulässig."
+
 #, python-format
 msgid "Role name seems ambiguous: %s"
 msgstr "Der Rollenname scheint mehrdeutig: %s"

watcher/locale/en_GB/LC_MESSAGES/watcher.po

@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: watcher VERSION\n"
 "Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
-"POT-Creation-Date: 2019-09-23 08:56+0000\n"
+"POT-Creation-Date: 2020-04-26 01:47+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

watcher/notifications/exception.py

@@ -13,8 +13,6 @@
 import inspect
 import sys
 
-import six
-
 from watcher.notifications import base as notificationbase
 from watcher.objects import base
 from watcher.objects import fields as wfields
@@ -42,7 +40,7 @@ class ExceptionPayload(notificationbase.NotificationPayloadBase):
             function_name=trace[3],
             module_name=inspect.getmodule(trace[0]).__name__,
             exception=fault.__class__.__name__,
-            exception_message=six.text_type(fault))
+            exception_message=str(fault))
 
 
 @notificationbase.notification_sample('infra-optim-exception.json')

watcher/objects/audit.py

@@ -75,6 +75,7 @@ class State(object):
 class AuditType(enum.Enum):
     ONESHOT = 'ONESHOT'
     CONTINUOUS = 'CONTINUOUS'
+    EVENT = 'EVENT'
 
 
 @base.WatcherObjectRegistry.register

watcher/tests/api/base.py

@@ -105,7 +105,6 @@ class FunctionalTest(base.DbTestCase):
         :param path_prefix: prefix of the url path
         """
         full_path = path_prefix + path
-        print('%s: %s %s' % (method.upper(), full_path, params))
 
         response = getattr(self.app, "%s_json" % method)(
             str(full_path),
@@ -115,7 +114,6 @@ class FunctionalTest(base.DbTestCase):
             extra_environ=extra_environ,
             expect_errors=expect_errors
         )
-        print('GOT:%s' % response)
         return response
 
     def put_json(self, path, params, expect_errors=False, headers=None,
@@ -192,13 +190,11 @@ class FunctionalTest(base.DbTestCase):
         :param path_prefix: prefix of the url path
         """
         full_path = path_prefix + path
-        print('DELETE: %s' % (full_path))
         response = self.app.delete(str(full_path),
                                    headers=headers,
                                    status=status,
                                    extra_environ=extra_environ,
                                    expect_errors=expect_errors)
-        print('GOT:%s' % response)
         return response
 
     def get_json(self, path, expect_errors=False, headers=None,
@@ -229,7 +225,6 @@ class FunctionalTest(base.DbTestCase):
         all_params.update(params)
         if q:
             all_params.update(query_params)
-        print('GET: %s %r' % (full_path, all_params))
 
         response = self.app.get(full_path,
                                 params=all_params,
@@ -238,7 +233,6 @@ class FunctionalTest(base.DbTestCase):
                                 expect_errors=expect_errors)
         if return_json and not expect_errors:
             response = response.json
-        print('GOT:%s' % response)
         return response
 
     def validate_link(self, link, bookmark=False):

watcher/tests/api/test_config.py

@@ -0,0 +1,35 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import imp
+from oslo_config import cfg
+from watcher.api import config as api_config
+from watcher.tests.api import base
+
+
+class TestRoot(base.FunctionalTest):
+
+    def test_config_enable_webhooks_auth(self):
+        acl_public_routes = ['/']
+        cfg.CONF.set_override('enable_webhooks_auth', True, 'api')
+        imp.reload(api_config)
+        self.assertEqual(acl_public_routes,
+                         api_config.app['acl_public_routes'])
+
+    def test_config_disable_webhooks_auth(self):
+        acl_public_routes = ['/', '/v1/webhooks/.*']
+        cfg.CONF.set_override('enable_webhooks_auth', False, 'api')
+        imp.reload(api_config)
+        self.assertEqual(acl_public_routes,
+                         api_config.app['acl_public_routes'])

watcher/tests/api/test_root.py

@@ -27,8 +27,9 @@ class TestRoot(base.FunctionalTest):
 
 class TestV1Root(base.FunctionalTest):
 
-    def test_get_v1_root(self):
+    def test_get_v1_root_all(self):
-        data = self.get_json('/')
+        data = self.get_json(
+            '/', headers={'OpenStack-API-Version': 'infra-optim 1.4'})
         self.assertEqual('v1', data['id'])
         # Check fields are not empty
         for f in data.keys():
@@ -38,6 +39,24 @@ class TestV1Root(base.FunctionalTest):
         actual_resources = tuple(set(data.keys()) - set(not_resources))
         expected_resources = ('audit_templates', 'audits', 'actions',
                               'action_plans', 'data_model', 'scoring_engines',
+                              'services', 'webhooks')
+        self.assertEqual(sorted(expected_resources), sorted(actual_resources))
+
+        self.assertIn({'type': 'application/vnd.openstack.watcher.v1+json',
+                       'base': 'application/json'}, data['media_types'])
+
+    def test_get_v1_root_without_datamodel(self):
+        data = self.get_json(
+            '/', headers={'OpenStack-API-Version': 'infra-optim 1.2'})
+        self.assertEqual('v1', data['id'])
+        # Check fields are not empty
+        for f in data.keys():
+            self.assertNotIn(f, ['', []])
+        # Check if all known resources are present and there are no extra ones.
+        not_resources = ('id', 'links', 'media_types')
+        actual_resources = tuple(set(data.keys()) - set(not_resources))
+        expected_resources = ('audit_templates', 'audits', 'actions',
+                              'action_plans', 'scoring_engines',
                               'services')
         self.assertEqual(sorted(expected_resources), sorted(actual_resources))
 

watcher/tests/api/v1/test_audit_templates.py

@@ -555,6 +555,35 @@ class TestPost(FunctionalTestWithSetup):
             response.json['created_at']).replace(tzinfo=None)
         self.assertEqual(test_time, return_created_at)
 
+    @mock.patch.object(timeutils, 'utcnow')
+    def test_create_audit_template_with_strategy_name(self, mock_utcnow):
+        audit_template_dict = post_get_test_audit_template(
+            goal=self.fake_goal1.uuid,
+            strategy=self.fake_strategy1.name)
+        test_time = datetime.datetime(2000, 1, 1, 0, 0)
+        mock_utcnow.return_value = test_time
+
+        response = self.post_json('/audit_templates', audit_template_dict)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(201, response.status_int)
+        # Check location header
+        self.assertIsNotNone(response.location)
+        expected_location = \
+            '/v1/audit_templates/%s' % response.json['uuid']
+        self.assertEqual(urlparse.urlparse(response.location).path,
+                         expected_location)
+        self.assertTrue(utils.is_uuid_like(response.json['uuid']))
+        self.assertNotIn('updated_at', response.json.keys)
+        self.assertNotIn('deleted_at', response.json.keys)
+        self.assertEqual(self.fake_goal1.uuid, response.json['goal_uuid'])
+        self.assertEqual(self.fake_strategy1.uuid,
+                         response.json['strategy_uuid'])
+        self.assertEqual(self.fake_strategy1.name,
+                         response.json['strategy_name'])
+        return_created_at = timeutils.parse_isotime(
+            response.json['created_at']).replace(tzinfo=None)
+        self.assertEqual(test_time, return_created_at)
+
     def test_create_audit_template_validation_with_aggregates(self):
         scope = [{'compute': [{'host_aggregates': [{'id': '*'}]},
                               {'availability_zones': [{'name': 'AZ1'},
@@ -663,6 +692,26 @@ class TestPost(FunctionalTestWithSetup):
         self.assertEqual(400, response.status_int)
         assert not cn_mock.called
 
+    def test_create_audit_template_with_old_scope(self):
+        scope = [{'host_aggregates': [{'id': '*'}]},
+                 {'availability_zones': [{'name': 'AZ1'},
+                                         {'name': 'AZ2'}]},
+                 {'exclude': [
+                     {'instances': [
+                         {'uuid': 'INSTANCE_1'},
+                         {'uuid': 'INSTANCE_2'}]},
+                     {'compute_nodes': [
+                         {'name': 'Node_1'},
+                         {'name': 'Node_2'}]},
+                 ]}
+                 ]
+        audit_template_dict = post_get_test_audit_template(
+            goal=self.fake_goal1.uuid,
+            strategy=self.fake_strategy1.uuid, scope=scope)
+        response = self.post_json('/audit_templates',
+                                  audit_template_dict)
+        self.assertEqual(201, response.status_int)
+
 
 class TestDelete(api_base.FunctionalTest):
 

watcher/tests/api/v1/test_data_model.py

@@ -32,9 +32,18 @@ class TestListDataModel(api_base.FunctionalTest):
         self.addCleanup(p_dcapi.stop)
 
     def test_get_all(self):
-        response = self.get_json('/data_model/?data_model_type=compute')
+        response = self.get_json(
+            '/data_model/?data_model_type=compute',
+            headers={'OpenStack-API-Version': 'infra-optim 1.3'})
         self.assertEqual('fake_response_value', response)
 
+    def test_get_all_not_acceptable(self):
+        response = self.get_json(
+            '/data_model/?data_model_type=compute',
+            headers={'OpenStack-API-Version': 'infra-optim 1.2'},
+            expect_errors=True)
+        self.assertEqual(406, response.status_int)
+
 
 class TestDataModelPolicyEnforcement(api_base.FunctionalTest):
 
@@ -60,6 +69,7 @@ class TestDataModelPolicyEnforcement(api_base.FunctionalTest):
         self._common_policy_check(
             "data_model:get_all", self.get_json,
             "/data_model/?data_model_type=compute",
+            headers={'OpenStack-API-Version': 'infra-optim 1.3'},
             expect_errors=True)
 
 

watcher/tests/api/v1/test_webhooks.py

@@ -0,0 +1,71 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import mock
+
+from watcher.decision_engine import rpcapi as deapi
+from watcher import objects
+from watcher.tests.api import base as api_base
+from watcher.tests.objects import utils as obj_utils
+
+
+class TestPost(api_base.FunctionalTest):
+
+    def setUp(self):
+        super(TestPost, self).setUp()
+        obj_utils.create_test_goal(self.context)
+        obj_utils.create_test_strategy(self.context)
+        obj_utils.create_test_audit_template(self.context)
+
+    @mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
+    def test_trigger_audit(self, mock_trigger_audit):
+        audit = obj_utils.create_test_audit(
+            self.context,
+            audit_type=objects.audit.AuditType.EVENT.value)
+        response = self.post_json(
+            '/webhooks/%s' % audit['uuid'], {},
+            headers={'OpenStack-API-Version': 'infra-optim 1.4'})
+        self.assertEqual(202, response.status_int)
+        mock_trigger_audit.assert_called_once_with(
+            mock.ANY, audit['uuid'])
+
+    def test_trigger_audit_with_no_audit(self):
+        response = self.post_json(
+            '/webhooks/no-audit', {},
+            headers={'OpenStack-API-Version': 'infra-optim 1.4'},
+            expect_errors=True)
+        self.assertEqual(404, response.status_int)
+        self.assertEqual('application/json', response.content_type)
+        self.assertTrue(response.json['error_message'])
+
+    def test_trigger_audit_with_not_allowed_audittype(self):
+        audit = obj_utils.create_test_audit(self.context)
+        response = self.post_json(
+            '/webhooks/%s' % audit['uuid'], {},
+            headers={'OpenStack-API-Version': 'infra-optim 1.4'},
+            expect_errors=True)
+        self.assertEqual(400, response.status_int)
+        self.assertEqual('application/json', response.content_type)
+        self.assertTrue(response.json['error_message'])
+
+    def test_trigger_audit_with_not_allowed_audit_state(self):
+        audit = obj_utils.create_test_audit(
+            self.context,
+            audit_type=objects.audit.AuditType.EVENT.value,
+            state=objects.audit.State.FAILED)
+        response = self.post_json(
+            '/webhooks/%s' % audit['uuid'], {},
+            headers={'OpenStack-API-Version': 'infra-optim 1.4'},
+            expect_errors=True)
+        self.assertEqual(400, response.status_int)
+        self.assertEqual('application/json', response.content_type)
+        self.assertTrue(response.json['error_message'])

watcher/tests/common/test_clients.py

@@ -27,7 +27,6 @@ from monascaclient.v2_0 import client as monclient_v2
 from neutronclient.neutron import client as netclient
 from neutronclient.v2_0 import client as netclient_v2
 from novaclient import client as nvclient
-import six
 
 from watcher.common import clients
 from watcher import conf
@@ -139,8 +138,7 @@ class TestClients(base.TestCase):
         osc = clients.OpenStackClients()
         osc._nova = None
         ex = self.assertRaises(ValueError, osc.nova)
-        self.assertIn('Invalid nova_client.api_version 2.47',
+        self.assertIn('Invalid nova_client.api_version 2.47', str(ex))
-                      six.text_type(ex))
 
     @mock.patch.object(clients.OpenStackClients, 'session')
     def test_clients_nova_diff_endpoint(self, mock_session):

watcher/tests/db/test_action.py

@@ -16,7 +16,6 @@
 """Tests for manipulating Action via the DB API"""
 
 import freezegun
-import six
 
 from watcher.common import exception
 from watcher.common import utils as w_utils
@@ -237,7 +236,7 @@ class DbActionTestCase(base.DbTestCase):
         uuids = []
         for _ in range(1, 4):
             action = utils.create_test_action(uuid=w_utils.generate_uuid())
-            uuids.append(six.text_type(action['uuid']))
+            uuids.append(str(action['uuid']))
         actions = self.dbapi.get_action_list(self.context)
         action_uuids = [a.uuid for a in actions]
         self.assertEqual(3, len(action_uuids))
@@ -254,7 +253,7 @@ class DbActionTestCase(base.DbTestCase):
             action = utils.create_test_action(
                 id=i, uuid=w_utils.generate_uuid(),
                 action_plan_id=action_plan.id)
-            uuids.append(six.text_type(action['uuid']))
+            uuids.append(str(action['uuid']))
         actions = self.dbapi.get_action_list(self.context, eager=True)
         action_map = {a.uuid: a for a in actions}
         self.assertEqual(sorted(uuids), sorted(action_map.keys()))

watcher/tests/db/test_action_plan.py

@@ -16,7 +16,6 @@
 """Tests for manipulating ActionPlan via the DB API"""
 
 import freezegun
-import six
 
 from watcher.common import exception
 from watcher.common import utils as w_utils
@@ -235,7 +234,7 @@ class DbActionPlanTestCase(base.DbTestCase):
         for _ in range(1, 4):
             action_plan = utils.create_test_action_plan(
                 uuid=w_utils.generate_uuid())
-            uuids.append(six.text_type(action_plan['uuid']))
+            uuids.append(str(action_plan['uuid']))
         action_plans = self.dbapi.get_action_plan_list(self.context)
         action_plan_uuids = [ap.uuid for ap in action_plans]
         self.assertEqual(sorted(uuids), sorted(action_plan_uuids))
@@ -253,7 +252,7 @@ class DbActionPlanTestCase(base.DbTestCase):
         for _ in range(1, 4):
             action_plan = utils.create_test_action_plan(
                 uuid=w_utils.generate_uuid())
-            uuids.append(six.text_type(action_plan['uuid']))
+            uuids.append(str(action_plan['uuid']))
         action_plans = self.dbapi.get_action_plan_list(
             self.context, eager=True)
         action_plan_map = {a.uuid: a for a in action_plans}

watcher/tests/db/test_audit.py

@@ -16,7 +16,6 @@
 """Tests for manipulating Audit via the DB API"""
 
 import freezegun
-import six
 
 from watcher.common import exception
 from watcher.common import utils as w_utils
@@ -273,7 +272,7 @@ class DbAuditTestCase(base.DbTestCase):
         for id_ in range(1, 4):
             audit = utils.create_test_audit(uuid=w_utils.generate_uuid(),
                                             name='My Audit {0}'.format(id_))
-            uuids.append(six.text_type(audit['uuid']))
+            uuids.append(str(audit['uuid']))
         audits = self.dbapi.get_audit_list(self.context)
         audit_uuids = [a.uuid for a in audits]
         self.assertEqual(sorted(uuids), sorted(audit_uuids))
@@ -293,7 +292,7 @@ class DbAuditTestCase(base.DbTestCase):
                 id=i, uuid=w_utils.generate_uuid(),
                 name='My Audit {0}'.format(i),
                 goal_id=goal.id, strategy_id=strategy.id)
-            uuids.append(six.text_type(audit['uuid']))
+            uuids.append(str(audit['uuid']))
         audits = self.dbapi.get_audit_list(self.context, eager=True)
         audit_map = {a.uuid: a for a in audits}
         self.assertEqual(sorted(uuids), sorted(audit_map.keys()))

watcher/tests/db/test_audit_template.py

@@ -16,7 +16,6 @@
 """Tests for manipulating AuditTemplate via the DB API"""
 
 import freezegun
-import six
 
 from watcher.common import exception
 from watcher.common import utils as w_utils
@@ -235,7 +234,7 @@ class DbAuditTemplateTestCase(base.DbTestCase):
                 id=i,
                 uuid=w_utils.generate_uuid(),
                 name='My Audit Template {0}'.format(i))
-            uuids.append(six.text_type(audit_template['uuid']))
+            uuids.append(str(audit_template['uuid']))
         audit_templates = self.dbapi.get_audit_template_list(self.context)
         audit_template_uuids = [at.uuid for at in audit_templates]
         self.assertEqual(sorted(uuids), sorted(audit_template_uuids))
@@ -255,7 +254,7 @@ class DbAuditTemplateTestCase(base.DbTestCase):
                 id=i, uuid=w_utils.generate_uuid(),
                 name='My Audit Template {0}'.format(i),
                 goal_id=goal.id, strategy_id=strategy.id)
-            uuids.append(six.text_type(audit_template['uuid']))
+            uuids.append(str(audit_template['uuid']))
         audit_templates = self.dbapi.get_audit_template_list(
             self.context, eager=True)
         audit_template_map = {a.uuid: a for a in audit_templates}

watcher/tests/db/test_efficacy_indicator.py

@@ -16,7 +16,6 @@
 """Tests for manipulating EfficacyIndicator via the DB API"""
 
 import freezegun
-import six
 
 from watcher.common import exception
 from watcher.common import utils as w_utils
@@ -249,7 +248,7 @@ class DbEfficacyIndicatorTestCase(base.DbTestCase):
             efficacy_indicator = utils.create_test_efficacy_indicator(
                 action_plan_id=action_plan.id, id=id_, uuid=None,
                 name="efficacy_indicator", description="Test Indicator ")
-            uuids.append(six.text_type(efficacy_indicator['uuid']))
+            uuids.append(str(efficacy_indicator['uuid']))
         efficacy_indicators = self.dbapi.get_efficacy_indicator_list(
             self.context)
         efficacy_indicator_uuids = [ei.uuid for ei in efficacy_indicators]
@@ -266,7 +265,7 @@ class DbEfficacyIndicatorTestCase(base.DbTestCase):
             efficacy_indicator = utils.create_test_efficacy_indicator(
                 id=i, uuid=w_utils.generate_uuid(),
                 action_plan_id=action_plan.id)
-            uuids.append(six.text_type(efficacy_indicator['uuid']))
+            uuids.append(str(efficacy_indicator['uuid']))
         efficacy_indicators = self.dbapi.get_efficacy_indicator_list(
             self.context, eager=True)
         efficacy_indicator_map = {a.uuid: a for a in efficacy_indicators}

watcher/tests/db/test_goal.py

@@ -16,7 +16,6 @@
 """Tests for manipulating Goal via the DB API"""
 
 import freezegun
-import six
 
 from watcher.common import exception
 from watcher.common import utils as w_utils
@@ -231,7 +230,7 @@ class DbGoalTestCase(base.DbTestCase):
                 uuid=w_utils.generate_uuid(),
                 name="GOAL_%s" % i,
                 display_name='My Goal %s' % i)
-            uuids.append(six.text_type(goal['uuid']))
+            uuids.append(str(goal['uuid']))
         goals = self.dbapi.get_goal_list(self.context)
         goal_uuids = [g.uuid for g in goals]
         self.assertEqual(sorted(uuids), sorted(goal_uuids))