Merge "add Ocata release notes"

Change-Id: Iff0c7601f6ea1b35de1afb1d71b8aff08a967eab

.coveragerc

@@ -1,7 +1,9 @@
 [run]
 branch = True
 source = watcher
-omit = watcher/tests/*
+omit =
+    watcher/tests/*
+    watcher/hacking/*
 
 [report]
 ignore_errors = True

MANIFEST.in

@@ -1,6 +0,0 @@
-include AUTHORS
-include ChangeLog
-exclude .gitignore
-exclude .gitreview
-
-global-exclude *.pyc

README.rst

@@ -1,3 +1,12 @@
+========================
+Team and repository tags
+========================
+
+.. image:: http://governance.openstack.org/badges/watcher.svg
+    :target: http://governance.openstack.org/reference/tags/index.html
+
+.. Change things from this point on
+
 ..
       Except where otherwise noted, this document is licensed under Creative
       Commons Attribution 3.0 License.  You can view the license at:
@@ -10,10 +19,8 @@ Watcher
 
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
-Watcher provides a complete optimization loop-including everything from a
+Watcher provides a robust framework to realize a wide range of cloud
-metrics receiver, complex event processor and profiler, optimization processor
+optimization goals, including the reduction of data center
-and an action plan applier. This provides a robust framework to realize a wide
-range of cloud optimization goals, including the reduction of data center
 operating costs, increased system performance via intelligent virtual machine
 migration, increased energy efficiency-and more!
 

devstack/lib/watcher

@@ -44,6 +44,9 @@ WATCHER_CONF_DIR=/etc/watcher
 WATCHER_CONF=$WATCHER_CONF_DIR/watcher.conf
 WATCHER_POLICY_JSON=$WATCHER_CONF_DIR/policy.json
 
+NOVA_CONF_DIR=/etc/nova
+NOVA_CONF=$NOVA_CONF_DIR/nova.conf
+
 if is_ssl_enabled_service "watcher" || is_service_enabled tls-proxy; then
     WATCHER_SERVICE_PROTOCOL="https"
 fi
@@ -123,6 +126,11 @@ function create_watcher_conf {
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
 
+    iniset $WATCHER_CONF oslo_messaging_notifications driver "messaging"
+
+    iniset $NOVA_CONF oslo_messaging_notifications topics "notifications,watcher_notifications"
+    iniset $NOVA_CONF notifications notify_on_state_change "vm_and_task_state"
+
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR "watcher_clients_auth"
 

devstack/local.conf.compute

@@ -44,3 +44,6 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
+notify_on_state_change = vm_and_task_state
+[notifications]
+notify_on_state_change = vm_and_task_state

devstack/local.conf.controller

@@ -28,7 +28,7 @@ ENABLED_SERVICES+=,q-svc,q-dhcp,q-meta,q-agt,q-l3,neutron
 enable_service n-cauth
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+# enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
 enable_plugin watcher git://git.openstack.org/openstack/watcher
@@ -45,3 +45,6 @@ LOGDAYS=2
 [[post-config|$NOVA_CONF]]
 [DEFAULT]
 compute_monitors=cpu.virt_driver
+notify_on_state_change = vm_and_task_state
+[notifications]
+notify_on_state_change = vm_and_task_state

doc/ext/term.py

@@ -1,4 +1,3 @@
-# -*- encoding: utf-8 -*-
 # Copyright (c) 2015 b<>com
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +21,6 @@ import inspect
 from docutils import nodes
 from docutils.parsers import rst
 from docutils import statemachine
-from stevedore import extension
 
 from watcher.version import version_info
 
@@ -76,7 +74,7 @@ class WatcherTerm(BaseWatcherDirective):
     # Inside your .rst file
     .. watcher-term:: import.path.to.your.DocumentedObject
 
-    This directive will then import the docstring and then interprete it.
+    This directive will then import the docstring and then interpret it.
     """
 
     # You need to put an import path as an argument for this directive to work
@@ -86,7 +84,12 @@ class WatcherTerm(BaseWatcherDirective):
         cls_path = self.arguments[0]
 
         try:
-            cls = importlib.import_module(cls_path)
+            try:
+                cls = importlib.import_module(cls_path)
+            except ImportError:
+                module_name, cls_name = cls_path.rsplit('.', 1)
+                mod = importlib.import_module(module_name)
+                cls = getattr(mod, cls_name)
         except Exception as exc:
             raise self.error(exc)
 
@@ -98,74 +101,72 @@ class WatcherTerm(BaseWatcherDirective):
         return node.children
 
 
-class DriversDoc(BaseWatcherDirective):
+class WatcherFunc(BaseWatcherDirective):
-    """Directive to import an RST formatted docstring into the Watcher doc
+    """Directive to import a value returned by a func into the Watcher doc
-
-    This directive imports the RST formatted docstring of every driver declared
-    within an entry point namespace provided as argument
 
     **How to use it**
 
     # inside your .py file
-    class DocumentedClassReferencedInEntrypoint(object):
+    class Bar(object):
-        '''My *.rst* docstring'''
+
+        def foo(object):
+            return foo_string
 
-        def foo(self):
-            '''Foo docstring'''
 
     # Inside your .rst file
-    .. drivers-doc:: entrypoint_namespace
+    .. watcher-func:: import.path.to.your.Bar.foo node_classname
-       :append_methods_doc: foo
 
-    This directive will then import the docstring and then interprete it.
+    node_classname is decumented here:
+    http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html
 
-    Note that no section/sub-section can be imported via this directive as it
+    This directive will then import the value and then interpret it.
-    is a Sphinx restriction.
     """
 
     # You need to put an import path as an argument for this directive to work
-    required_arguments = 1
+    # required_arguments = 1
-    optional_arguments = 0
+    # optional_arguments = 1
-    final_argument_whitespace = True
-    has_content = False
 
-    option_spec = dict(
+    option_spec = {'format': rst.directives.unchanged}
-        # CSV formatted list of method names whose return values will be zipped
+    has_content = True
-        # together in the given order
-        append_methods_doc=lambda opts: [
-            opt.strip() for opt in opts.split(",") if opt.strip()],
-        # By default, we always start by adding the driver object docstring
-        exclude_driver_docstring=rst.directives.flag,
-    )
 
     def run(self):
-        ext_manager = extension.ExtensionManager(namespace=self.arguments[0])
+        if not self.content:
-        extensions = ext_manager.extensions
+            error = self.state_machine.reporter.error(
-        # Aggregates drivers based on their module name (i.e import path)
+                'The "%s" directive is empty; content required.' % self.name,
-        classes = [(ext.name, ext.plugin) for ext in extensions]
+                nodes.literal_block(self.block_text, self.block_text),
+                line=self.lineno)
+            return [error]
 
-        for name, cls in classes:
+        func_path = self.content[0]
-            self.add_line(".. rubric:: %s" % name)
+        try:
-            self.add_line("")
+            cls_path, func_name = func_path.rsplit('.', 1)
+            module_name, cls_name = cls_path.rsplit('.', 1)
+            mod = importlib.import_module(module_name)
+            cls = getattr(mod, cls_name)
+        except Exception as exc:
+            raise self.error(exc)
 
-            if "exclude_driver_docstring" not in self.options:
+        cls_obj = cls()
-                self.add_object_docstring(cls)
+        func = getattr(cls_obj, func_name)
-                self.add_line("")
+        textblock = func()
+        if not isinstance(textblock, str):
+            textblock = str(textblock)
 
-            for method_name in self.options.get("append_methods_doc", []):
+        self.add_textblock(textblock)
-                if hasattr(cls, method_name):
-                    method = getattr(cls, method_name)
-                    method_result = inspect.cleandoc(method)
-                    self.add_textblock(method_result())
-                    self.add_line("")
 
-        node = nodes.paragraph()
+        try:
+            node_class = getattr(nodes,
+                                 self.options.get('format', 'paragraph'))
+        except Exception as exc:
+            raise self.error(exc)
+
+        node = node_class()
         node.document = self.state.document
         self.state.nested_parse(self.result, 0, node)
-        return node.children
+        return [node]
 
 
 def setup(app):
-    app.add_directive('drivers-doc', DriversDoc)
     app.add_directive('watcher-term', WatcherTerm)
+    app.add_directive('watcher-func', WatcherFunc)
     return {'version': version_info.version_string()}

doc/ext/versioned_notifications.py

@@ -0,0 +1,133 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+This provides a sphinx extension able to list the implemented versioned
+notifications into the developer documentation.
+
+It is used via a single directive in the .rst file
+
+  .. versioned_notifications::
+
+"""
+
+from sphinx.util.compat import Directive
+from docutils import nodes
+
+from watcher.notifications import base as notification
+from watcher.objects import base
+
+
+class VersionedNotificationDirective(Directive):
+
+    SAMPLE_ROOT = 'doc/notification_samples/'
+    TOGGLE_SCRIPT = """
+<script>
+jQuery(document).ready(function(){
+    jQuery('#%s-div').toggle('show');
+    jQuery('#%s-hideshow').on('click', function(event) {
+        jQuery('#%s-div').toggle('show');
+    });
+});
+</script>
+"""
+
+    def run(self):
+        notifications = self._collect_notifications()
+        return self._build_markup(notifications)
+
+    def _collect_notifications(self):
+        base.WatcherObjectRegistry.register_notification_objects()
+        notifications = []
+        ovos = base.WatcherObjectRegistry.obj_classes()
+        for name, cls in ovos.items():
+            cls = cls[0]
+            if (issubclass(cls, notification.NotificationBase) and
+                    cls != notification.NotificationBase):
+
+                payload_name = cls.fields['payload'].objname
+                payload_cls = ovos[payload_name][0]
+                for sample in cls.samples:
+                    notifications.append((cls.__name__,
+                                          payload_cls.__name__,
+                                          sample))
+        return sorted(notifications)
+
+    def _build_markup(self, notifications):
+        content = []
+        cols = ['Event type', 'Notification class', 'Payload class', 'Sample']
+        table = nodes.table()
+        content.append(table)
+        group = nodes.tgroup(cols=len(cols))
+        table.append(group)
+
+        head = nodes.thead()
+        group.append(head)
+
+        for _ in cols:
+            group.append(nodes.colspec(colwidth=1))
+
+        body = nodes.tbody()
+        group.append(body)
+
+        # fill the table header
+        row = nodes.row()
+        body.append(row)
+        for col_name in cols:
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.strong(text=col_name)
+            col.append(text)
+
+        # fill the table content, one notification per row
+        for name, payload, sample_file in notifications:
+            event_type = sample_file[0: -5].replace('-', '.')
+
+            row = nodes.row()
+            body.append(row)
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=event_type)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=name)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=payload)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+
+            with open(self.SAMPLE_ROOT + sample_file, 'r') as f:
+                sample_content = f.read()
+
+            event_type = sample_file[0: -5]
+            html_str = self.TOGGLE_SCRIPT % ((event_type, ) * 3)
+            html_str += ("<input type='button' id='%s-hideshow' "
+                         "value='hide/show sample'>" % event_type)
+            html_str += ("<div id='%s-div'><pre>%s</pre></div>"
+                         % (event_type, sample_content))
+
+            raw = nodes.raw('', html_str, format="html")
+            col.append(raw)
+
+        return content
+
+
+def setup(app):
+    app.add_directive('versioned_notifications',
+                      VersionedNotificationDirective)

doc/notification_samples/audit-create.json

@@ -0,0 +1,69 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "PENDING",
+      "updated_at": null,
+      "deleted_at": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditCreatePayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.create",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-delete.json

@@ -0,0 +1,69 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "DELETED",
+      "updated_at": null,
+      "deleted_at": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditDeletePayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.delete",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-end.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.end",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-error.json

@@ -0,0 +1,80 @@
+{
+  "priority": "ERROR",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": {
+        "watcher_object.data": {
+          "exception": "WatcherException",
+          "exception_message": "TEST",
+          "function_name": "test_send_audit_action_with_error",
+          "module_name": "watcher.tests.notifications.test_audit_notification"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.error",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-start.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.start",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-end.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.end",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-error.json

@@ -0,0 +1,80 @@
+{
+  "priority": "ERROR",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": {
+        "watcher_object.data": {
+          "exception": "WatcherException",
+          "exception_message": "TEST",
+          "function_name": "test_send_audit_action_with_error",
+          "module_name": "watcher.tests.notifications.test_audit_notification"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.error",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-start.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.start",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-update.json

@@ -0,0 +1,78 @@
+{
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:51:38.722986   ",
+  "payload": {
+    "watcher_object.name": "AuditUpdatePayload",
+    "watcher_object.data": {
+      "strategy": {
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.data": {
+          "name": "dummy",
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "default": "hello",
+                "type": "string",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "maximum": 10.2,
+                "default": 3.2,
+                "minimum": 1.0,
+                "description": "number parameter example",
+                "type": "number"
+              }
+            }
+          },
+          "updated_at": null,
+          "display_name": "Dummy strategy",
+          "deleted_at": null,
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "created_at": "2016-11-04T16:25:35Z"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "scope": [],
+      "created_at": "2016-11-04T16:51:21Z",
+      "uuid": "f1e0d912-afd9-4bf2-91ef-c99cd08cc1ef",
+      "goal": {
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.data": {
+          "efficacy_specification": [],
+          "updated_at": null,
+          "name": "dummy",
+          "display_name": "Dummy goal",
+          "deleted_at": null,
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "created_at": "2016-11-04T16:25:35Z"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "deleted_at": null,
+      "state_update": {
+        "watcher_object.name": "AuditStateUpdatePayload",
+        "watcher_object.data": {
+          "state": "ONGOING",
+          "old_state": "PENDING"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "interval": null,
+      "updated_at": null,
+      "state": "ONGOING",
+      "audit_type": "ONESHOT"
+    },
+    "watcher_object.namespace": "watcher",
+    "watcher_object.version": "1.0"
+  },
+  "priority": "INFO",
+  "event_type": "audit.update",
+  "message_id": "697fdf55-7252-4b6c-a2c2-5b9e85f6342c"
+}

doc/notification_samples/infra-optim-exception.json

@@ -0,0 +1,16 @@
+{
+    "event_type": "infra-optim.exception",
+    "payload": {
+        "watcher_object.data": {
+            "exception": "NoAvailableStrategyForGoal",
+            "exception_message": "No strategy could be found to achieve the server_consolidation goal.",
+            "function_name": "_aggregate_create_in_db",
+            "module_name": "watcher.objects.aggregate"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+    },
+    "priority": "ERROR",
+    "publisher_id": "watcher-api:fake-mini"
+}

doc/source/architecture.rst

@@ -139,7 +139,7 @@ The Watcher Dashboard can be used to interact with the Watcher system through
 Horizon in order to control it or to know its current status.
 
 Please, read `the detailed documentation about Watcher Dashboard
-<https://factory.b-com.com/www/watcher/doc/watcher-dashboard/>`_.
+<http://docs.openstack.org/developer/watcher-dashboard/>`_.
 
 .. _archi_watcher_database_definition:
 
@@ -171,35 +171,47 @@ This component is responsible for computing a set of potential optimization
 :ref:`Actions <action_definition>` in order to fulfill
 the :ref:`Goal <goal_definition>` of an :ref:`Audit <audit_definition>`.
 
-It first reads the parameters of the :ref:`Audit <audit_definition>` from the
+It first reads the parameters of the :ref:`Audit <audit_definition>` to know
-associated :ref:`Audit Template <audit_template_definition>` and knows the
+the :ref:`Goal <goal_definition>` to achieve.
-:ref:`Goal <goal_definition>` to achieve.
 
-It then selects the most appropriate :ref:`Strategy <strategy_definition>`
+Unless specified, it then selects the most appropriate :ref:`strategy
-depending on how Watcher was configured for this :ref:`Goal <goal_definition>`.
+<strategy_definition>` from the list of available strategies achieving this
+goal.
 
 The :ref:`Strategy <strategy_definition>` is then dynamically loaded (via
-`stevedore <https://github.com/openstack/stevedore/>`_). The
+`stevedore <http://docs.openstack.org/developer/stevedore/>`_). The
-:ref:`Watcher Decision Engine <watcher_decision_engine_definition>` calls the
+:ref:`Watcher Decision Engine <watcher_decision_engine_definition>` executes
-**execute()** method of the :ref:`Strategy <strategy_definition>` class which
+the strategy.
-generates a solution composed of a set of :ref:`Actions <action_definition>`.
+
+In order to compute the potential :ref:`Solution <solution_definition>` for the
+Audit, the :ref:`Strategy <strategy_definition>` relies on different sets of
+data:
+
+- :ref:`Cluster data models <cluster_data_model_definition>` that are
+  periodically synchronized through pluggable cluster data model collectors.
+  These models contain the current state of various
+  :ref:`Managed resources <managed_resource_definition>` (e.g., the data stored
+  in the Nova database). These models gives a strategy the ability to reason on
+  the current state of a given :ref:`cluster <cluster_definition>`.
+- The data stored in the :ref:`Cluster History Database
+  <cluster_history_db_definition>` which provides information about the past of
+  the :ref:`Cluster <cluster_definition>`.
+
+Here below is a sequence diagram showing how the Decision Engine builds and
+maintains the :ref:`cluster data models <cluster_data_model_definition>` that
+are used by the strategies.
+
+.. image:: ./images/sequence_architecture_cdmc_sync.png
+   :width: 100%
+
+The execution of a strategy then yields a solution composed of a set of
+:ref:`Actions <action_definition>` as well as a set of :ref:`efficacy
+indicators <efficacy_indicator_definition>`.
 
 These :ref:`Actions <action_definition>` are scheduled in time by the
 :ref:`Watcher Planner <watcher_planner_definition>` (i.e., it generates an
 :ref:`Action Plan <action_plan_definition>`).
 
-In order to compute the potential :ref:`Solution <solution_definition>` for the
-Audit, the :ref:`Strategy <strategy_definition>` relies on two sets of data:
-
--   the current state of the
-    :ref:`Managed resources <managed_resource_definition>`
-    (e.g., the data stored in the Nova database)
--   the data stored in the
-    :ref:`Cluster History Database <cluster_history_db_definition>`
-    which provides information about the past of the
-    :ref:`Cluster <cluster_definition>`
-
-
 .. _data_model:
 
 Data model
@@ -216,8 +228,6 @@ Here below is a diagram representing the main objects in Watcher from a
 database perspective:
 
 .. image:: ./images/watcher_db_schema_diagram.png
-   :width: 100%
-
 
 
 .. _sequence_diagrams:
@@ -280,7 +290,7 @@ the Audit parameters from the
 :ref:`Watcher Database <watcher_database_definition>`. It instantiates the
 appropriate :ref:`strategy <strategy_definition>` (using entry points)
 given both the :ref:`goal <goal_definition>` and the strategy associated to the
-parent :ref:`audit template <audit_template_definition>` of the :ref:`Audit
+parent :ref:`audit template <audit_template_definition>` of the :ref:`audit
 <audit_definition>`. If no strategy is associated to the audit template, the
 strategy is dynamically selected by the Decision Engine.
 
@@ -288,7 +298,7 @@ The :ref:`Watcher Decision Engine <watcher_decision_engine_definition>` also
 builds the :ref:`Cluster Data Model <cluster_data_model_definition>`. This
 data model is needed by the :ref:`Strategy <strategy_definition>` to know the
 current state and topology of the audited
-:ref:`Openstack cluster <cluster_definition>`.
+:ref:`OpenStack cluster <cluster_definition>`.
 
 The :ref:`Watcher Decision Engine <watcher_decision_engine_definition>` calls
 the **execute()** method of the instantiated
@@ -306,7 +316,7 @@ This method finds an appropriate scheduling of
 :ref:`Actions <action_definition>` taking into account some scheduling rules
 (such as priorities between actions).
 It generates a new :ref:`Action Plan <action_plan_definition>` with status
-**RECOMMENDED** and saves it into the:ref:`Watcher Database
+**RECOMMENDED** and saves it into the :ref:`Watcher Database
 <watcher_database_definition>`. The saved action plan is now a scheduled flow
 of actions to which a global efficacy is associated alongside a number of
 :ref:`Efficacy Indicators <efficacy_indicator_definition>` as specified by the
@@ -398,7 +408,7 @@ be one of the following:
    :ref:`Watcher Decision Engine <watcher_decision_engine_definition>`
 -  **SUCCEEDED** : the :ref:`Audit <audit_definition>` has been executed
    successfully and at least one solution was found
--  **FAILED** : an error occured while executing the
+-  **FAILED** : an error occurred while executing the
    :ref:`Audit <audit_definition>`
 -  **DELETED** : the :ref:`Audit <audit_definition>` is still stored in the
    :ref:`Watcher database <watcher_database_definition>` but is not returned
@@ -434,13 +444,13 @@ state may be one of the following:
 -  **SUCCEEDED** : the :ref:`Action Plan <action_plan_definition>` has been
    executed successfully (i.e. all :ref:`Actions <action_definition>` that it
    contains have been executed successfully)
--  **FAILED** : an error occured while executing the
+-  **FAILED** : an error occurred while executing the
    :ref:`Action Plan <action_plan_definition>`
 -  **DELETED** : the :ref:`Action Plan <action_plan_definition>` is still
    stored in the :ref:`Watcher database <watcher_database_definition>` but is
    not returned any more through the Watcher APIs.
 -  **CANCELLED** : the :ref:`Action Plan <action_plan_definition>` was in
-   **PENDING** or **ONGOING** state and was cancelled by the
+   **RECOMMENDED**, **PENDING** or **ONGOING** state and was cancelled by the
    :ref:`Administrator <administrator_definition>`
 
 

doc/source/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,7 +10,21 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+import sys
+
 from watcher import version as watcher_version
+from watcher import objects
+
+objects.register_all()
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../'))
+sys.path.insert(0, os.path.abspath('./'))
 
 # -- General configuration ----------------------------------------------------
 
@@ -19,13 +32,15 @@ from watcher import version as watcher_version
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'oslo_config.sphinxconfiggen',
+    'oslosphinx',
     'sphinx.ext.autodoc',
     'sphinx.ext.viewcode',
     'sphinxcontrib.httpdomain',
     'sphinxcontrib.pecanwsme.rest',
+    'stevedore.sphinxext',
     'wsmeext.sphinxext',
-    'oslosphinx',
+    'ext.term',
-    'watcher.doc',
+    'ext.versioned_notifications',
 ]
 
 wsme_protocols = ['restjson']
@@ -66,6 +81,8 @@ exclude_patterns = [
     # them when scanning for input files.
     'man/footer.rst',
     'man/general-options.rst',
+    'strategies/strategy-template.rst',
+    'image_src/plantuml/README.rst',
 ]
 
 # If true, '()' will be appended to :func: etc. cross-reference text.

doc/source/deploy/configuration.rst

@@ -15,7 +15,7 @@ Service overview
 ================
 
 The Watcher system is a collection of services that provides support to
-optimize your IAAS plateform. The Watcher service may, depending upon
+optimize your IAAS platform. The Watcher service may, depending upon
 configuration, interact with several other OpenStack services. This includes:
 
 - the OpenStack Identity service (`keystone`_) for request authentication and
@@ -37,7 +37,7 @@ The Watcher service includes the following components:
 - `watcher-dashboard`_: An Horizon plugin for interacting with the Watcher
   service.
 
-Additionally, the Bare Metal service has certain external dependencies, which
+Additionally, the Watcher service has certain external dependencies, which
 are very similar to other OpenStack services:
 
 - A database to store audit and action plan information and state. You can set
@@ -86,7 +86,6 @@ Configure the Identity service for the Watcher service
         --tenant=KEYSTONE_SERVICE_PROJECT_NAME
       $ keystone user-role-add --user=watcher \
         --tenant=KEYSTONE_SERVICE_PROJECT_NAME --role=admin
-      $ keystone user-role-add --user=watcher --tenant=admin --role=admin
 
    or (by using python-openstackclient 1.8.0+)
 
@@ -97,7 +96,6 @@ Configure the Identity service for the Watcher service
         --project=KEYSTONE_SERVICE_PROJECT_NAME
       $ openstack role add --project KEYSTONE_SERVICE_PROJECT_NAME \
         --user watcher admin
-      $ openstack role add --user watcher --project admin admin
 
 
 #. You must register the Watcher Service with the Identity Service so that
@@ -169,7 +167,7 @@ these following commands::
 
     $ git clone git://git.openstack.org/openstack/watcher
     $ cd watcher/
-    $ tox -econfig
+    $ tox -e genconfig
     $ vi etc/watcher/watcher.conf.sample
 
 
@@ -368,7 +366,7 @@ Configure Nova compute
 Please check your hypervisor configuration to correctly handle
 `instance migration`_.
 
-.. _`instance migration`: http://docs.openstack.org/admin-guide-cloud/compute-configuring-migrations.html
+.. _`instance migration`: http://docs.openstack.org/admin-guide/compute-live-migration-usage.html
 
 Configure Measurements
 ======================
@@ -403,6 +401,35 @@ own storage driver using whatever technology you want.
 For more information : https://wiki.openstack.org/wiki/Gnocchi
 
 
+Configure Nova Notifications
+============================
+
+Watcher can consume notifications generated by the Nova services, in order to
+build or update, in real time, its cluster data model related to computing
+resources.
+
+Nova publishes, by default, notifications on ``notifications`` AMQP queue
+(configurable) and ``versioned_notifications`` AMQP queue (not
+configurable). ``notifications`` queue is mainly used by ceilometer, so we can
+not use it. And some events, related to nova-compute service state, are only
+sent into the ``versioned_notifications`` queue.
+
+By default, Watcher listens to AMQP queues named ``watcher_notifications``
+and ``versioned_notifications``. So you have to update the Nova
+configuration file on controller and compute nodes, in order
+to Watcher receives Nova notifications in ``watcher_notifications`` as well.
+
+  * In the file ``/etc/nova/nova.conf``, update the section
+    ``[oslo_messaging_notifications]``, by redefining the list of topics
+    into which Nova services will publish events ::
+
+      [oslo_messaging_notifications]
+      driver = messaging
+      topics = notifications,watcher_notifications
+
+  * Restart the Nova services.
+
+
 Workers
 =======
 

doc/source/deploy/installation.rst

@@ -108,3 +108,54 @@ installed on your system.
 Once installed, you still need to declare Watcher as a new service into
 Keystone and to configure its different modules, which you can find described
 in :doc:`configuration`.
+
+
+Installing from packages: Debian (experimental)
+-----------------------------------------------
+
+Experimental Debian packages are available on `Debian repositories`_. The best
+way to use them is to install them into a Docker_ container.
+
+Here is single Dockerfile snippet you can use to run your Docker container:
+
+.. code-block:: bash
+
+    FROM debian:experimental
+    MAINTAINER David TARDIVEL <david.tardivel@b-com.com>
+
+    RUN  apt-get update
+    RUN  apt-get dist-upgrade -y
+    RUN  apt-get install -y vim  net-tools
+    RUN  apt-get install -yt experimental watcher-api
+
+    CMD ["/usr/bin/watcher-api"]
+
+Build your container from this Dockerfile:
+
+.. code-block:: bash
+
+    $ docker build -t watcher/api .
+
+To run your container, execute this command:
+
+.. code-block:: bash
+
+    $ docker run -d -p 9322:9322 watcher/api
+
+Check in your logs Watcher API is started
+
+.. code-block:: bash
+
+    $ docker logs <container ID>
+
+You can run similar container with Watcher Decision Engine (package
+``watcher-decision-engine``) and with the Watcher Applier (package
+``watcher-applier``).
+
+.. _Docker: https://www.docker.com/
+.. _`Debian repositories`: https://packages.debian.org/experimental/allpackages
+
+
+
+
+

doc/source/deploy/user-guide.rst

@@ -39,10 +39,10 @@ named ``watcher``, or by using the `OpenStack CLI`_ ``openstack``.
 If you want to deploy Watcher in Horizon, please refer to the `Watcher Horizon
 plugin installation guide`_.
 
-.. _`installation guide`: https://factory.b-com.com/www/watcher/doc/python-watcherclient
+.. _`installation guide`: http://docs.openstack.org/developer/python-watcherclient
-.. _`Watcher Horizon plugin installation guide`: https://factory.b-com.com/www/watcher/doc/watcher-dashboard/deploy/installation.html
+.. _`Watcher Horizon plugin installation guide`: http://docs.openstack.org/developer/watcher-dashboard/deploy/installation.html
 .. _`OpenStack CLI`: http://docs.openstack.org/developer/python-openstackclient/man/openstack.html
-.. _`Watcher CLI`: https://factory.b-com.com/www/watcher/doc/python-watcherclient/index.html
+.. _`Watcher CLI`: http://docs.openstack.org/developer/python-watcherclient/index.html
 
 Seeing what the Watcher CLI can do ?
 ------------------------------------
@@ -93,11 +93,11 @@ following command:
 
 .. code:: bash
 
-  $ watcher strategy list --goal-uuid <your_goal_uuid>
+  $ watcher strategy list --goal <your_goal_uuid_or_name>
 
 or::
 
-  $ openstack optimize strategy list --goal-uuid <your_goal_uuid>
+  $ openstack optimize strategy list --goal <your_goal_uuid_or_name>
 
 You can use the following command to check strategy details including which
 parameters of which format it supports:
@@ -172,7 +172,7 @@ Input parameter could cause audit creation failure, when:
 
 Watcher service will compute an :ref:`Action Plan <action_plan_definition>`
 composed of a list of potential optimization :ref:`actions <action_definition>`
-(instance migration, disabling of an hypervisor, ...) according to the
+(instance migration, disabling of a compute node, ...) according to the
 :ref:`goal <goal_definition>` to achieve. You can see all of the goals
 available in section ``[watcher_strategies]`` of the Watcher service
 configuration file.

doc/source/dev/devstack.rst

@@ -160,7 +160,7 @@ Edit `/etc/libvirt/libvirtd.conf` to make sure the following values are set::
 
 Edit `/etc/default/libvirt-bin`::
 
-    libvirt_opts="-d -l"
+    libvirtd_opts="-d -l"
 
 Restart the libvirt service::
 
@@ -193,6 +193,37 @@ must exist in every other compute node's stack user's authorized_keys file and
 every compute node's public ECDSA key needs to be in every other compute
 node's root user's known_hosts file.
 
+Disable serial console
+----------------------
+
+Serial console needs to be disabled for live migration to work.
+
+On both the controller and compute node, in /etc/nova/nova.conf
+
+[serial_console]
+enabled = False
+
+Alternatively, in devstack's local.conf:
+
+[[post-config|$NOVA_CONF]]
+[serial_console]
+#enabled=false
+
+
+VNC server configuration
+------------------------
+
+The VNC server listening parameter needs to be set to any address so
+that the server can accept connections from all of the compute nodes.
+
+On both the controller and compute node, in /etc/nova/nova.conf
+
+vncserver_listen = 0.0.0.0
+
+Alternatively, in devstack's local.conf:
+
+VNCSERVER_LISTEN=0.0.0.0
+
 
 Environment final checkup
 -------------------------

doc/source/dev/environment.rst

@@ -85,6 +85,9 @@ your platform.
 
     $ sudo yum install openssl-devel libffi-devel mysql-devel
 
+* CentOS 7::
+
+    $ sudo yum install gcc python-devel libxml2-devel libxslt-devel mariadb-devel
 
 PyPi Packages and VirtualEnv
 ----------------------------

doc/source/dev/notifications.rst

@@ -0,0 +1,13 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+.. _watcher_notifications:
+
+========================
+Notifications in Watcher
+========================
+
+.. versioned_notifications::

doc/source/dev/plugin/action-plugin.rst

@@ -30,12 +30,12 @@ implement:
    implement. This is the first function to be called by the
    :ref:`applier <watcher_applier_definition>` before any further processing
    and its role is to validate the input parameters that were provided to it.
- - The :py:meth:`~.BaseAction.precondition` is called before the execution of
+ - The :py:meth:`~.BaseAction.pre_condition` is called before the execution of
    an action. This method is a hook that can be used to perform some
    initializations or to make some more advanced validation on its input
    parameters. If you wish to block the execution based on this factor, you
    simply have to ``raise`` an exception.
- - The :py:meth:`~.BaseAction.postcondition` is called after the execution of
+ - The :py:meth:`~.BaseAction.post_condition` is called after the execution of
    an action. As this function is called regardless of whether an action
    succeeded or not, this can prove itself useful to perform cleanup
    operations.
@@ -71,21 +71,18 @@ Here is an example showing how you can write a plugin called ``DummyAction``:
             # Does nothing
             pass
 
-        def precondition(self):
+        def pre_condition(self):
             # No pre-checks are done here
             pass
 
-        def postcondition(self):
+        def post_condition(self):
             # Nothing done here
             pass
 
 
-This implementation is the most basic one. So if you want to have more advanced
+This implementation is the most basic one. So in order to get a better
-examples, have a look at the implementation of the actions already provided
+understanding on how to implement a more advanced action, have a look at the
-by Watcher like.
+:py:class:`~watcher.applier.actions.migration.Migrate` class.
-To get a better understanding on how to implement a more advanced action,
-have a look at the :py:class:`~watcher.applier.actions.migration.Migrate`
-class.
 
 Input validation
 ----------------
@@ -117,12 +114,15 @@ tune the action to its needs. To do so, you can implement the
         def execute(self):
             assert self.config.test_opt == 0
 
-        def get_config_opts(self):
+        @classmethod
-            return [
+        def get_config_opts(cls):
+            return super(
+                DummyAction, cls).get_config_opts() + [
                 cfg.StrOpt('test_opt', help="Demo Option.", default=0),
                 # Some more options ...
             ]
 
+
 The configuration options defined within this class method will be included
 within the global ``watcher.conf`` configuration file under a section named by
 convention: ``{namespace}.{plugin_name}``. In our case, the ``watcher.conf``

doc/source/dev/plugin/base-setup.rst

@@ -88,10 +88,13 @@ Now that the project skeleton has been created, you can start the
 implementation of your plugin. As of now, you can implement the following
 plugins for Watcher:
 
+- A :ref:`goal plugin <implement_goal_plugin>`
 - A :ref:`strategy plugin <implement_strategy_plugin>`
-- A :ref:`planner plugin <implement_planner_plugin>`
 - An :ref:`action plugin <implement_action_plugin>`
-- A :ref:`workflow engine plugin <implement_workflow_engine_plugin>`
+- A :ref:`planner plugin <implement_planner_plugin>`
+- A workflow engine plugin
+- A :ref:`cluster data model collector plugin
+  <implement_cluster_data_model_collector_plugin>`
 
 If you want to learn more on how to implement them, you can refer to their
 dedicated documentation.

doc/source/dev/plugin/cdmc-plugin.rst

@@ -0,0 +1,272 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+.. _implement_cluster_data_model_collector_plugin:
+
+========================================
+Build a new cluster data model collector
+========================================
+
+Watcher Decision Engine has an external cluster data model (CDM) plugin
+interface which gives anyone the ability to integrate an external cluster data
+model collector (CDMC) in order to extend the initial set of cluster data model
+collectors Watcher provides.
+
+This section gives some guidelines on how to implement and integrate custom
+cluster data model collectors within Watcher.
+
+
+Creating a new plugin
+=====================
+
+In order to create a new cluster data model collector, you have to:
+
+- Extend the :py:class:`~.base.BaseClusterDataModelCollector` class.
+- Implement its :py:meth:`~.BaseClusterDataModelCollector.execute` abstract
+  method to return your entire cluster data model that this method should
+  build.
+- Implement its :py:meth:`~.Goal.notification_endpoints` abstract property to
+  return the list of all the :py:class:`~.base.NotificationEndpoint` instances
+  that will be responsible for handling incoming notifications in order to
+  incrementally update your cluster data model.
+
+First of all, you have to extend the :class:`~.BaseClusterDataModelCollector`
+base class which defines the :py:meth:`~.BaseClusterDataModelCollector.execute`
+abstract method you will have to implement. This method is responsible for
+building an entire cluster data model.
+
+Here is an example showing how you can write a plugin called
+``DummyClusterDataModelCollector``:
+
+.. code-block:: python
+
+    # Filepath = <PROJECT_DIR>/thirdparty/dummy.py
+    # Import path = thirdparty.dummy
+
+    from watcher.decision_engine.model import model_root
+    from watcher.decision_engine.model.collector import base
+
+
+    class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
+
+        def execute(self):
+            model = model_root.ModelRoot()
+            # Do something here...
+            return model
+
+        @property
+        def notification_endpoints(self):
+            return []
+
+This implementation is the most basic one. So in order to get a better
+understanding on how to implement a more advanced cluster data model collector,
+have a look at the :py:class:`~.NovaClusterDataModelCollector` class.
+
+Define a custom model
+=====================
+
+As you may have noticed in the above example, we are reusing an existing model
+provided by Watcher. However, this model can be easily customized by
+implementing a new class that would implement the :py:class:`~.Model` abstract
+base class. Here below is simple example on how to proceed in implementing a
+custom Model:
+
+.. code-block:: python
+
+    # Filepath = <PROJECT_DIR>/thirdparty/dummy.py
+    # Import path = thirdparty.dummy
+
+    from watcher.decision_engine.model import base as modelbase
+    from watcher.decision_engine.model.collector import base
+
+
+    class MyModel(modelbase.Model):
+
+        def to_string(self):
+            return 'MyModel'
+
+
+    class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
+
+        def execute(self):
+            model = MyModel()
+            # Do something here...
+            return model
+
+        @property
+        def notification_endpoints(self):
+            return []
+
+Here below is the abstract ``Model`` class that every single cluster data model
+should implement:
+
+.. autoclass:: watcher.decision_engine.model.base.Model
+    :members:
+    :special-members: __init__
+    :noindex:
+
+Define configuration parameters
+===============================
+
+At this point, you have a fully functional cluster data model collector.
+By default, cluster data model collectors define a ``period`` option (see
+:py:meth:`~.BaseClusterDataModelCollector.get_config_opts`) that corresponds
+to the interval of time between each synchronization of the in-memory model.
+
+However, in more complex implementation, you may want to define some
+configuration options so one can tune the cluster data model collector to your
+needs. To do so, you can implement the :py:meth:`~.Loadable.get_config_opts`
+class method as followed:
+
+.. code-block:: python
+
+    from oslo_config import cfg
+    from watcher.decision_engine.model import model_root
+    from watcher.decision_engine.model.collector import base
+
+
+    class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
+
+        def execute(self):
+            model = model_root.ModelRoot()
+            # Do something here...
+            return model
+
+        @property
+        def notification_endpoints(self):
+            return []
+
+        @classmethod
+        def get_config_opts(cls):
+            return super(
+                DummyClusterDataModelCollector, cls).get_config_opts() + [
+                cfg.StrOpt('test_opt', help="Demo Option.", default=0),
+                # Some more options ...
+            ]
+
+The configuration options defined within this class method will be included
+within the global ``watcher.conf`` configuration file under a section named by
+convention: ``{namespace}.{plugin_name}`` (see section :ref:`Register a new
+entry point <register_new_cdmc_entrypoint>`). The namespace for CDMC plugins is
+``watcher_cluster_data_model_collectors``, so in our case, the ``watcher.conf``
+configuration would have to be modified as followed:
+
+.. code-block:: ini
+
+    [watcher_cluster_data_model_collectors.dummy]
+    # Option used for testing.
+    test_opt = test_value
+
+Then, the configuration options you define within this method will then be
+injected in each instantiated object via the  ``config`` parameter of the
+:py:meth:`~.BaseClusterDataModelCollector.__init__` method.
+
+
+Abstract Plugin Class
+=====================
+
+Here below is the abstract ``BaseClusterDataModelCollector`` class that every
+single cluster data model collector should implement:
+
+.. autoclass:: watcher.decision_engine.model.collector.base.BaseClusterDataModelCollector
+    :members:
+    :special-members: __init__
+    :noindex:
+
+
+.. _register_new_cdmc_entrypoint:
+
+Register a new entry point
+==========================
+
+In order for the Watcher Decision Engine to load your new cluster data model
+collector, the latter must be registered as a named entry point under the
+``watcher_cluster_data_model_collectors`` entry point namespace of your
+``setup.py`` file. If you are using pbr_, this entry point should be placed in
+your ``setup.cfg`` file.
+
+The name you give to your entry point has to be unique.
+
+Here below is how to register ``DummyClusterDataModelCollector`` using pbr_:
+
+.. code-block:: ini
+
+    [entry_points]
+    watcher_cluster_data_model_collectors =
+        dummy = thirdparty.dummy:DummyClusterDataModelCollector
+
+.. _pbr: http://docs.openstack.org/developer/pbr/
+
+
+Add new notification endpoints
+==============================
+
+At this point, you have a fully functional cluster data model collector.
+However, this CDMC is only refreshed periodically via a background scheduler.
+As you may sometimes execute a strategy with a stale CDM due to a high activity
+on your infrastructure, you can define some notification endpoints that will be
+responsible for incrementally updating the CDM based on notifications emitted
+by other services such as Nova. To do so, you can implement and register a new
+``DummyEndpoint`` notification endpoint regarding a ``dummy`` event as shown
+below:
+
+.. code-block:: python
+
+    from watcher.decision_engine.model import model_root
+    from watcher.decision_engine.model.collector import base
+
+
+    class DummyNotification(base.NotificationEndpoint):
+
+        @property
+        def filter_rule(self):
+            return filtering.NotificationFilter(
+                publisher_id=r'.*',
+                event_type=r'^dummy$',
+            )
+
+        def info(self, ctxt, publisher_id, event_type, payload, metadata):
+            # Do some CDM modifications here...
+            pass
+
+
+    class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
+
+        def execute(self):
+            model = model_root.ModelRoot()
+            # Do something here...
+            return model
+
+        @property
+        def notification_endpoints(self):
+            return [DummyNotification(self)]
+
+
+Note that if the event you are trying to listen to is published by a new
+service, you may have to also add a new topic Watcher will have to subscribe to
+in the ``notification_topics`` option of the ``[watcher_decision_engine]``
+section.
+
+
+Using cluster data model collector plugins
+==========================================
+
+The Watcher Decision Engine service will automatically discover any installed
+plugins when it is restarted. If a Python package containing a custom plugin is
+installed within the same environment as Watcher, Watcher will automatically
+make that plugin available for use.
+
+At this point, you can use your new cluster data model plugin in your
+:ref:`strategy plugin <implement_strategy_plugin>` by using the
+:py:attr:`~.BaseStrategy.collector_manager` property as followed:
+
+.. code-block:: python
+
+    # [...]
+    dummy_collector = self.collector_manager.get_cluster_model_collector(
+        "dummy")  # "dummy" is the name of the entry point we declared earlier
+    dummy_model = dummy_collector.get_latest_cluster_data_model()
+    # Do some stuff with this model

doc/source/dev/plugin/goal-plugin.rst

@@ -60,8 +60,8 @@ Here is an example showing how you can define a new ``NewGoal`` goal plugin:
     # import path: thirdparty.new
 
     from watcher._i18n import _
+    from watcher.decision_engine.goal import base
     from watcher.decision_engine.goal.efficacy import specs
-    from watcher.decision_engine.strategy.strategies import base
 
     class NewGoal(base.Goal):
 
@@ -79,11 +79,11 @@ Here is an example showing how you can define a new ``NewGoal`` goal plugin:
 
         @classmethod
         def get_efficacy_specification(cls):
-            return specs.UnclassifiedStrategySpecification()
+            return specs.Unclassified()
 
 
 As you may have noticed, the :py:meth:`~.Goal.get_efficacy_specification`
-method returns an :py:meth:`~.UnclassifiedStrategySpecification` instance which
+method returns an :py:meth:`~.Unclassified` instance which
 is provided by Watcher. This efficacy specification is useful during the
 development process of your goal as it corresponds to an empty specification.
 If you want to learn more about what efficacy specifications are used for or to

doc/source/dev/plugin/planner-plugin.rst

@@ -39,7 +39,7 @@ Here is an example showing how you can write a planner plugin called
 
     # Filepath = third-party/third_party/dummy.py
     # Import path = third_party.dummy
-    import uuid
+    from oslo_utils import uuidutils
     from watcher.decision_engine.planner import base
 
 
@@ -47,7 +47,7 @@ Here is an example showing how you can write a planner plugin called
 
         def _create_action_plan(self, context, audit_id):
             action_plan_dict = {
-                'uuid': uuid.uuid4(),
+                'uuid': uuidutils.generate_uuid(),
                 'audit_id': audit_id,
                 'first_action_id': None,
                 'state': objects.action_plan.State.RECOMMENDED
@@ -91,8 +91,10 @@ tune the planner to its needs. To do so, you can implement the
             assert self.config.test_opt == 0
             # [...]
 
-        def get_config_opts(self):
+        @classmethod
-            return [
+        def get_config_opts(cls):
+            return super(
+                DummyPlanner, cls).get_config_opts() + [
                 cfg.StrOpt('test_opt', help="Demo Option.", default=0),
                 # Some more options ...
             ]

doc/source/dev/plugin/scoring-engine-plugin.rst

@@ -0,0 +1,210 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+.. _implement_scoring_engine_plugin:
+
+==========================
+Build a new scoring engine
+==========================
+
+Watcher Decision Engine has an external :ref:`scoring engine
+<scoring_engine_definition>` plugin interface which gives anyone the ability
+to integrate an external scoring engine in order to make use of it in a
+:ref:`strategy <strategy_definition>`.
+
+This section gives some guidelines on how to implement and integrate custom
+scoring engines with Watcher. If you wish to create a third-party package for
+your plugin, you can refer to our :ref:`documentation for third-party package
+creation <plugin-base_setup>`.
+
+
+Pre-requisites
+==============
+
+Because scoring engines execute a purely mathematical tasks, they typically do
+not have any additional dependencies. Additional requirements might be defined
+by specific scoring engine implementations. For example, some scoring engines
+might require to prepare learning data, which has to be loaded during the
+scoring engine startup. Some other might require some external services to be
+available (e.g. if the scoring infrastructure is running in the cloud).
+
+
+Create a new scoring engine plugin
+==================================
+
+In order to create a new scoring engine you have to:
+
+- Extend the :py:class:`~.ScoringEngine` class
+- Implement its :py:meth:`~.ScoringEngine.get_name` method to return the
+  **unique** ID of the new scoring engine you want to create. This unique ID
+  should be the same as the name of :ref:`the entry point we will declare later
+  on <scoring_engine_plugin_add_entrypoint>`.
+- Implement its :py:meth:`~.ScoringEngine.get_description` method to return the
+  user-friendly description of the implemented scoring engine. It might contain
+  information about algorithm used, learning data etc.
+- Implement its :py:meth:`~.ScoringEngine.get_metainfo` method to return the
+  machine-friendly metadata about this scoring engine. For example, it could be
+  a JSON formatted text with information about the data model used, its input
+  and output data format, column names, etc.
+- Implement its :py:meth:`~.ScoringEngine.calculate_score` method to return the
+  result calculated by this scoring engine.
+
+Here is an example showing how you can write a plugin called ``NewScorer``:
+
+.. code-block:: python
+
+    # filepath: thirdparty/new.py
+    # import path: thirdparty.new
+    from watcher.decision_engine.scoring import base
+
+
+    class NewScorer(base.ScoringEngine):
+
+        def get_name(self):
+            return 'new_scorer'
+
+        def get_description(self):
+            return ''
+
+        def get_metainfo(self):
+            return """{
+                "feature_columns": [
+                    "column1",
+                    "column2",
+                    "column3"],
+                "result_columns": [
+                    "value",
+                    "probability"]
+                }"""
+
+        def calculate_score(self, features):
+            return '[12, 0.83]'
+
+As you can see in the above example, the
+:py:meth:`~.ScoringEngine.calculate_score` method returns a string. Both this
+class and the client (caller) should perform all the necessary serialization
+or deserialization.
+
+
+(Optional) Create a new scoring engine container plugin
+=======================================================
+
+Optionally, it's possible to implement a container plugin, which can return a
+list of scoring engines. This list can be re-evaluated multiple times during
+the lifecycle of :ref:`Watcher Decision Engine
+<watcher_decision_engine_definition>` and synchronized with :ref:`Watcher
+Database <watcher_database_definition>` using the ``watcher-sync`` command line
+tool.
+
+Below is an example of a container using some scoring engine implementation
+that is simply made of a client responsible for communicating with a real
+scoring engine deployed as a web service on external servers:
+
+.. code-block:: python
+
+    class NewScoringContainer(base.ScoringEngineContainer):
+
+        @classmethod
+        def get_scoring_engine_list(self):
+            return [
+                RemoteScoringEngine(
+                    name='scoring_engine1',
+                    description='Some remote Scoring Engine 1',
+                    remote_url='http://engine1.example.com/score'),
+                RemoteScoringEngine(
+                    name='scoring_engine2',
+                    description='Some remote Scoring Engine 2',
+                    remote_url='http://engine2.example.com/score'),
+            ]
+
+
+Abstract Plugin Class
+=====================
+
+Here below is the abstract :py:class:`~.ScoringEngine` class:
+
+.. autoclass:: watcher.decision_engine.scoring.base.ScoringEngine
+    :members:
+    :special-members: __init__
+    :noindex:
+
+
+Abstract Plugin Container Class
+===============================
+
+Here below is the abstract :py:class:`~.ScoringContainer` class:
+
+.. autoclass:: watcher.decision_engine.scoring.base.ScoringEngineContainer
+    :members:
+    :special-members: __init__
+    :noindex:
+
+
+.. _scoring_engine_plugin_add_entrypoint:
+
+Add a new entry point
+=====================
+
+In order for the Watcher Decision Engine to load your new scoring engine, it
+must be registered as a named entry point under the ``watcher_scoring_engines``
+entry point of your ``setup.py`` file. If you are using pbr_, this entry point
+should be placed in your ``setup.cfg`` file.
+
+The name you give to your entry point has to be unique and should be the same
+as the value returned by the :py:meth:`~.ScoringEngine.get_name` method of your
+strategy.
+
+Here below is how you would proceed to register ``NewScorer`` using pbr_:
+
+.. code-block:: ini
+
+    [entry_points]
+    watcher_scoring_engines =
+        new_scorer = thirdparty.new:NewScorer
+
+
+To get a better understanding on how to implement a more advanced scoring
+engine, have a look at the :py:class:`~.DummyScorer` class. This implementation
+is not really using machine learning, but other than that it contains all the
+pieces which the "real" implementation would have.
+
+In addition, for some use cases there is a need to register a list (possibly
+dynamic, depending on the implementation and configuration) of scoring engines
+in a single plugin, so there is no need to restart :ref:`Watcher Decision
+Engine <watcher_decision_engine_definition>` every time such list changes. For
+these cases, an additional ``watcher_scoring_engine_containers`` entry point
+can be used.
+
+For the example how to use scoring engine containers, please have a look at
+the :py:class:`~.DummyScoringContainer` and the way it is configured in
+``setup.cfg``. For new containers it could be done like this:
+
+.. code-block:: ini
+
+    [entry_points]
+    watcher_scoring_engine_containers =
+        new_scoring_container = thirdparty.new:NewContainer
+
+.. _pbr: http://docs.openstack.org/developer/pbr/
+
+
+Using scoring engine plugins
+============================
+
+The Watcher Decision Engine service will automatically discover any installed
+plugins when it is restarted. If a Python package containing a custom plugin is
+installed within the same environment as Watcher, Watcher will automatically
+make that plugin available for use.
+
+At this point, Watcher will scan and register inside the :ref:`Watcher Database
+<watcher_database_definition>` all the scoring engines you implemented upon
+restarting the :ref:`Watcher Decision Engine
+<watcher_decision_engine_definition>`.
+
+In addition, ``watcher-sync`` tool can be used to trigger :ref:`Watcher
+Database <watcher_database_definition>` synchronization. This might be used for
+"dynamic" scoring containers, which can return different scoring engines based
+on some external configuration (if they support that).

doc/source/dev/plugin/strategy-plugin.rst

@@ -43,7 +43,7 @@ In order to create a new strategy, you have to:
   Note: Do not use a variable to return the translated string so it can be
   automatically collected by the translation tool.
 - Implement its :py:meth:`~.BaseStrategy.get_translatable_display_name`
-  class method to return the translation key (actually the english display
+  class method to return the translation key (actually the English display
   name) of your new strategy. The value return should be the same as the
   string translated in :py:meth:`~.BaseStrategy.get_display_name`.
 - Implement its :py:meth:`~.BaseStrategy.execute` method to return the
@@ -245,22 +245,30 @@ Querying metrics
 
 A large set of metrics, generated by OpenStack modules, can be used in your
 strategy implementation. To collect these metrics, Watcher provides a
-`Helper`_ to the Ceilometer API, which makes this API reusable and easier
+`Helper`_ for two data sources which are `Ceilometer`_ and `Monasca`_. If you
-to used.
+wish to query metrics from a different data source, you can implement your own
+and directly use it from within your new strategy. Indeed, strategies in
+Watcher have the cluster data models decoupled from the data sources which
+means that you may keep the former while changing the latter.
+The recommended way for you to support a new data source is to implement a new
+helper that would encapsulate within separate methods the queries you need to
+perform. To then use it, you would just have to instantiate it within your
+strategy.
 
-If you want to use your own metrics database backend, please refer to the
+If you want to use Ceilometer but with your own metrics database backend,
-`Ceilometer developer guide`_. Indeed, Ceilometer's pluggable model allows
+please refer to the `Ceilometer developer guide`_. The list of the available
-for various types of backends.  A list of the available backends is located
+Ceilometer backends is located here_. The `Ceilosca`_ project is a good example
-here_. The Ceilosca project is a good example of how to create your own
+of how to create your own pluggable backend. Moreover, if your strategy
-pluggable backend.
+requires new metrics not covered by Ceilometer, you can add them through a
+`Ceilometer plugin`_.
 
-Finally, if your strategy requires new metrics not covered by Ceilometer, you
-can add them through a Ceilometer `plugin`_.
 
-.. _`Helper`: https://github.com/openstack/watcher/blob/master/watcher/metrics_engine/cluster_history/ceilometer.py#L31
+.. _`Helper`: https://github.com/openstack/watcher/blob/master/watcher/decision_engine/cluster/history/ceilometer.py
 .. _`Ceilometer developer guide`: http://docs.openstack.org/developer/ceilometer/architecture.html#storing-the-data
+.. _`Ceilometer`: http://docs.openstack.org/developer/ceilometer/
+.. _`Monasca`: https://github.com/openstack/monasca-api/blob/master/docs/monasca-api-spec.md
 .. _`here`: http://docs.openstack.org/developer/ceilometer/install/dbreco.html#choosing-a-database-backend
-.. _`plugin`: http://docs.openstack.org/developer/ceilometer/plugins.html
+.. _`Ceilometer plugin`: http://docs.openstack.org/developer/ceilometer/plugins.html
 .. _`Ceilosca`: https://github.com/openstack/monasca-ceilometer/blob/master/ceilosca/ceilometer/storage/impl_monasca.py
 
 
@@ -296,16 +304,15 @@ Read usage metrics using the Watcher Cluster History Helper
 
 Here below is the abstract ``BaseClusterHistory`` class of the Helper.
 
-.. autoclass:: watcher.metrics_engine.cluster_history.base.BaseClusterHistory
+.. autoclass:: watcher.decision_engine.cluster.history.base.BaseClusterHistory
     :members:
     :noindex:
 
-
 The following code snippet shows how to create a Cluster History class:
 
 .. code-block:: py
 
-    from watcher.metrics_engine.cluster_history import ceilometer as ceil
+    from watcher.decision_engine.cluster.history import ceilometer as ceil
 
     query_history  = ceil.CeilometerClusterHistory()
 
@@ -313,7 +320,7 @@ Using that you can now query the values for that specific metric:
 
 .. code-block:: py
 
-    query_history.statistic_aggregation(resource_id=hypervisor.uuid,
+    query_history.statistic_aggregation(resource_id=compute_node.uuid,
                                   meter_name='compute.node.cpu.percent',
                                   period="7200",
                                   aggregate='avg'

doc/source/dev/plugins.rst

@@ -18,32 +18,59 @@ use the :ref:`Guru Meditation Reports <watcher_gmr>` to display them.
 Goals
 =====
 
-.. drivers-doc:: watcher_goals
+.. list-plugins:: watcher_goals
+   :detailed:
+
+.. _watcher_scoring_engines:
+
+Scoring Engines
+===============
+
+.. list-plugins:: watcher_scoring_engines
+   :detailed:
+
+.. _watcher_scoring_engine_containers:
+
+Scoring Engine Containers
+=========================
+
+.. list-plugins:: watcher_scoring_engine_containers
+   :detailed:
 
 .. _watcher_strategies:
 
 Strategies
 ==========
 
-.. drivers-doc:: watcher_strategies
+.. list-plugins:: watcher_strategies
+   :detailed:
 
 .. _watcher_actions:
 
 Actions
 =======
 
-.. drivers-doc:: watcher_actions
+.. list-plugins:: watcher_actions
+   :detailed:
 
 .. _watcher_workflow_engines:
 
 Workflow Engines
 ================
 
-.. drivers-doc:: watcher_workflow_engines
+.. list-plugins:: watcher_workflow_engines
+   :detailed:
 
 .. _watcher_planners:
 
 Planners
 ========
 
-.. drivers-doc:: watcher_planners
+.. list-plugins:: watcher_planners
+   :detailed:
+
+Cluster Data Model Collectors
+=============================
+
+.. list-plugins:: watcher_cluster_data_model_collectors
+   :detailed:

doc/source/dev/rally_link.rst

@@ -0,0 +1 @@
+.. include:: ../../../rally-jobs/README.rst

doc/source/glossary.rst

@@ -96,17 +96,17 @@ The :ref:`Cluster <cluster_definition>` may be divided in one or several
 
 .. _cluster_data_model_definition:
 
-Cluster Data Model
+Cluster Data Model (CDM)
-==================
+========================
 
-.. watcher-term:: watcher.metrics_engine.cluster_model_collector.base
+.. watcher-term:: watcher.decision_engine.model.collector.base
 
 .. _cluster_history_definition:
 
 Cluster History
 ===============
 
-.. watcher-term:: watcher.metrics_engine.cluster_history.base
+.. watcher-term:: watcher.decision_engine.cluster.history.base
 
 .. _controller_node_definition:
 
@@ -132,7 +132,7 @@ Compute node
 ============
 
 Please, read `the official OpenStack definition of a Compute Node
-<http://docs.openstack.org/openstack-ops/content/compute_nodes.html>`_.
+<http://docs.openstack.org/ops-guide/arch-compute-nodes.html>`_.
 
 .. _customer_definition:
 
@@ -164,7 +164,8 @@ Goal
 Host Aggregate
 ==============
 
-Please, read `the official OpenStack definition of a Host Aggregate <http://docs.openstack.org/developer/nova/aggregates.html>`_.
+Please, read `the official OpenStack definition of a Host Aggregate
+<http://docs.openstack.org/developer/nova/aggregates.html>`_.
 
 .. _instance_definition:
 
@@ -280,6 +281,12 @@ specific domain.
 Please, read `the official OpenStack definition of a Project
 <http://docs.openstack.org/glossary/content/glossary.html>`_.
 
+.. _scoring_engine_definition:
+
+Scoring Engine
+==============
+
+.. watcher-term::  watcher.api.controllers.v1.scoring_engine
 
 .. _sla_definition:
 

doc/source/image_src/plantuml/sequence_architecture_cdmc_sync.txt

@@ -0,0 +1,41 @@
+@startuml
+skinparam maxMessageSize 100
+
+actor "Administrator"
+
+== Initialization ==
+
+"Administrator" -> "Decision Engine" : Start all services
+"Decision Engine" -> "Background Task Scheduler" : Start
+
+activate "Background Task Scheduler"
+"Background Task Scheduler" -> "Cluster Model Collector Loader"\
+: List available cluster data models
+"Cluster Model Collector Loader" --> "Background Task Scheduler"\
+: list of BaseClusterModelCollector instances
+
+loop for every available cluster data model collector
+    "Background Task Scheduler" -> "Background Task Scheduler"\
+    : add periodic synchronization job
+    create "Jobs Pool"
+    "Background Task Scheduler" -> "Jobs Pool" : Create sync job
+end
+deactivate "Background Task Scheduler"
+
+hnote over "Background Task Scheduler" : Idle
+
+== Job workflow ==
+
+"Background Task Scheduler" -> "Jobs Pool" : Trigger synchronization job
+"Jobs Pool" -> "Nova Cluster Data Model Collector" : synchronize
+
+activate "Nova Cluster Data Model Collector"
+    "Nova Cluster Data Model Collector" -> "Nova API"\
+    : Fetch needed data to build the cluster data model
+    "Nova API" --> "Nova Cluster Data Model Collector" : Needed data
+    "Nova Cluster Data Model Collector" -> "Nova Cluster Data Model Collector"\
+    : Build an in-memory cluster data model
+    ]o<-- "Nova Cluster Data Model Collector" : Done
+deactivate "Nova Cluster Data Model Collector"
+
+@enduml

doc/source/image_src/plantuml/sequence_overview_watcher_usage.txt

@@ -4,7 +4,7 @@ actor Administrator
 
 == Create some Audit settings ==
 
-Administrator -> Watcher : create new Audit Template (i.e. Audit settings : goal, scope, deadline,...)
+Administrator -> Watcher : create new Audit Template (i.e. Audit settings : goal, scope, ...)
 Watcher -> Watcher : save Audit Template in database
 Administrator <-- Watcher : Audit Template UUID
 

doc/source/image_src/plantuml/sequence_trigger_audit_in_decision_engine.txt

@@ -10,45 +10,41 @@ activate "Decision Engine"
 "AMQP Bus" <[#blue]- "Decision Engine" : notify new audit state = ONGOING
 "Decision Engine" -> "Database" : get audit parameters (goal, strategy, ...)
 "Decision Engine" <-- "Database" : audit parameters (goal, strategy, ...)
-"Decision Engine" --> "Decision Engine": select appropriate \
+"Decision Engine" --> "Decision Engine"\
-optimization strategy (via the Strategy Selector)
+: select appropriate optimization strategy (via the Strategy Selector)
 create Strategy
-"Decision Engine" -> "Strategy" : execute()
+"Decision Engine" -> "Strategy" : execute strategy
 activate "Strategy"
-create "Cluster Data Model Collector"
+    "Strategy" -> "Cluster Data Model Collector" : get cluster data model
-"Strategy" -> "Cluster Data Model Collector" : get cluster data model
+    "Cluster Data Model Collector" --> "Strategy"\
-
+    : copy of the in-memory cluster data model
-activate "Cluster Data Model Collector"
+    loop while enough history data for the strategy
-    loop while enough data to build cluster data model
+        "Strategy" -> "Ceilometer API" : get necessary metrics
-      "Cluster Data Model Collector" -> "Nova API" : get resource state (\
+        "Strategy" <-- "Ceilometer API" : aggregated metrics
-host, instance, ...)
-      "Cluster Data Model Collector" <-- "Nova API" : resource state
     end
-"Cluster Data Model Collector" -> "Strategy" : cluster data model
+    "Strategy" -> "Strategy"\
-deactivate "Cluster Data Model Collector"
+    : compute/set needed actions for the solution so it achieves its goal
-
+    "Strategy" -> "Strategy" : compute/set efficacy indicators for the solution
-loop while enough history data for the strategy
+    "Strategy" -> "Strategy" : compute/set the solution global efficacy
-    "Strategy" -> "Ceilometer API": get necessary metrics
+    "Decision Engine" <-- "Strategy"\
-    "Strategy" <-- "Ceilometer API": aggregated metrics
+    : solution (unordered actions, efficacy indicators and global efficacy)
-end
-"Strategy" -> "Strategy" : compute/set needed actions for the solution \
-so it achieves its goal
-"Strategy" -> "Strategy" : compute/set efficacy indicators for the solution
-"Strategy" -> "Strategy" : compute/set the solution global efficacy
-"Decision Engine" <-- "Strategy" : solution (contains a list of unordered \
-actions alongside its efficacy indicators as well as its global efficacy)
 deactivate "Strategy"
 
-"Decision Engine" --> "Planner": load actions scheduler (i.e. Planner plugin)
 create "Planner"
-"Decision Engine" -> "Planner": schedule()
+"Decision Engine" -> "Planner" : load actions scheduler
-"Planner" -> "Planner": schedule actions according to \
+"Planner" --> "Decision Engine" : planner plugin
-scheduling rules/policies
+"Decision Engine" -> "Planner" : schedule actions
-"Decision Engine" <-- "Planner": new action plan
+activate "Planner"
+    "Planner" -> "Planner"\
+    : schedule actions according to scheduling rules/policies
+    "Decision Engine" <-- "Planner" : new action plan
+deactivate "Planner"
 "Decision Engine" -> "Database" : save new action plan in database
 "Decision Engine" -> "Database" : update audit.state = SUCCEEDED
 "AMQP Bus" <[#blue]- "Decision Engine" : notify new audit state = SUCCEEDED
 
 deactivate "Decision Engine"
 
+hnote over "Decision Engine" : Idle
+
 @enduml

doc/source/image_src/plantuml/watcher_db_schema_diagram.txt

@@ -5,11 +5,12 @@
 hide methods
 hide stereotypes
 
-table(goal) {
+table(goals) {
     primary_key(id: Integer)
     uuid : String[36]
     name : String[63]
     display_name : String[63]
+    efficacy_specification : JSONEncodedList, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -18,12 +19,13 @@ table(goal) {
 }
 
 
-table(strategy) {
+table(strategies) {
     primary_key(id: Integer)
     foreign_key(goal_id : Integer)
     uuid : String[36]
     name : String[63]
     display_name : String[63]
+    parameters_spec : JSONEncodedDict, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -32,16 +34,14 @@ table(strategy) {
 }
 
 
-table(audit_template) {
+table(audit_templates) {
     primary_key(id: Integer)
     foreign_key("goal_id : Integer")
     foreign_key("strategy_id : Integer, nullable")
     uuid : String[36]
     name : String[63], nullable
     description : String[255], nullable
-    host_aggregate : Integer, nullable
+    scope : JSONEncodedList
-    extra : JSONEncodedDict
-    version : String[15], nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -50,14 +50,16 @@ table(audit_template) {
 }
 
 
-table(audit) {
+table(audits) {
     primary_key(id: Integer)
-    foreign_key("audit_template_id : Integer")
+    foreign_key("goal_id : Integer")
+    foreign_key("strategy_id : Integer, nullable")
     uuid : String[36]
     audit_type : String[20]
     state : String[20], nullable
-    deadline  :DateTime, nullable
     interval : Integer, nullable
+    parameters : JSONEncodedDict, nullable
+    scope : JSONEncodedList, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -66,9 +68,10 @@ table(audit) {
 }
 
 
-table(action_plan) {
+table(action_plans) {
     primary_key(id: Integer)
     foreign_key("audit_id : Integer, nullable")
+    foreign_key("strategy_id : Integer")
     uuid : String[36]
     first_action_id : Integer
     state : String[20], nullable
@@ -81,7 +84,7 @@ table(action_plan) {
 }
 
 
-table(action) {
+table(actions) {
     primary_key(id: Integer)
     foreign_key("action_plan_id : Integer")
     uuid : String[36]
@@ -97,7 +100,7 @@ table(action) {
 }
 
 
-table(efficacy_indicator) {
+table(efficacy_indicators) {
     primary_key(id: Integer)
     foreign_key("action_plan_id : Integer")
     uuid : String[36]
@@ -112,12 +115,39 @@ table(efficacy_indicator) {
     deleted : Integer
 }
 
- "goal" <.. "strategy" : Foreign Key
+table(scoring_engines) {
- "goal" <.. "audit_template" : Foreign Key
+    primary_key(id: Integer)
- "strategy" <.. "audit_template" : Foreign Key
+    uuid : String[36]
- "audit_template" <.. "audit" : Foreign Key
+    name : String[63]
- "action_plan" <.. "action" : Foreign Key
+    description : String[255], nullable
- "action_plan" <.. "efficacy_indicator" : Foreign Key
+    metainfo : Text, nullable
- "audit" <.. "action_plan" : Foreign Key
+
+    created_at : DateTime
+    updated_at : DateTime
+    deleted_at : DateTime
+    deleted : Integer
+}
+
+table(service) {
+    primary_key(id: Integer)
+    name: String[255]
+    host: String[255]
+    last_seen_up: DateTime
+
+    created_at : DateTime
+    updated_at : DateTime
+    deleted_at : DateTime
+    deleted : Integer
+}
+
+ "goals" <.. "strategies" : Foreign Key
+ "goals" <.. "audit_templates" : Foreign Key
+ "strategies" <.. "audit_templates" : Foreign Key
+ "goals" <.. "audits" : Foreign Key
+ "strategies" <.. "audits" : Foreign Key
+ "action_plans" <.. "actions" : Foreign Key
+ "action_plans" <.. "efficacy_indicators" : Foreign Key
+ "strategies" <.. "action_plans" : Foreign Key
+ "audits" <.. "action_plans" : Foreign Key
 
 @enduml

doc/source/index.rst

@@ -56,7 +56,9 @@ Getting Started
   dev/devstack
   deploy/configuration
   deploy/conf-files
+  dev/notifications
   dev/testing
+  dev/rally_link
 
 API References
 --------------
@@ -74,7 +76,9 @@ Plugins
 
   dev/plugin/base-setup
   dev/plugin/goal-plugin
+  dev/plugin/scoring-engine-plugin
   dev/plugin/strategy-plugin
+  dev/plugin/cdmc-plugin
   dev/plugin/action-plugin
   dev/plugin/planner-plugin
   dev/plugins
@@ -93,6 +97,7 @@ Introduction
   deploy/user-guide
   deploy/policy
   deploy/gmr
+  strategies/strategies
 
 Watcher Manual Pages
 ====================

doc/source/strategies/basic-server-consolidation.rst

@@ -0,0 +1,99 @@
+==================================
+Basic Offline Server Consolidation
+==================================
+
+Synopsis
+--------
+
+**display name**: ``basic``
+
+**goal**: ``server_consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.basic_consolidation
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *basic* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameter is:
+
+====================== ====== ============= ===================================
+parameter              type   default Value description
+====================== ====== ============= ===================================
+``migration_attempts`` Number 0             Maximum number of combinations to
+                                            be tried by the strategy while
+                                            searching for potential candidates.
+                                            To remove the limit, set it to 0
+``period``             Number 7200          The time interval in seconds
+                                            for getting statistic aggregation
+                                            from metric data source
+====================== ====== ============= ===================================
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 server_consolidation --strategy basic
+
+    $ openstack optimize audit create -a at1 -p migration_attempts=4
+
+External Links
+--------------
+None.

doc/source/strategies/outlet_temp_control.rst

@@ -0,0 +1,101 @@
+=================================
+Outlet Temperature Based Strategy
+=================================
+
+Synopsis
+--------
+
+**display name**: ``outlet_temperature``
+
+**goal**: ``thermal_optimization``
+
+Outlet (Exhaust Air) temperature is a new thermal telemetry which can be
+used to measure the host's thermal/workload status. This strategy makes
+decisions to migrate workloads to the hosts with good thermal condition
+(lowest outlet temperature) when the outlet temperature of source hosts
+reach a configurable threshold.
+
+Requirements
+------------
+
+This strategy has a dependency on the host having Intel's Power
+Node Manager 3.0 or later enabled.
+
+
+Metrics
+*******
+
+The *outlet_temperature* strategy requires the following metrics:
+
+========================================= ============ ======= =======
+metric                                    service name plugins comment
+========================================= ============ ======= =======
+``hardware.ipmi.node.outlet_temperature`` ceilometer_  IPMI
+========================================= ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#ipmi-based-meters
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameter is:
+
+============== ====== ============= ====================================
+parameter      type   default Value description
+============== ====== ============= ====================================
+``threshold``  Number 35.0          Temperature threshold for migration
+============== ====== ============= ====================================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Outlet Temperature Based Strategy please refer to:
+https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/outlet-temperature-based-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 thermal_optimization --strategy outlet_temperature
+
+    $ openstack optimize audit create -a at1 -p threshold=31.0
+
+External Links
+--------------
+
+- `Intel Power Node Manager 3.0 <http://www.intel.com/content/www/us/en/power-management/intelligent-power-node-manager-3-0-specification.html>`_

doc/source/strategies/strategies.rst

@@ -0,0 +1,8 @@
+Strategies
+==========
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   ./*

doc/source/strategies/strategy-template.rst

@@ -0,0 +1,115 @@
+=============
+Strategy name
+=============
+
+Synopsis
+--------
+
+**display name**:
+
+**goal**:
+
+Add here a complete description of your strategy
+
+Requirements
+------------
+
+Metrics
+*******
+
+Write here the list of metrics required by your strategy algorithm (in the form
+ of a table). If these metrics requires specific Telemetry plugin or other
+ additional software, please explain here how to deploy them (and add link to
+ dedicated installation guide).
+
+Example:
+
+======================= ============ ======= =======
+metric                  service name plugins comment
+======================= ============ ======= =======
+compute.node.*          ceilometer_  none    one point every 60s
+vm.cpu.utilization_perc monasca_     none
+power                   ceilometer_  kwapi_  one point every 60s
+======================= ============ ======= =======
+
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+.. _monasca: https://github.com/openstack/monasca-agent/blob/master/docs/Libvirt.md
+.. _kwapi: https://kwapi.readthedocs.io/en/latest/index.html
+
+
+Cluster data model
+******************
+
+Default Watcher's cluster data model.
+
+or
+
+If your strategy implementation requires a new cluster data model, please
+ describe it in this section, with a link to model plugin's installation guide.
+
+Actions
+*******
+
+Default Watcher's actions.
+
+or
+
+If your strategy implementation requires new actions, add the list of Action
+ plugins here (in the form of a table) with a link to the plugin's installation
+ procedure.
+
+======== =================
+action   description
+======== =================
+action1_ This action1 ...
+action2_ This action2 ...
+======== =================
+
+.. _action1 : https://github.com/myrepo/watcher/plugins/action1
+.. _action2 : https://github.com/myrepo/watcher/plugins/action2
+
+Planner
+*******
+
+Default Watcher's planner.
+
+or
+
+If your strategy requires also a new planner to schedule built actions in time,
+ please describe it in this section, with a link to planner plugin's
+ installation guide.
+
+Configuration
+-------------
+
+If your strategy use configurable parameters, explain here how to tune them.
+
+
+Efficacy Indicator
+------------------
+
+Add here the Efficacy indicator computed by your strategy.
+
+Algorithm
+---------
+
+Add here either the description of your algorithm or
+link to the existing description.
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ Write the command line to create an audit with your strategy.
+
+External Links
+--------------
+
+If you have written papers, blog articles .... about your strategy into Watcher,
+ or if your strategy is based from external publication(s), please add HTTP
+ links and references in this section.
+
+- `link1 <http://www.link1.papers.com>`_
+- `link2 <http://www.link2.papers.com>`_

doc/source/strategies/uniform_airflow.rst

@@ -0,0 +1,107 @@
+==================================
+Uniform Airflow Migration Strategy
+==================================
+
+Synopsis
+--------
+
+**display name**: ``uniform_airflow``
+
+**goal**: ``airflow_optimization``
+
+.. watcher-term:: watcher.decision_engine.strategy.strategies.uniform_airflow
+
+Requirements
+------------
+
+This strategy has a dependency on the server having Intel's Power
+Node Manager 3.0 or later enabled.
+
+Metrics
+*******
+
+The *uniform_airflow* strategy requires the following metrics:
+
+================================== ============ ======= =======
+metric                             service name plugins comment
+================================== ============ ======= =======
+``hardware.ipmi.node.airflow``     ceilometer_  IPMI
+``hardware.ipmi.node.temperature`` ceilometer_  IPMI
+``hardware.ipmi.node.power``       ceilometer_  IPMI
+================================== ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#ipmi-based-meters
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+====================== ====== ============= ===========================
+parameter              type   default Value description
+====================== ====== ============= ===========================
+``threshold_airflow``  Number 400.0         Airflow threshold for
+                                            migration Unit is 0.1CFM
+``threshold_inlet_t``  Number 28.0          Inlet temperature threshold
+                                            for migration decision
+``threshold_power``    Number 350.0         System power threshold for
+                                            migration decision
+``period``             Number 300           Aggregate time period of
+                                            ceilometer
+====================== ====== ============= ===========================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Uniform Airflow Migration Strategy please refer to:
+https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/uniform-airflow-migration-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 airflow_optimization --strategy uniform_airflow
+
+    $ openstack optimize audit create -a at1 -p threshold_airflow=410 \
+           -p threshold_inlet_t=29.0 -p threshold_power=355.0 -p period=310
+
+External Links
+--------------
+
+- `Intel Power Node Manager 3.0 <http://www.intel.com/content/www/us/en/power-management/intelligent-power-node-manager-3-0-specification.html>`_

doc/source/strategies/vm_workload_consolidation.rst

@@ -0,0 +1,100 @@
+==================================
+VM Workload Consolidation Strategy
+==================================
+
+Synopsis
+--------
+
+**display name**: ``vm_workload_consolidation``
+
+**goal**: ``vm_consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.vm_workload_consolidation
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *vm_workload_consolidation* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory``             	     ceilometer_  none
+``disk.root.size``           ceilometer_  none
+============================ ============ ======= =======
+
+The following metrics are not required but increase the accuracy of
+the strategy if available:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory.usage``             ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+Algorithm
+---------
+
+For more information on the VM Workload consolidation strategy please refer to: https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/zhaw-load-consolidation.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 server_consolidation --strategy vm_workload_consolidation
+
+    $ openstack optimize audit create -a at1
+
+External Links
+--------------
+
+*Spec URL*
+https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/zhaw-load-consolidation.html

doc/source/strategies/workload-stabilization.rst

@@ -0,0 +1,141 @@
+=============================================
+Watcher Overload standard deviation algorithm
+=============================================
+
+Synopsis
+--------
+
+**display name**: ``workload_stabilization``
+
+**goal**: ``workload_balancing``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.workload_stabilization
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *workload_stabilization* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``hardware.memory.used``     ceilometer_  SNMP_
+``cpu_util``                 ceilometer_  none
+``memory.resident``          ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+.. _SNMP: http://docs.openstack.org/admin-guide/telemetry-measurements.html
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+==================== ====== ===================== =============================
+parameter            type   default Value         description
+==================== ====== ===================== =============================
+``metrics``          array  |metrics|             Metrics used as rates of
+                                                  cluster loads.
+``thresholds``       object |thresholds|          Dict where key is a metric
+                                                  and value is a trigger value.
+
+``weights``          object |weights|             These weights used to
+                                                  calculate common standard
+                                                  deviation. Name of weight
+                                                  contains meter name and
+                                                  _weight suffix.
+``instance_metrics`` object |instance_metrics|    Mapping to get hardware
+                                                  statistics using instance
+                                                  metrics.
+``host_choice``      string retry                 Method of host's choice.
+                                                  There are cycle, retry and
+                                                  fullsearch methods. Cycle
+                                                  will iterate hosts in cycle.
+                                                  Retry will get some hosts
+                                                  random (count defined in
+                                                  retry_count option).
+                                                  Fullsearch will return each
+                                                  host from list.
+``retry_count``      number 1                     Count of random returned
+                                                  hosts.
+``periods``          object |periods|             These periods are used to get
+                                                  statistic aggregation for
+                                                  instance and host metrics.
+                                                  The period is simply a
+                                                  repeating interval of time
+                                                  into which the samples are
+                                                  grouped for aggregation.
+                                                  Watcher uses only the last
+                                                  period of all recieved ones.
+==================== ====== ===================== =============================
+
+.. |metrics| replace:: ["cpu_util", "memory.resident"]
+.. |thresholds| replace:: {"cpu_util": 0.2, "memory.resident": 0.2}
+.. |weights| replace:: {"cpu_util_weight": 1.0, "memory.resident_weight": 1.0}
+.. |instance_metrics| replace:: {"cpu_util": "compute.node.cpu.percent", "memory.resident": "hardware.memory.used"}
+.. |periods| replace:: {"instance": 720, "node": 600}
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+Algorithm
+---------
+
+You can find description of overload algorithm and role of standard deviation
+here: https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 workload_balancing --strategy workload_stabilization
+
+    $ openstack optimize audit create -a at1 \
+      -p thresholds='{"memory.resident": 0.05}' \
+      -p metrics='["memory.resident"]'
+
+External Links
+--------------
+
+- `Watcher Overload standard deviation algorithm spec <https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html>`_

doc/source/strategies/workload_balance.rst

@@ -0,0 +1,98 @@
+===================================
+Workload Balance Migration Strategy
+===================================
+
+Synopsis
+--------
+
+**display name**: ``workload_balance``
+
+**goal**: ``workload_balancing``
+
+.. watcher-term:: watcher.decision_engine.strategy.strategies.workload_balance
+
+Requirements
+------------
+
+None.
+
+Metrics
+*******
+
+The *workload_balance* strategy requires the following metrics:
+
+======================= ============ ======= =======
+metric                  service name plugins comment
+======================= ============ ======= =======
+``cpu_util``            ceilometer_  none
+======================= ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+============== ====== ============= ====================================
+parameter      type   default Value description
+============== ====== ============= ====================================
+``threshold``  Number 25.0          Workload threshold for migration
+``period``     Number 300           Aggregate time period of ceilometer
+============== ====== ============= ====================================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Workload Balance Migration Strategy please refer
+to: https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/workload-balance-migration-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 workload_balancing --strategy workload_balance
+
+    $ openstack optimize audit create -a at1 -p threshold=26.0 \
+            -p period=310
+
+External Links
+--------------
+
+None.

etc/watcher/README-watcher.conf.txt

@@ -1,4 +1,4 @@
 To generate the sample watcher.conf file, run the following
 command from the top level of the watcher directory:
 
-tox -econfig
+tox -e genconfig

etc/watcher/policy.json

@@ -31,7 +31,15 @@
     "goal:get": "rule:default",
     "goal:get_all": "rule:default",
 
+    "scoring_engine:detail": "rule:default",
+    "scoring_engine:get": "rule:default",
+    "scoring_engine:get_all": "rule:default",
+
     "strategy:detail": "rule:default",
     "strategy:get": "rule:default",
-    "strategy:get_all": "rule:default"
+    "strategy:get_all": "rule:default",
+
+    "service:detail": "rule:default",
+    "service:get": "rule:default",
+    "service:get_all": "rule:default"
 }

rally-jobs/README.rst

@@ -0,0 +1,42 @@
+Rally job
+=========
+
+We provide, with Watcher, a Rally plugin you can use to benchmark the optimization service.
+
+To launch this task with configured Rally you just need to run:
+
+::
+
+  rally task start watcher/rally-jobs/watcher-watcher.yaml
+
+Structure
+---------
+
+* plugins - directory where you can add rally plugins. Almost everything in
+  Rally is a plugin. Benchmark context, Benchmark scenario, SLA checks, Generic
+  cleanup resources, ....
+
+* extra - all files from this directory will be copy pasted to gates, so you
+  are able to use absolute paths in rally tasks.
+  Files will be located in ~/.rally/extra/*
+
+* watcher.yaml is a task that is run in gates against OpenStack
+  deployed by DevStack
+
+
+Useful links
+------------
+
+* How to install: http://docs.openstack.org/developer/rally/install.html
+
+* How to set Rally up and launch your first scenario:  https://rally.readthedocs.io/en/latest/tutorial/step_1_setting_up_env_and_running_benchmark_from_samples.html
+
+* More about Rally: https://rally.readthedocs.org/en/latest/
+
+* Rally release notes: https://rally.readthedocs.org/en/latest/release_notes.html
+
+* How to add rally-gates: https://rally.readthedocs.org/en/latest/gates.html
+
+* About plugins:  https://rally.readthedocs.org/en/latest/plugins.html
+
+* Plugin samples: https://github.com/openstack/rally/tree/master/samples/plugins

rally-jobs/watcher-watcher.yaml

@@ -0,0 +1,63 @@
+---
+  Watcher.create_audit_and_delete:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+        audit_templates:
+          audit_templates_per_admin: 5
+          fill_strategy: "round_robin"
+          params:
+            - goal:
+                  name: "dummy"
+              strategy:
+                  name: "dummy"
+      sla:
+        failure_rate:
+          max: 0
+
+  Watcher.create_audit_template_and_delete:
+    -
+      args:
+        goal:
+          name: "dummy"
+        strategy:
+          name: "dummy"
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      sla:
+        failure_rate:
+          max: 0
+
+  Watcher.list_audit_templates:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+        audit_templates:
+          audit_templates_per_admin: 5
+          fill_strategy: "random"
+          params:
+            - goal:
+                  name: "workload_balancing"
+              strategy:
+                  name: "workload_stabilization"
+            - goal:
+                  name: "dummy"
+              strategy:
+                  name: "dummy"
+      sla:
+        failure_rate:
+          max: 0

releasenotes/notes/add-plugins-parameters-376eb6b0b8978b44.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - Added a standard way to both declare and fetch
+    configuration options so that whenever the
+    administrator generates the Watcher
+    configuration sample file, it contains the
+    configuration options of the plugins that are
+    currently available.

releasenotes/notes/add-scoring-module-fa00d013ed2d614e.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - Added a generic scoring engine module, which
+    will standardize interactions with scoring engines
+    through the common API. It is possible to use the
+    scoring engine by different Strategies, which
+    improve the code and data model re-use.

releasenotes/notes/audit-versioned-notifications-api-bca7738e16954bad.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Add notifications related to Audit object.

releasenotes/notes/automatic-triggering-audit-8a9b0540d547db60.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Watcher can continuously optimize the OpenStack cloud for a specific
+    strategy or goal by triggering an audit periodically which generates
+    an action plan and run it automatically.

releasenotes/notes/centralise-config-opts-95670987dfbdb0e7.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Centralize all configuration options for Watcher.

releasenotes/notes/cluster-model-objects-wrapper-9c799ea262c56a5b.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Added an in-memory cache of the cluster model
+    built up and kept fresh via notifications from
+    services of interest in addition to periodic
+    syncing logic.

releasenotes/notes/configurable-weights-default-planner-3746b33160bc7347.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Added a way to add a new action without having to
+    amend the source code of the default planner.

releasenotes/notes/continuously-optimization-35364f4d2c0b81fc.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Added a way to create periodic audit to be able to
+    optimize continuously the cloud infrastructure.

releasenotes/notes/db-migration-e1a705a8b54ccdd2.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Watcher database can now be upgraded thanks to Alembic.

releasenotes/notes/define-the-audit-scope-e89edc5051dcf3f2.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Provides a generic way to define the scope of an audit. The set of audited
+    resources will be called "Audit scope" and will be defined in each audit
+    template (which contains the audit settings).

releasenotes/notes/efficacy-indicator-95380ad7b84e3be2.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Added a way to compare the efficacy of different
+    strategies for a give optimization goal.

releasenotes/notes/get-goal-from-strategy-396c9b13a38bb650.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Added a way to return the of available goals depending
+    on which strategies have been deployed on the node
+    where the decision engine is running.

releasenotes/notes/graph-based-cluster-model-523937a6f5e66537.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - The graph model describes how VMs are associated to compute hosts.
+    This allows for seeing relationships upfront between the entities and hence
+    can be used to identify hot/cold spots in the data center and influence
+    a strategy decision.

releasenotes/notes/monasca-support-0b0486b8572ac38b.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Watcher supports multiple metrics backend and relies on Ceilometer and
+    Monasca.

releasenotes/notes/optimization-threshold-21ad38f0470d0e1a.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Allow decision engine to pass strategy parameters,
+    like optimization threshold, to selected strategy,
+    also strategy to provide parameters info to end user.

releasenotes/notes/persistent-audit-parameters-ae41dd7252ba9672.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Copy all audit templates parameters into
+    audit instead of having a reference to the
+    audit template.
+

releasenotes/notes/planner-storage-action-plan-26ef37893c5e8648.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Watcher can now run specific actions in parallel improving the performances
+    dramatically when executing an action plan.

releasenotes/notes/stale-action-plan-b6a6b08df873c128.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add superseded state for an action plan if the cluster data model has
+    changed after it has been created.

releasenotes/notes/standard-deviation-strategy-cd1d0c443fdfde9c.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - Added a strategy that monitors if there is a higher
+    load on some hosts compared to other hosts in the
+    cluster and re-balances the work across hosts to
+    minimize the standard deviation of the loads in
+    the cluster.

releasenotes/notes/uniform-airflow-strategy-68cdba1419c3f770.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Added a new strategy based on the airflow
+    of servers. This strategy makes decisions
+    to migrate VMs to make the airflow uniform.

releasenotes/notes/watcher-notifications-ovo-7b44d52ef6400dd0.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - Provide a notification mechanism into Watcher that supports versioning.
+    Whenever a Watcher object is created, updated or deleted, a versioned
+    notification will, if it's relevant, be automatically sent to notify in order
+    to allow an event-driven style of architecture within Watcher. Moreover, it
+    will also give other services and/or 3rd party softwares (e.g. monitoring
+    solutions or rules engines) the ability to react to such events.

releasenotes/notes/watcher-policies-1e86a30f0f11c6fa.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Added policies to handle user rights
+    to access Watcher API.

releasenotes/notes/watcher-service-list-7b2f4b64f71e9b89.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Add a service supervisor to watch Watcher deamons.

releasenotes/notes/watcher-versioned-objects-fc5abf5c81c4590c.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - all Watcher objects have been refactored to support OVO
+    (oslo.versionedobjects) which was a prerequisite step in order to implement
+    versioned notifications.

releasenotes/notes/workload-balance-migration-strategy-a0b05148a57815c0.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - Added a strategy based on the VM workloads of
+    hypervisors. This strategy makes decisions to
+    migrate workloads to make the total VM workloads
+    of each hypervisor balanced, when the total VM
+    workloads of hypervisor reaches threshold.

releasenotes/source/conf.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-#
 # watcher documentation build configuration file, created by
 # sphinx-quickstart on Fri Jun  3 11:37:52 2016.
 #
@@ -242,3 +240,6 @@ texinfo_documents = [
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
+
+# -- Options for Internationalization output ------------------------------
+locale_dirs = ['locale/']

releasenotes/source/index.rst

@@ -6,5 +6,6 @@ Contents:
 .. toctree::
    :maxdepth: 1
 
-   unreleased.rst
+   unreleased
+   newton
 

releasenotes/source/newton.rst

@@ -0,0 +1,6 @@
+===================================
+ Newton Series Release Notes
+===================================
+
+.. release-notes::
+   :branch: origin/stable/newton

requirements.txt

@@ -5,35 +5,41 @@
 apscheduler # MIT License
 enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
 jsonpatch>=1.1 # BSD
-keystoneauth1>=2.7.0 # Apache-2.0
+keystoneauth1>=2.18.0 # Apache-2.0
-keystonemiddleware!=4.1.0,!=4.5.0,>=4.0.0 # Apache-2.0
+keystonemiddleware>=4.12.0 # Apache-2.0
+lxml!=3.7.0,>=2.3 # BSD
 oslo.concurrency>=3.8.0 # Apache-2.0
 oslo.cache>=1.5.0 # Apache-2.0
-oslo.config>=3.10.0 # Apache-2.0
+oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
-oslo.context>=2.4.0 # Apache-2.0
+oslo.context>=2.9.0 # Apache-2.0
-oslo.db>=4.1.0 # Apache-2.0
+oslo.db>=4.15.0 # Apache-2.0
 oslo.i18n>=2.1.0 # Apache-2.0
-oslo.log>=1.14.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
-oslo.messaging>=5.2.0 # Apache-2.0
+oslo.messaging>=5.14.0 # Apache-2.0
-oslo.policy>=1.9.0 # Apache-2.0
+oslo.policy>=1.17.0 # Apache-2.0
 oslo.reports>=0.6.0 # Apache-2.0
+oslo.serialization>=1.10.0 # Apache-2.0
 oslo.service>=1.10.0 # Apache-2.0
-oslo.utils>=3.14.0 # Apache-2.0
+oslo.utils>=3.18.0 # Apache-2.0
+oslo.versionedobjects>=1.17.0 # Apache-2.0
 PasteDeploy>=1.5.0 # MIT
-pbr>=1.6 # Apache-2.0
+pbr>=1.8 # Apache-2.0
-pecan>=1.0.0 # BSD
+pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
-PrettyTable<0.8,>=0.7 # BSD
+PrettyTable<0.8,>=0.7.1 # BSD
 voluptuous>=0.8.9 # BSD License
-python-ceilometerclient>=2.2.1 # Apache-2.0
+python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0
-python-glanceclient>=2.0.0 # Apache-2.0
+python-glanceclient>=2.5.0 # Apache-2.0
-python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache-2.0
+python-keystoneclient>=3.8.0 # Apache-2.0
-python-neutronclient>=4.2.0 # Apache-2.0
+python-monascaclient>=1.1.0 # Apache-2.0
-python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
+python-neutronclient>=5.1.0 # Apache-2.0
-python-openstackclient>=2.1.0 # Apache-2.0
+python-novaclient!=7.0.0,>=6.0.0 # Apache-2.0
+python-openstackclient>=3.3.0 # Apache-2.0
 six>=1.9.0 # MIT
 SQLAlchemy<1.1.0,>=1.0.10 # MIT
-stevedore>=1.10.0 # Apache-2.0
+stevedore>=1.17.1 # Apache-2.0
-taskflow>=1.26.0 # Apache-2.0
+taskflow>=2.7.0 # Apache-2.0
-WebOb>=1.2.3 # MIT
+WebOb>=1.6.0 # MIT
 WSME>=0.8 # MIT
+networkx>=1.10 # BSD
+

setup.cfg

@@ -1,11 +1,11 @@
 [metadata]
 name = python-watcher
-summary = Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physical resources usage through better VM placement. Watcher can improve your cloud optimization by reducing energy footprint and increasing profits.
+summary = OpenStack Watcher provides a flexible and scalable resource optimization service for multi-tenant OpenStack-based clouds.
 description-file =
     README.rst
 author = OpenStack
 author-email = openstack-dev@lists.openstack.org
-home-page = http://www.openstack.org/
+home-page = http://docs.openstack.org/developer/watcher/
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -17,6 +17,7 @@ classifier =
     Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.5
 
 [files]
 packages =
@@ -31,13 +32,14 @@ setup-hooks =
 
 [entry_points]
 oslo.config.opts =
-    watcher = watcher.opts:list_opts
+    watcher = watcher.conf.opts:list_opts
 
 console_scripts =
     watcher-api = watcher.cmd.api:main
     watcher-db-manage = watcher.cmd.dbmanage:main
     watcher-decision-engine = watcher.cmd.decisionengine:main
     watcher-applier = watcher.cmd.applier:main
+    watcher-sync = watcher.cmd.sync:main
 
 tempest.test_plugins =
     watcher_tests = watcher_tempest_plugin.plugin:WatcherTempestPlugin
@@ -53,8 +55,16 @@ watcher_goals =
     workload_balancing = watcher.decision_engine.goal.goals:WorkloadBalancing
     airflow_optimization = watcher.decision_engine.goal.goals:AirflowOptimization
 
+watcher_scoring_engines =
+    dummy_scorer = watcher.decision_engine.scoring.dummy_scorer:DummyScorer
+
+watcher_scoring_engine_containers =
+    dummy_scoring_container = watcher.decision_engine.scoring.dummy_scoring_container:DummyScoringContainer
+
 watcher_strategies =
     dummy = watcher.decision_engine.strategy.strategies.dummy_strategy:DummyStrategy
+    dummy_with_scorer = watcher.decision_engine.strategy.strategies.dummy_with_scorer:DummyWithScorer
+    dummy_with_resize = watcher.decision_engine.strategy.strategies.dummy_with_resize:DummyWithResize
     basic = watcher.decision_engine.strategy.strategies.basic_consolidation:BasicConsolidation
     outlet_temperature = watcher.decision_engine.strategy.strategies.outlet_temp_control:OutletTempControl
     vm_workload_consolidation = watcher.decision_engine.strategy.strategies.vm_workload_consolidation:VMWorkloadConsolidation
@@ -67,12 +77,17 @@ watcher_actions =
     nop = watcher.applier.actions.nop:Nop
     sleep = watcher.applier.actions.sleep:Sleep
     change_nova_service_state = watcher.applier.actions.change_nova_service_state:ChangeNovaServiceState
+    resize = watcher.applier.actions.resize:Resize
 
 watcher_workflow_engines =
     taskflow = watcher.applier.workflow_engine.default:DefaultWorkFlowEngine
 
 watcher_planners =
-    default  = watcher.decision_engine.planner.default:DefaultPlanner
+    weight = watcher.decision_engine.planner.weight:WeightPlanner
+    workload_stabilization  = watcher.decision_engine.planner.workload_stabilization:WorkloadStabilizationPlanner
+
+watcher_cluster_data_model_collectors =
+    compute = watcher.decision_engine.model.collector.nova:NovaClusterDataModelCollector
 
 [pbr]
 warnerrors = true

test-requirements.txt

@@ -2,26 +2,25 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-coverage>=3.6 # Apache-2.0
+coverage>=4.0 # Apache-2.0
-discover # BSD
 doc8 # Apache-2.0
-freezegun # Apache-2.0
+freezegun>=0.3.6 # Apache-2.0
 hacking<0.11,>=0.10.2
 mock>=2.0 # BSD
 oslotest>=1.10.0 # Apache-2.0
-os-testr>=0.7.0 # Apache-2.0
+os-testr>=0.8.0 # Apache-2.0
 python-subunit>=0.0.18 # Apache-2.0/BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
 
 # Doc requirements
-oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+oslosphinx>=4.7.0 # Apache-2.0
-sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
 sphinxcontrib-pecanwsme>=0.8 # Apache-2.0
 
 # releasenotes
-reno>=1.8.0 # Apache2
+reno>=1.8.0 # Apache-2.0
 
 # bandit
-bandit>=1.0.1 # Apache-2.0
+bandit>=1.1.0 # Apache-2.0

tox.ini

@@ -1,33 +1,33 @@
 [tox]
-minversion = 1.6
+minversion = 1.8
-envlist = py34,py27,pep8
+envlist = py35,py34,py27,pep8
 skipsdist = True
 
 [testenv]
 usedevelop = True
 whitelist_externals = find
-install_command = pip install -U {opts} {packages}
+install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/requirements.txt
+deps = -r{toxinidir}/test-requirements.txt
-       -r{toxinidir}/test-requirements.txt
 commands =
-  find . -type f -name "*.pyc" -delete
+  find . -type f -name "*.py[c|o]" -delete
-  find . -type d -name "__pycache__" -delete
   ostestr --concurrency=6 {posargs}
 
 [testenv:pep8]
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    bandit -r watcher -x tests -n5 -ll
+    bandit -r watcher -x tests -n5 -ll -s B320
 
 [testenv:venv]
 setenv = PYTHONHASHSEED=0
 commands = {posargs}
 
 [testenv:cover]
-commands = python setup.py testr --coverage --testr-args='{posargs}'
+commands =
+    python setup.py testr --coverage --testr-args='{posargs}'
+    coverage report
 
 [testenv:docs]
 setenv = PYTHONHASHSEED=0
@@ -38,7 +38,7 @@ commands =
 [testenv:debug]
 commands = oslo_debug_helper -t watcher/tests {posargs}
 
-[testenv:config]
+[testenv:genconfig]
 sitepackages = False
 commands =
     oslo-config-generator --config-file etc/watcher/watcher-config-generator.conf
@@ -54,6 +54,7 @@ commands = python setup.py bdist_wheel
 
 [hacking]
 import_exceptions = watcher._i18n
+local-check-factory = watcher.hacking.checks.factory
 
 [doc8]
 extension=.rst
@@ -65,4 +66,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen
 
 [testenv:bandit]
 deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r watcher -x tests -n5 -ll
+commands = bandit -r watcher -x tests -n5 -ll -s B320

watcher/__init__.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at

watcher/api/acl.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 #
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
+# Copyright (c) 2016 Intel Corp
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -17,32 +18,23 @@
 
 """Access Control Lists (ACL's) control access the API server."""
 
-from oslo_config import cfg
 from watcher.api.middleware import auth_token
+from watcher import conf
 
-
+CONF = conf.CONF
-AUTH_OPTS = [
-    cfg.BoolOpt('enable_authentication',
-                default=True,
-                help='This option enables or disables user authentication '
-                'via keystone. Default value is True.'),
-]
-
-CONF = cfg.CONF
-CONF.register_opts(AUTH_OPTS)
 
 
 def install(app, conf, public_routes):
     """Install ACL check on application.
 
-    :param app: A WSGI applicatin.
+    :param app: A WSGI application.
     :param conf: Settings. Dict'ified and passed to keystonemiddleware
     :param public_routes: The list of the routes which will be allowed to
                           access without authentication.
     :return: The same WSGI application with ACL installed.
 
     """
-    if not cfg.CONF.get('enable_authentication'):
+    if not CONF.get('enable_authentication'):
         return app
     return auth_token.AuthTokenMiddleware(app,
                                           conf=dict(conf),

watcher/api/app.py

@@ -2,6 +2,7 @@
 
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
 # All Rights Reserved.
+# Copyright (c) 2016 Intel Corp
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -16,49 +17,14 @@
 #    under the License.
 
 
-from oslo_config import cfg
 import pecan
 
-from watcher._i18n import _
 from watcher.api import acl
 from watcher.api import config as api_config
 from watcher.api import middleware
+from watcher import conf
 
-# Register options for the service
+CONF = conf.CONF
-API_SERVICE_OPTS = [
-    cfg.PortOpt('port',
-                default=9322,
-                help=_('The port for the watcher API server')),
-    cfg.StrOpt('host',
-               default='127.0.0.1',
-               help=_('The listen IP for the watcher API server')),
-    cfg.IntOpt('max_limit',
-               default=1000,
-               help=_('The maximum number of items returned in a single '
-                      'response from a collection resource')),
-    cfg.IntOpt('workers',
-               min=1,
-               help=_('Number of workers for Watcher API service. '
-                      'The default is equal to the number of CPUs available '
-                      'if that can be determined, else a default worker '
-                      'count of 1 is returned.')),
-
-    cfg.BoolOpt('enable_ssl_api',
-                default=False,
-                help=_("Enable the integrated stand-alone API to service "
-                       "requests via HTTPS instead of HTTP. If there is a "
-                       "front-end service performing HTTPS offloading from "
-                       "the service, this option should be False; note, you "
-                       "will want to change public API endpoint to represent "
-                       "SSL termination URL with 'public_endpoint' option.")),
-]
-
-CONF = cfg.CONF
-opt_group = cfg.OptGroup(name='api',
-                         title='Options for the watcher-api service')
-
-CONF.register_group(opt_group)
-CONF.register_opts(API_SERVICE_OPTS, opt_group)
 
 
 def get_pecan_config():

watcher/api/controllers/root.py

@@ -61,7 +61,7 @@ class Root(base.APIBase):
         root = Root()
         root.name = "OpenStack Watcher API"
         root.description = ("Watcher is an OpenStack project which aims to "
-                            "to improve physical resources usage through "
+                            "improve physical resources usage through "
                             "better VM placement.")
         root.versions = [Version.convert('v1')]
         root.default_version = Version.convert('v1')

watcher/api/controllers/v1/__init__.py

@@ -34,6 +34,8 @@ from watcher.api.controllers.v1 import action_plan
 from watcher.api.controllers.v1 import audit
 from watcher.api.controllers.v1 import audit_template
 from watcher.api.controllers.v1 import goal
+from watcher.api.controllers.v1 import scoring_engine
+from watcher.api.controllers.v1 import service
 from watcher.api.controllers.v1 import strategy
 
 
@@ -101,6 +103,12 @@ class V1(APIBase):
     action_plans = [link.Link]
     """Links to the action plans resource"""
 
+    scoring_engines = [link.Link]
+    """Links to the Scoring Engines resource"""
+
+    services = [link.Link]
+    """Links to the services resource"""
+
     links = [link.Link]
     """Links that point to a specific URL for this version and documentation"""
 
@@ -147,6 +155,22 @@ class V1(APIBase):
                                 'action_plans', '',
                                 bookmark=True)
             ]
+
+        v1.scoring_engines = [link.Link.make_link(
+            'self', pecan.request.host_url, 'scoring_engines', ''),
+            link.Link.make_link('bookmark',
+                                pecan.request.host_url,
+                                'scoring_engines', '',
+                                bookmark=True)
+            ]
+
+        v1.services = [link.Link.make_link(
+            'self', pecan.request.host_url, 'services', ''),
+            link.Link.make_link('bookmark',
+                                pecan.request.host_url,
+                                'services', '',
+                                bookmark=True)
+            ]
         return v1
 
 
@@ -158,6 +182,8 @@ class Controller(rest.RestController):
     actions = action.ActionsController()
     action_plans = action_plan.ActionPlansController()
     goals = goal.GoalsController()
+    scoring_engines = scoring_engine.ScoringEngineController()
+    services = service.ServicesController()
     strategies = strategy.StrategiesController()
 
     @wsme_pecan.wsexpose(V1)

watcher/api/controllers/v1/action.py

@@ -27,7 +27,7 @@ of the OpenStack :ref:`Cluster <cluster_definition>` such as:
 -  Live migration of an instance from one compute node to another compute
    node with Nova
 -  Changing the power level of a compute node (ACPI level, ...)
--  Changing the current state of an hypervisor (enable or disable) with Nova
+-  Changing the current state of a compute node (enable or disable) with Nova
 
 In most cases, an :ref:`Action <action_definition>` triggers some concrete
 commands on an existing OpenStack module (Nova, Neutron, Cinder, Ironic, etc.).
@@ -41,7 +41,7 @@ be one of the following:
    processed by the :ref:`Watcher Applier <watcher_applier_definition>`
 -  **SUCCEEDED** : the :ref:`Action <action_definition>` has been executed
    successfully
--  **FAILED** : an error occured while trying to execute the
+-  **FAILED** : an error occurred while trying to execute the
    :ref:`Action <action_definition>`
 -  **DELETED** : the :ref:`Action <action_definition>` is still stored in the
    :ref:`Watcher database <watcher_database_definition>` but is not returned
@@ -88,7 +88,6 @@ class Action(base.APIBase):
     between the internal object model and the API representation of a action.
     """
     _action_plan_uuid = None
-    _next_uuid = None
 
     def _get_action_plan_uuid(self):
         return self._action_plan_uuid
@@ -105,22 +104,6 @@ class Action(base.APIBase):
             except exception.ActionPlanNotFound:
                 self._action_plan_uuid = None
 
-    def _get_next_uuid(self):
-        return self._next_uuid
-
-    def _set_next_uuid(self, value):
-        if value == wtypes.Unset:
-            self._next_uuid = wtypes.Unset
-        elif value and self._next_uuid != value:
-            try:
-                action_next = objects.Action.get(
-                    pecan.request.context, value)
-                self._next_uuid = action_next.uuid
-                self.next = action_next.id
-            except exception.ActionNotFound:
-                self.action_next_uuid = None
-                # raise e
-
     uuid = wtypes.wsattr(types.uuid, readonly=True)
     """Unique UUID for this action"""
 
@@ -138,10 +121,8 @@ class Action(base.APIBase):
     input_parameters = types.jsontype
     """One or more key/value pairs """
 
-    next_uuid = wsme.wsproperty(types.uuid, _get_next_uuid,
+    parents = wtypes.wsattr(types.jsontype, readonly=True)
-                                _set_next_uuid,
+    """UUIDs of parent actions"""
-                                mandatory=True)
-    """This next action UUID"""
 
     links = wsme.wsattr([link.Link], readonly=True)
     """A list containing a self link and associated action links"""
@@ -151,10 +132,7 @@ class Action(base.APIBase):
 
         self.fields = []
         fields = list(objects.Action.fields)
-        # audit_template_uuid is not part of objects.Audit.fields
-        # because it's an API-only attribute.
         fields.append('action_plan_uuid')
-        fields.append('next_uuid')
         for field in fields:
             # Skip fields we do not expose.
             if not hasattr(self, field):
@@ -165,15 +143,13 @@ class Action(base.APIBase):
         self.fields.append('action_plan_id')
         setattr(self, 'action_plan_uuid', kwargs.get('action_plan_id',
                 wtypes.Unset))
-        setattr(self, 'next_uuid', kwargs.get('next',
-                wtypes.Unset))
 
     @staticmethod
     def _convert_with_links(action, url, expand=True):
         if not expand:
-            action.unset_fields_except(['uuid', 'state', 'next', 'next_uuid',
+            action.unset_fields_except(['uuid', 'state', 'action_plan_uuid',
-                                        'action_plan_uuid', 'action_plan_id',
+                                        'action_plan_id', 'action_type',
-                                        'action_type'])
+                                        'parents'])
 
         action.links = [link.Link.make_link('self', url,
                                             'actions', action.uuid),
@@ -195,9 +171,9 @@ class Action(base.APIBase):
                      state='PENDING',
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
-                     updated_at=datetime.datetime.utcnow())
+                     updated_at=datetime.datetime.utcnow(),
+                     parents=[])
         sample._action_plan_uuid = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
-        sample._next_uuid = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -218,17 +194,6 @@ class ActionCollection(collection.Collection):
         collection.actions = [Action.convert_with_links(p, expand)
                               for p in actions]
 
-        if 'sort_key' in kwargs:
-            reverse = False
-            if kwargs['sort_key'] == 'next_uuid':
-                if 'sort_dir' in kwargs:
-                    reverse = True if kwargs['sort_dir'] == 'desc' else False
-                collection.actions = sorted(
-                    collection.actions,
-                    key=lambda action: action.next_uuid or '',
-                    reverse=reverse)
-
-        collection.next = collection.get_next(limit, url=url, **kwargs)
         return collection
 
     @classmethod
@@ -270,10 +235,7 @@ class ActionsController(rest.RestController):
         if audit_uuid:
             filters['audit_uuid'] = audit_uuid
 
-        if sort_key == 'next_uuid':
+        sort_db_key = sort_key
-            sort_db_key = None
-        else:
-            sort_db_key = sort_key
 
         actions = objects.Action.list(pecan.request.context,
                                       limit,
@@ -381,7 +343,7 @@ class ActionsController(rest.RestController):
         action_dict = action.as_dict()
         context = pecan.request.context
         new_action = objects.Action(context, **action_dict)
-        new_action.create(context)
+        new_action.create()
 
         # Set the HTTP Location Header
         pecan.response.location = link.build_url('actions', new_action.uuid)

watcher/api/controllers/v1/action_plan.py

@@ -73,6 +73,7 @@ from watcher.api.controllers.v1 import utils as api_utils
 from watcher.applier import rpcapi
 from watcher.common import exception
 from watcher.common import policy
+from watcher.common import utils
 from watcher import objects
 from watcher.objects import action_plan as ap_objects
 
@@ -105,7 +106,7 @@ class ActionPlanPatchType(types.JsonPatchType):
 
     @staticmethod
     def mandatory_attrs():
-        return ["audit_id", "state", "first_action_id"]
+        return ["audit_id", "state"]
 
 
 class ActionPlan(base.APIBase):
@@ -117,7 +118,8 @@ class ActionPlan(base.APIBase):
     """
 
     _audit_uuid = None
-    _first_action_uuid = None
+    _strategy_uuid = None
+    _strategy_name = None
     _efficacy_indicators = None
 
     def _get_audit_uuid(self):
@@ -134,21 +136,6 @@ class ActionPlan(base.APIBase):
             except exception.AuditNotFound:
                 self._audit_uuid = None
 
-    def _get_first_action_uuid(self):
-        return self._first_action_uuid
-
-    def _set_first_action_uuid(self, value):
-        if value == wtypes.Unset:
-            self._first_action_uuid = wtypes.Unset
-        elif value and self._first_action_uuid != value:
-            try:
-                first_action = objects.Action.get(pecan.request.context,
-                                                  value)
-                self._first_action_uuid = first_action.uuid
-                self.first_action_id = first_action.id
-            except exception.ActionNotFound:
-                self._first_action_uuid = None
-
     def _get_efficacy_indicators(self):
         if self._efficacy_indicators is None:
             self._set_efficacy_indicators(wtypes.Unset)
@@ -177,18 +164,58 @@ class ActionPlan(base.APIBase):
         elif value and self._efficacy_indicators != value:
             self._efficacy_indicators = value
 
+    def _get_strategy(self, value):
+        if value == wtypes.Unset:
+            return None
+        strategy = None
+        try:
+            if utils.is_uuid_like(value) or utils.is_int_like(value):
+                strategy = objects.Strategy.get(
+                    pecan.request.context, value)
+            else:
+                strategy = objects.Strategy.get_by_name(
+                    pecan.request.context, value)
+        except exception.StrategyNotFound:
+            pass
+        if strategy:
+            self.strategy_id = strategy.id
+        return strategy
+
+    def _get_strategy_uuid(self):
+        return self._strategy_uuid
+
+    def _set_strategy_uuid(self, value):
+        if value and self._strategy_uuid != value:
+            self._strategy_uuid = None
+            strategy = self._get_strategy(value)
+            if strategy:
+                self._strategy_uuid = strategy.uuid
+
+    def _get_strategy_name(self):
+        return self._strategy_name
+
+    def _set_strategy_name(self, value):
+        if value and self._strategy_name != value:
+            self._strategy_name = None
+            strategy = self._get_strategy(value)
+            if strategy:
+                self._strategy_name = strategy.name
+
     uuid = wtypes.wsattr(types.uuid, readonly=True)
     """Unique UUID for this action plan"""
 
-    first_action_uuid = wsme.wsproperty(
-        types.uuid, _get_first_action_uuid, _set_first_action_uuid,
-        mandatory=True)
-    """The UUID of the first action this action plans links to"""
-
     audit_uuid = wsme.wsproperty(types.uuid, _get_audit_uuid, _set_audit_uuid,
                                  mandatory=True)
     """The UUID of the audit this port belongs to"""
 
+    strategy_uuid = wsme.wsproperty(
+        wtypes.text, _get_strategy_uuid, _set_strategy_uuid, mandatory=False)
+    """Strategy UUID the action plan refers to"""
+
+    strategy_name = wsme.wsproperty(
+        wtypes.text, _get_strategy_name, _set_strategy_name, mandatory=False)
+    """The name of the strategy this action plan refers to"""
+
     efficacy_indicators = wsme.wsproperty(
         types.jsontype, _get_efficacy_indicators, _set_efficacy_indicators,
         mandatory=True)
@@ -215,19 +242,20 @@ class ActionPlan(base.APIBase):
             setattr(self, field, kwargs.get(field, wtypes.Unset))
 
         self.fields.append('audit_uuid')
-        self.fields.append('first_action_uuid')
         self.fields.append('efficacy_indicators')
 
         setattr(self, 'audit_uuid', kwargs.get('audit_id', wtypes.Unset))
-        setattr(self, 'first_action_uuid',
+        fields.append('strategy_uuid')
-                kwargs.get('first_action_id', wtypes.Unset))
+        setattr(self, 'strategy_uuid', kwargs.get('strategy_id', wtypes.Unset))
+        fields.append('strategy_name')
+        setattr(self, 'strategy_name', kwargs.get('strategy_id', wtypes.Unset))
 
     @staticmethod
     def _convert_with_links(action_plan, url, expand=True):
         if not expand:
             action_plan.unset_fields_except(
                 ['uuid', 'state', 'efficacy_indicators', 'global_efficacy',
-                 'updated_at', 'audit_uuid', 'first_action_uuid'])
+                 'updated_at', 'audit_uuid', 'strategy_uuid', 'strategy_name'])
 
         action_plan.links = [
             link.Link.make_link(
@@ -252,7 +280,6 @@ class ActionPlan(base.APIBase):
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
                      updated_at=datetime.datetime.utcnow())
-        sample._first_action_uuid = '57eaf9ab-5aaa-4f7e-bdf7-9a140ac7a720'
         sample._audit_uuid = 'abcee106-14d3-4515-b744-5a26885cf6f6'
         sample._efficacy_indicators = [{'description': 'Test indicator',
                                         'name': 'test_indicator',
@@ -275,8 +302,8 @@ class ActionPlanCollection(collection.Collection):
     @staticmethod
     def convert_with_links(rpc_action_plans, limit, url=None, expand=False,
                            **kwargs):
-        collection = ActionPlanCollection()
+        ap_collection = ActionPlanCollection()
-        collection.action_plans = [ActionPlan.convert_with_links(
+        ap_collection.action_plans = [ActionPlan.convert_with_links(
             p, expand) for p in rpc_action_plans]
 
         if 'sort_key' in kwargs:
@@ -284,13 +311,13 @@ class ActionPlanCollection(collection.Collection):
             if kwargs['sort_key'] == 'audit_uuid':
                 if 'sort_dir' in kwargs:
                     reverse = True if kwargs['sort_dir'] == 'desc' else False
-                collection.action_plans = sorted(
+                ap_collection.action_plans = sorted(
-                    collection.action_plans,
+                    ap_collection.action_plans,
                     key=lambda action_plan: action_plan.audit_uuid,
                     reverse=reverse)
 
-        collection.next = collection.get_next(limit, url=url, **kwargs)
+        ap_collection.next = ap_collection.get_next(limit, url=url, **kwargs)
-        return collection
+        return ap_collection
 
     @classmethod
     def sample(cls):
@@ -301,6 +328,7 @@ class ActionPlanCollection(collection.Collection):
 
 class ActionPlansController(rest.RestController):
     """REST controller for Actions."""
+
     def __init__(self):
         super(ActionPlansController, self).__init__()
 
@@ -314,7 +342,8 @@ class ActionPlansController(rest.RestController):
 
     def _get_action_plans_collection(self, marker, limit,
                                      sort_key, sort_dir, expand=False,
-                                     resource_url=None, audit_uuid=None):
+                                     resource_url=None, audit_uuid=None,
+                                     strategy=None):
 
         limit = api_utils.validate_limit(limit)
         api_utils.validate_sort_dir(sort_dir)
@@ -328,6 +357,12 @@ class ActionPlansController(rest.RestController):
         if audit_uuid:
             filters['audit_uuid'] = audit_uuid
 
+        if strategy:
+            if utils.is_uuid_like(strategy):
+                filters['strategy_uuid'] = strategy
+            else:
+                filters['strategy_name'] = strategy
+
         if sort_key == 'audit_uuid':
             sort_db_key = None
         else:
@@ -347,9 +382,9 @@ class ActionPlansController(rest.RestController):
             sort_dir=sort_dir)
 
     @wsme_pecan.wsexpose(ActionPlanCollection, types.uuid, int, wtypes.text,
-                         wtypes.text, types.uuid)
+                         wtypes.text, types.uuid, wtypes.text)
     def get_all(self, marker=None, limit=None,
-                sort_key='id', sort_dir='asc', audit_uuid=None):
+                sort_key='id', sort_dir='asc', audit_uuid=None, strategy=None):
         """Retrieve a list of action plans.
 
         :param marker: pagination marker for large data sets.
@@ -358,18 +393,20 @@ class ActionPlansController(rest.RestController):
         :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
         :param audit_uuid: Optional UUID of an audit, to get only actions
             for that audit.
+        :param strategy: strategy UUID or name to filter by
         """
         context = pecan.request.context
         policy.enforce(context, 'action_plan:get_all',
                        action='action_plan:get_all')
 
         return self._get_action_plans_collection(
-            marker, limit, sort_key, sort_dir, audit_uuid=audit_uuid)
+            marker, limit, sort_key, sort_dir,
+            audit_uuid=audit_uuid, strategy=strategy)
 
     @wsme_pecan.wsexpose(ActionPlanCollection, types.uuid, int, wtypes.text,
-                         wtypes.text, types.uuid)
+                         wtypes.text, types.uuid, wtypes.text)
     def detail(self, marker=None, limit=None,
-               sort_key='id', sort_dir='asc', audit_uuid=None):
+               sort_key='id', sort_dir='asc', audit_uuid=None, strategy=None):
         """Retrieve a list of action_plans with detail.
 
         :param marker: pagination marker for large data sets.
@@ -378,6 +415,7 @@ class ActionPlansController(rest.RestController):
         :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
         :param audit_uuid: Optional UUID of an audit, to get only actions
             for that audit.
+        :param strategy: strategy UUID or name to filter by
         """
         context = pecan.request.context
         policy.enforce(context, 'action_plan:detail',
@@ -391,9 +429,8 @@ class ActionPlansController(rest.RestController):
         expand = True
         resource_url = '/'.join(['action_plans', 'detail'])
         return self._get_action_plans_collection(
-            marker, limit,
+            marker, limit, sort_key, sort_dir, expand,
-            sort_key, sort_dir, expand,
+            resource_url, audit_uuid=audit_uuid, strategy=strategy)
-            resource_url, audit_uuid=audit_uuid)
 
     @wsme_pecan.wsexpose(ActionPlan, types.uuid)
     def get_one(self, action_plan_uuid):
@@ -433,7 +470,6 @@ class ActionPlansController(rest.RestController):
         :param action_plan_uuid: UUID of a action plan.
         :param patch: a json PATCH document to apply to this action plan.
         """
-        launch_action_plan = True
         if self.from_actionsPlans:
             raise exception.OperationNotPermitted
 
@@ -491,8 +527,8 @@ class ActionPlansController(rest.RestController):
             if action_plan_to_update[field] != patch_val:
                 action_plan_to_update[field] = patch_val
 
-            if (field == 'state'
+            if (field == 'state'and
-                    and patch_val == objects.action_plan.State.PENDING):
+                    patch_val == objects.action_plan.State.PENDING):
                 launch_action_plan = True
 
         action_plan_to_update.save()

watcher/api/controllers/v1/audit.py

@@ -52,12 +52,14 @@ from watcher import objects
 
 class AuditPostType(wtypes.Base):
 
-    audit_template_uuid = wtypes.wsattr(types.uuid, mandatory=True)
+    audit_template_uuid = wtypes.wsattr(types.uuid, mandatory=False)
+
+    goal = wtypes.wsattr(wtypes.text, mandatory=False)
+
+    strategy = wtypes.wsattr(wtypes.text, mandatory=False)
 
     audit_type = wtypes.wsattr(wtypes.text, mandatory=True)
 
-    deadline = wtypes.wsattr(datetime.datetime, mandatory=False)
-
     state = wsme.wsattr(wtypes.text, readonly=True,
                         default=objects.audit.State.PENDING)
 
@@ -65,26 +67,58 @@ class AuditPostType(wtypes.Base):
                                default={})
     interval = wsme.wsattr(int, mandatory=False)
 
-    def as_audit(self):
+    scope = wtypes.wsattr(types.jsontype, readonly=True)
+
+    auto_trigger = wtypes.wsattr(bool, mandatory=False)
+
+    def as_audit(self, context):
         audit_type_values = [val.value for val in objects.audit.AuditType]
         if self.audit_type not in audit_type_values:
             raise exception.AuditTypeNotFound(audit_type=self.audit_type)
 
         if (self.audit_type == objects.audit.AuditType.ONESHOT.value and
-           self.interval != wtypes.Unset):
+                self.interval not in (wtypes.Unset, None)):
             raise exception.AuditIntervalNotAllowed(audit_type=self.audit_type)
 
         if (self.audit_type == objects.audit.AuditType.CONTINUOUS.value and
-           self.interval == wtypes.Unset):
+                self.interval in (wtypes.Unset, None)):
             raise exception.AuditIntervalNotSpecified(
                 audit_type=self.audit_type)
 
+        # If audit_template_uuid was provided, we will provide any
+        # variables not included in the request, but not override
+        # those variables that were included.
+        if self.audit_template_uuid:
+            try:
+                audit_template = objects.AuditTemplate.get(
+                    context, self.audit_template_uuid)
+            except exception.AuditTemplateNotFound:
+                raise exception.Invalid(
+                    message=_('The audit template UUID or name specified is '
+                              'invalid'))
+            at2a = {
+                'goal': 'goal_id',
+                'strategy': 'strategy_id',
+                'scope': 'scope',
+            }
+            to_string_fields = set(['goal', 'strategy'])
+            for k in at2a:
+                if not getattr(self, k):
+                    try:
+                        at_attr = getattr(audit_template, at2a[k])
+                        if at_attr and (k in to_string_fields):
+                            at_attr = str(at_attr)
+                        setattr(self, k, at_attr)
+                    except AttributeError:
+                        pass
         return Audit(
-            audit_template_id=self.audit_template_uuid,
             audit_type=self.audit_type,
-            deadline=self.deadline,
             parameters=self.parameters,
-            interval=self.interval)
+            goal_id=self.goal,
+            strategy_id=self.strategy,
+            interval=self.interval,
+            scope=self.scope,
+            auto_trigger=self.auto_trigger)
 
 
 class AuditPatchType(types.JsonPatchType):
@@ -110,45 +144,84 @@ class Audit(base.APIBase):
     This class enforces type checking and value constraints, and converts
     between the internal object model and the API representation of a audit.
     """
-    _audit_template_uuid = None
+    _goal_uuid = None
-    _audit_template_name = None
+    _goal_name = None
+    _strategy_uuid = None
+    _strategy_name = None
 
-    def _get_audit_template(self, value):
+    def _get_goal(self, value):
         if value == wtypes.Unset:
             return None
-        audit_template = None
+        goal = None
         try:
             if utils.is_uuid_like(value) or utils.is_int_like(value):
-                audit_template = objects.AuditTemplate.get(
+                goal = objects.Goal.get(
                     pecan.request.context, value)
             else:
-                audit_template = objects.AuditTemplate.get_by_name(
+                goal = objects.Goal.get_by_name(
                     pecan.request.context, value)
-        except exception.AuditTemplateNotFound:
+        except exception.GoalNotFound:
             pass
-        if audit_template:
+        if goal:
-            self.audit_template_id = audit_template.id
+            self.goal_id = goal.id
-        return audit_template
+        return goal
 
-    def _get_audit_template_uuid(self):
+    def _get_goal_uuid(self):
-        return self._audit_template_uuid
+        return self._goal_uuid
 
-    def _set_audit_template_uuid(self, value):
+    def _set_goal_uuid(self, value):
-        if value and self._audit_template_uuid != value:
+        if value and self._goal_uuid != value:
-            self._audit_template_uuid = None
+            self._goal_uuid = None
-            audit_template = self._get_audit_template(value)
+            goal = self._get_goal(value)
-            if audit_template:
+            if goal:
-                self._audit_template_uuid = audit_template.uuid
+                self._goal_uuid = goal.uuid
 
-    def _get_audit_template_name(self):
+    def _get_goal_name(self):
-        return self._audit_template_name
+        return self._goal_name
 
-    def _set_audit_template_name(self, value):
+    def _set_goal_name(self, value):
-        if value and self._audit_template_name != value:
+        if value and self._goal_name != value:
-            self._audit_template_name = None
+            self._goal_name = None
-            audit_template = self._get_audit_template(value)
+            goal = self._get_goal(value)
-            if audit_template:
+            if goal:
-                self._audit_template_name = audit_template.name
+                self._goal_name = goal.name
+
+    def _get_strategy(self, value):
+        if value == wtypes.Unset:
+            return None
+        strategy = None
+        try:
+            if utils.is_uuid_like(value) or utils.is_int_like(value):
+                strategy = objects.Strategy.get(
+                    pecan.request.context, value)
+            else:
+                strategy = objects.Strategy.get_by_name(
+                    pecan.request.context, value)
+        except exception.StrategyNotFound:
+            pass
+        if strategy:
+            self.strategy_id = strategy.id
+        return strategy
+
+    def _get_strategy_uuid(self):
+        return self._strategy_uuid
+
+    def _set_strategy_uuid(self, value):
+        if value and self._strategy_uuid != value:
+            self._strategy_uuid = None
+            strategy = self._get_strategy(value)
+            if strategy:
+                self._strategy_uuid = strategy.uuid
+
+    def _get_strategy_name(self):
+        return self._strategy_name
+
+    def _set_strategy_name(self, value):
+        if value and self._strategy_name != value:
+            self._strategy_name = None
+            strategy = self._get_strategy(value)
+            if strategy:
+                self._strategy_name = strategy.name
 
     uuid = types.uuid
     """Unique UUID for this audit"""
@@ -156,23 +229,24 @@ class Audit(base.APIBase):
     audit_type = wtypes.text
     """Type of this audit"""
 
-    deadline = datetime.datetime
-    """deadline of the audit"""
-
     state = wtypes.text
     """This audit state"""
 
-    audit_template_uuid = wsme.wsproperty(wtypes.text,
+    goal_uuid = wsme.wsproperty(
-                                          _get_audit_template_uuid,
+        wtypes.text, _get_goal_uuid, _set_goal_uuid, mandatory=True)
-                                          _set_audit_template_uuid,
+    """Goal UUID the audit template refers to"""
-                                          mandatory=True)
-    """The UUID of the audit template this audit refers to"""
 
-    audit_template_name = wsme.wsproperty(wtypes.text,
+    goal_name = wsme.wsproperty(
-                                          _get_audit_template_name,
+        wtypes.text, _get_goal_name, _set_goal_name, mandatory=False)
-                                          _set_audit_template_name,
+    """The name of the goal this audit template refers to"""
-                                          mandatory=False)
+
-    """The name of the audit template this audit refers to"""
+    strategy_uuid = wsme.wsproperty(
+        wtypes.text, _get_strategy_uuid, _set_strategy_uuid, mandatory=False)
+    """Strategy UUID the audit template refers to"""
+
+    strategy_name = wsme.wsproperty(
+        wtypes.text, _get_strategy_name, _set_strategy_name, mandatory=False)
+    """The name of the strategy this audit template refers to"""
 
     parameters = {wtypes.text: types.jsontype}
     """The strategy parameters for this audit"""
@@ -183,10 +257,15 @@ class Audit(base.APIBase):
     interval = wsme.wsattr(int, mandatory=False)
     """Launch audit periodically (in seconds)"""
 
+    scope = wsme.wsattr(types.jsontype, mandatory=False)
+    """Audit Scope"""
+
+    auto_trigger = wsme.wsattr(bool, mandatory=False, default=False)
+    """Autoexecute action plan once audit is succeeded"""
+
     def __init__(self, **kwargs):
         self.fields = []
         fields = list(objects.Audit.fields)
-
         for k in fields:
             # Skip fields we do not expose.
             if not hasattr(self, k):
@@ -194,27 +273,28 @@ class Audit(base.APIBase):
             self.fields.append(k)
             setattr(self, k, kwargs.get(k, wtypes.Unset))
 
-        self.fields.append('audit_template_id')
+        self.fields.append('goal_id')
-
+        self.fields.append('strategy_id')
-        # audit_template_uuid & audit_template_name are not part of
+        fields.append('goal_uuid')
-        # objects.Audit.fields because they're API-only attributes.
+        setattr(self, 'goal_uuid', kwargs.get('goal_id',
-        fields.append('audit_template_uuid')
-        setattr(self, 'audit_template_uuid', kwargs.get('audit_template_id',
                 wtypes.Unset))
-        fields.append('audit_template_name')
+        fields.append('goal_name')
-        setattr(self, 'audit_template_name', kwargs.get('audit_template_id',
+        setattr(self, 'goal_name', kwargs.get('goal_id',
+                wtypes.Unset))
+        fields.append('strategy_uuid')
+        setattr(self, 'strategy_uuid', kwargs.get('strategy_id',
+                wtypes.Unset))
+        fields.append('strategy_name')
+        setattr(self, 'strategy_name', kwargs.get('strategy_id',
                 wtypes.Unset))
 
     @staticmethod
     def _convert_with_links(audit, url, expand=True):
         if not expand:
-            audit.unset_fields_except(['uuid', 'audit_type', 'deadline',
+            audit.unset_fields_except(['uuid', 'audit_type', 'state',
-                                       'state', 'audit_template_uuid',
+                                       'goal_uuid', 'interval', 'scope',
-                                       'audit_template_name', 'interval'])
+                                       'strategy_uuid', 'goal_name',
-
+                                       'strategy_name'])
-        # The numeric ID should not be exposed to
-        # the user, it's internal only.
-        audit.audit_template_id = wtypes.Unset
 
         audit.links = [link.Link.make_link('self', url,
                                            'audits', audit.uuid),
@@ -235,12 +315,16 @@ class Audit(base.APIBase):
         sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
                      audit_type='ONESHOT',
                      state='PENDING',
-                     deadline=None,
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
                      updated_at=datetime.datetime.utcnow(),
-                     interval=7200)
+                     interval=7200,
-        sample._audit_template_uuid = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
+                     scope=[],
+                     auto_trigger=False)
+
+        sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
+        sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff'
+
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -263,12 +347,12 @@ class AuditCollection(collection.Collection):
 
         if 'sort_key' in kwargs:
             reverse = False
-            if kwargs['sort_key'] == 'audit_template_uuid':
+            if kwargs['sort_key'] == 'goal_uuid':
                 if 'sort_dir' in kwargs:
                     reverse = True if kwargs['sort_dir'] == 'desc' else False
                 collection.audits = sorted(
                     collection.audits,
-                    key=lambda audit: audit.audit_template_uuid,
+                    key=lambda audit: audit.goal_uuid,
                     reverse=reverse)
 
         collection.next = collection.get_next(limit, url=url, **kwargs)
@@ -296,24 +380,34 @@ class AuditsController(rest.RestController):
 
     def _get_audits_collection(self, marker, limit,
                                sort_key, sort_dir, expand=False,
-                               resource_url=None, audit_template=None):
+                               resource_url=None, goal=None,
+                               strategy=None):
         limit = api_utils.validate_limit(limit)
         api_utils.validate_sort_dir(sort_dir)
-
         marker_obj = None
         if marker:
             marker_obj = objects.Audit.get_by_uuid(pecan.request.context,
                                                    marker)
 
         filters = {}
-        if audit_template:
+        if goal:
-            if utils.is_uuid_like(audit_template):
+            if utils.is_uuid_like(goal):
-                filters['audit_template_uuid'] = audit_template
+                filters['goal_uuid'] = goal
             else:
-                filters['audit_template_name'] = audit_template
+                # TODO(michaelgugino): add method to get goal by name.
+                filters['goal_name'] = goal
 
-        if sort_key == 'audit_template_uuid':
+        if strategy:
-            sort_db_key = None
+            if utils.is_uuid_like(strategy):
+                filters['strategy_uuid'] = strategy
+            else:
+                # TODO(michaelgugino): add method to get goal by name.
+                filters['strategy_name'] = strategy
+
+        if sort_key == 'goal_uuid':
+            sort_db_key = 'goal_id'
+        elif sort_key == 'strategy_uuid':
+            sort_db_key = 'strategy_id'
         else:
             sort_db_key = sort_key
 
@@ -328,33 +422,35 @@ class AuditsController(rest.RestController):
                                                   sort_key=sort_key,
                                                   sort_dir=sort_dir)
 
-    @wsme_pecan.wsexpose(AuditCollection, wtypes.text, types.uuid, int,
+    @wsme_pecan.wsexpose(AuditCollection, types.uuid, int, wtypes.text,
-                         wtypes.text, wtypes.text)
+                         wtypes.text, wtypes.text, wtypes.text, int)
-    def get_all(self, audit_template=None, marker=None, limit=None,
+    def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc',
-                sort_key='id', sort_dir='asc'):
+                goal=None, strategy=None):
         """Retrieve a list of audits.
 
-        :param audit_template: Optional UUID or name of an audit
         :param marker: pagination marker for large data sets.
         :param limit: maximum number of resources to return in a single result.
         :param sort_key: column to sort results by. Default: id.
         :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
-         template, to get only audits for that audit template.
+        :param goal: goal UUID or name to filter by
+        :param strategy: strategy UUID or name to filter by
         """
+
         context = pecan.request.context
         policy.enforce(context, 'audit:get_all',
                        action='audit:get_all')
+
         return self._get_audits_collection(marker, limit, sort_key,
-                                           sort_dir,
+                                           sort_dir, goal=goal,
-                                           audit_template=audit_template)
+                                           strategy=strategy)
 
     @wsme_pecan.wsexpose(AuditCollection, wtypes.text, types.uuid, int,
                          wtypes.text, wtypes.text)
-    def detail(self, audit_template=None, marker=None, limit=None,
+    def detail(self, goal=None, marker=None, limit=None,
                sort_key='id', sort_dir='asc'):
         """Retrieve a list of audits with detail.
 
-        :param audit_template: Optional UUID or name of an audit
+        :param goal: goal UUID or name to filter by
         :param marker: pagination marker for large data sets.
         :param limit: maximum number of resources to return in a single result.
         :param sort_key: column to sort results by. Default: id.
@@ -373,7 +469,7 @@ class AuditsController(rest.RestController):
         return self._get_audits_collection(marker, limit,
                                            sort_key, sort_dir, expand,
                                            resource_url,
-                                           audit_template=audit_template)
+                                           goal=goal)
 
     @wsme_pecan.wsexpose(Audit, types.uuid)
     def get_one(self, audit_uuid):
@@ -399,28 +495,27 @@ class AuditsController(rest.RestController):
         context = pecan.request.context
         policy.enforce(context, 'audit:create',
                        action='audit:create')
+        audit = audit_p.as_audit(context)
 
-        audit = audit_p.as_audit()
         if self.from_audits:
             raise exception.OperationNotPermitted
 
-        if not audit._audit_template_uuid:
+        if not audit._goal_uuid:
             raise exception.Invalid(
-                message=_('The audit template UUID or name specified is '
+                message=_('A valid goal_id or audit_template_id '
-                          'invalid'))
+                          'must be provided'))
 
-        audit_template = objects.AuditTemplate.get(pecan.request.context,
+        strategy_uuid = audit.strategy_uuid
-                                                   audit._audit_template_uuid)
-        strategy_id = audit_template.strategy_id
         no_schema = True
-        if strategy_id is not None:
+        if strategy_uuid is not None:
             # validate parameter when predefined strategy in audit template
-            strategy = objects.Strategy.get(pecan.request.context, strategy_id)
+            strategy = objects.Strategy.get(pecan.request.context,
+                                            strategy_uuid)
             schema = strategy.parameters_spec
             if schema:
                 # validate input parameter with default value feedback
                 no_schema = False
-                utils.DefaultValidatingDraft4Validator(schema).validate(
+                utils.StrictDefaultValidatingDraft4Validator(schema).validate(
                     audit.parameters)
 
         if no_schema and audit.parameters:
@@ -429,15 +524,14 @@ class AuditsController(rest.RestController):
                                       'parameter spec in predefined strategy'))
 
         audit_dict = audit.as_dict()
-        context = pecan.request.context
+
         new_audit = objects.Audit(context, **audit_dict)
-        new_audit.create(context)
+        new_audit.create()
 
         # Set the HTTP Location Header
         pecan.response.location = link.build_url('audits', new_audit.uuid)
 
         # trigger decision-engine to run the audit
-
         if new_audit.audit_type == objects.audit.AuditType.ONESHOT.value:
             dc_client = rpcapi.DecisionEngineAPI()
             dc_client.trigger_audit(context, new_audit.uuid)
@@ -456,13 +550,11 @@ class AuditsController(rest.RestController):
             raise exception.OperationNotPermitted
 
         context = pecan.request.context
-        audit_to_update = api_utils.get_resource('Audit',
+        audit_to_update = api_utils.get_resource(
-                                                 audit_uuid)
+            'Audit', audit_uuid, eager=True)
         policy.enforce(context, 'audit:update', audit_to_update,
                        action='audit:update')
 
-        audit_to_update = objects.Audit.get_by_uuid(pecan.request.context,
-                                                    audit_uuid)
         try:
             audit_dict = audit_to_update.as_dict()
             audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch))
@@ -491,7 +583,8 @@ class AuditsController(rest.RestController):
         :param audit_uuid: UUID of a audit.
         """
         context = pecan.request.context
-        audit_to_delete = api_utils.get_resource('Audit', audit_uuid)
+        audit_to_delete = api_utils.get_resource(
+            'Audit', audit_uuid, eager=True)
         policy.enforce(context, 'audit:update', audit_to_delete,
                        action='audit:update')
 

watcher/api/controllers/v1/audit_template.py

@@ -41,11 +41,6 @@ settings related to the level of automation for the
 A flag will indicate whether the :ref:`Action Plan <action_plan_definition>`
 will be launched automatically or will need a manual confirmation from the
 :ref:`Administrator <administrator_definition>`.
-
-Last but not least, an :ref:`Audit Template <audit_template_definition>` may
-contain a list of extra parameters related to the
-:ref:`Strategy <strategy_definition>` configuration. These parameters can be
-provided as a list of key-value pairs.
 """
 
 import datetime
@@ -66,6 +61,7 @@ from watcher.common import context as context_utils
 from watcher.common import exception
 from watcher.common import policy
 from watcher.common import utils as common_utils
+from watcher.decision_engine.scope import default
 from watcher import objects
 
 
@@ -78,37 +74,24 @@ class AuditTemplatePostType(wtypes.Base):
     description = wtypes.wsattr(wtypes.text, mandatory=False)
     """Short description of this audit template"""
 
-    deadline = wsme.wsattr(datetime.datetime, mandatory=False)
-    """deadline of the audit template"""
-
-    host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1),
-                                 mandatory=False)
-    """ID of the Nova host aggregate targeted by the audit template"""
-
-    extra = wtypes.wsattr({wtypes.text: types.jsontype}, mandatory=False)
-    """The metadata of the audit template"""
-
     goal = wtypes.wsattr(wtypes.text, mandatory=True)
     """Goal UUID or name of the audit template"""
 
     strategy = wtypes.wsattr(wtypes.text, mandatory=False)
     """Strategy UUID or name of the audit template"""
 
-    version = wtypes.text
+    scope = wtypes.wsattr(types.jsontype, mandatory=False, default=[])
-    """Internal version of the audit template"""
+    """Audit Scope"""
 
     def as_audit_template(self):
         return AuditTemplate(
             name=self.name,
             description=self.description,
-            deadline=self.deadline,
-            host_aggregate=self.host_aggregate,
-            extra=self.extra,
             goal_id=self.goal,  # Dirty trick ...
             goal=self.goal,
             strategy_id=self.strategy,  # Dirty trick ...
             strategy_uuid=self.strategy,
-            version=self.version,
+            scope=self.scope,
         )
 
     @staticmethod
@@ -123,6 +106,9 @@ class AuditTemplatePostType(wtypes.Base):
         else:
             raise exception.InvalidGoal(goal=audit_template.goal)
 
+        common_utils.Draft4Validator(
+            default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope)
+
         if audit_template.strategy:
             available_strategies = objects.Strategy.list(
                 AuditTemplatePostType._ctx)
@@ -305,18 +291,9 @@ class AuditTemplate(base.APIBase):
     name = wtypes.text
     """Name of this audit template"""
 
-    description = wtypes.text
+    description = wtypes.wsattr(wtypes.text, mandatory=False)
     """Short description of this audit template"""
 
-    deadline = datetime.datetime
-    """deadline of the audit template"""
-
-    host_aggregate = wtypes.IntegerType(minimum=1)
-    """ID of the Nova host aggregate targeted by the audit template"""
-
-    extra = {wtypes.text: types.jsontype}
-    """The metadata of the audit template"""
-
     goal_uuid = wsme.wsproperty(
         wtypes.text, _get_goal_uuid, _set_goal_uuid, mandatory=True)
     """Goal UUID the audit template refers to"""
@@ -333,15 +310,15 @@ class AuditTemplate(base.APIBase):
         wtypes.text, _get_strategy_name, _set_strategy_name, mandatory=False)
     """The name of the strategy this audit template refers to"""
 
-    version = wtypes.text
-    """Internal version of the audit template"""
-
     audits = wsme.wsattr([link.Link], readonly=True)
     """Links to the collection of audits contained in this audit template"""
 
     links = wsme.wsattr([link.Link], readonly=True)
     """A list containing a self link and associated audit template links"""
 
+    scope = wsme.wsattr(types.jsontype, mandatory=False)
+    """Audit Scope"""
+
     def __init__(self, **kwargs):
         super(AuditTemplate, self).__init__()
         self.fields = []
@@ -374,8 +351,8 @@ class AuditTemplate(base.APIBase):
     def _convert_with_links(audit_template, url, expand=True):
         if not expand:
             audit_template.unset_fields_except(
-                ['uuid', 'name', 'host_aggregate', 'goal_uuid', 'goal_name',
+                ['uuid', 'name', 'goal_uuid', 'goal_name',
-                 'strategy_uuid', 'strategy_name'])
+                 'scope', 'strategy_uuid', 'strategy_name'])
 
         # The numeric ID should not be exposed to
         # the user, it's internal only.
@@ -402,13 +379,12 @@ class AuditTemplate(base.APIBase):
         sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
                      name='My Audit Template',
                      description='Description of my audit template',
-                     host_aggregate=5,
                      goal_uuid='83e44733-b640-40e2-8d8a-7dd3be7134e6',
                      strategy_uuid='367d826e-b6a4-4b70-bc44-c3f6fe1c9986',
-                     extra={'automatic': True},
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
-                     updated_at=datetime.datetime.utcnow())
+                     updated_at=datetime.datetime.utcnow(),
+                     scope=[],)
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -592,11 +568,11 @@ class AuditTemplatesController(rest.RestController):
         audit_template_dict = audit_template.as_dict()
         new_audit_template = objects.AuditTemplate(context,
                                                    **audit_template_dict)
-        new_audit_template.create(context)
+        new_audit_template.create()
 
         # Set the HTTP Location Header
-        pecan.response.location = link.build_url('audit_templates',
+        pecan.response.location = link.build_url(
-                                                 new_audit_template.uuid)
+            'audit_templates', new_audit_template.uuid)
         return AuditTemplate.convert_with_links(new_audit_template)
 
     @wsme.validate(types.uuid, [AuditTemplatePatchType])

watcher/api/controllers/v1/collection.py

@@ -35,7 +35,7 @@ class Collection(base.APIBase):
         """Return whether collection has more items."""
         return len(self.collection) and len(self.collection) == limit
 
-    def get_next(self, limit, url=None, **kwargs):
+    def get_next(self, limit, url=None, marker_field="uuid", **kwargs):
         """Return a link to the next subset of the collection."""
         if not self.has_next(limit):
             return wtypes.Unset
@@ -44,7 +44,7 @@ class Collection(base.APIBase):
         q_args = ''.join(['%s=%s&' % (key, kwargs[key]) for key in kwargs])
         next_args = '?%(args)slimit=%(limit)d&marker=%(marker)s' % {
             'args': q_args, 'limit': limit,
-            'marker': getattr(self.collection[-1], "uuid")}
+            'marker': getattr(self.collection[-1], marker_field)}
 
         return link.Link.make_link('next', pecan.request.host_url,
                                    resource_url, next_args).href