Implemented clients and auth config module

Implemented clients and auth config module

Implements: blueprint centralise-config-opts

Change-Id: I28ea8376aa34114331cbae61596839ebae6cf7eb

MANIFEST.in

@@ -1,6 +0,0 @@
-include AUTHORS
-include ChangeLog
-exclude .gitignore
-exclude .gitreview
-
-global-exclude *.pyc

README.rst

@@ -1,3 +1,12 @@
+========================
+Team and repository tags
+========================
+
+.. image:: http://governance.openstack.org/badges/watcher.svg
+    :target: http://governance.openstack.org/reference/tags/index.html
+
+.. Change things from this point on
+
 ..
       Except where otherwise noted, this document is licensed under Creative
       Commons Attribution 3.0 License.  You can view the license at:
@@ -10,10 +19,8 @@ Watcher
 
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
-Watcher provides a complete optimization loop-including everything from a
-metrics receiver, complex event processor and profiler, optimization processor
-and an action plan applier. This provides a robust framework to realize a wide
-range of cloud optimization goals, including the reduction of data center
+Watcher provides a robust framework to realize a wide range of cloud
+optimization goals, including the reduction of data center
 operating costs, increased system performance via intelligent virtual machine
 migration, increased energy efficiency-and more!
 

devstack/lib/watcher

@@ -126,6 +126,8 @@ function create_watcher_conf {
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
 
+    iniset $WATCHER_CONF oslo_messaging_notifications driver "messaging"
+
     iniset $NOVA_CONF oslo_messaging_notifications topics "notifications,watcher_notifications"
 
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR

devstack/local.conf.controller

@@ -28,7 +28,7 @@ ENABLED_SERVICES+=,q-svc,q-dhcp,q-meta,q-agt,q-l3,neutron
 enable_service n-cauth
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+# enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
 enable_plugin watcher git://git.openstack.org/openstack/watcher

doc/ext/term.py

@@ -75,7 +75,7 @@ class WatcherTerm(BaseWatcherDirective):
     # Inside your .rst file
     .. watcher-term:: import.path.to.your.DocumentedObject
 
-    This directive will then import the docstring and then interprete it.
+    This directive will then import the docstring and then interpret it.
     """
 
     # You need to put an import path as an argument for this directive to work
@@ -85,7 +85,12 @@ class WatcherTerm(BaseWatcherDirective):
         cls_path = self.arguments[0]
 
         try:
-            cls = importlib.import_module(cls_path)
+            try:
+                cls = importlib.import_module(cls_path)
+            except ImportError:
+                module_name, cls_name = cls_path.rsplit('.', 1)
+                mod = importlib.import_module(module_name)
+                cls = getattr(mod, cls_name)
         except Exception as exc:
             raise self.error(exc)
 
@@ -97,6 +102,72 @@ class WatcherTerm(BaseWatcherDirective):
         return node.children
 
 
+class WatcherFunc(BaseWatcherDirective):
+    """Directive to import a value returned by a func into the Watcher doc
+
+    **How to use it**
+
+    # inside your .py file
+    class Bar(object):
+
+        def foo(object):
+            return foo_string
+
+
+    # Inside your .rst file
+    .. watcher-func:: import.path.to.your.Bar.foo node_classname
+
+    node_classname is decumented here:
+    http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html
+
+    This directive will then import the value and then interpret it.
+    """
+
+    # You need to put an import path as an argument for this directive to work
+    # required_arguments = 1
+    # optional_arguments = 1
+
+    option_spec = {'format': rst.directives.unchanged}
+    has_content = True
+
+    def run(self):
+        if not self.content:
+            error = self.state_machine.reporter.error(
+                'The "%s" directive is empty; content required.' % self.name,
+                nodes.literal_block(self.block_text, self.block_text),
+                line=self.lineno)
+            return [error]
+
+        func_path = self.content[0]
+        try:
+            cls_path, func_name = func_path.rsplit('.', 1)
+            module_name, cls_name = cls_path.rsplit('.', 1)
+            mod = importlib.import_module(module_name)
+            cls = getattr(mod, cls_name)
+        except Exception as exc:
+            raise self.error(exc)
+
+        cls_obj = cls()
+        func = getattr(cls_obj, func_name)
+        textblock = func()
+        if not isinstance(textblock, str):
+            textblock = str(textblock)
+
+        self.add_textblock(textblock)
+
+        try:
+            node_class = getattr(nodes,
+                                 self.options.get('format', 'paragraph'))
+        except Exception as exc:
+            raise self.error(exc)
+
+        node = node_class()
+        node.document = self.state.document
+        self.state.nested_parse(self.result, 0, node)
+        return [node]
+
+
 def setup(app):
     app.add_directive('watcher-term', WatcherTerm)
+    app.add_directive('watcher-func', WatcherFunc)
     return {'version': version_info.version_string()}

doc/ext/versioned_notifications.py

@@ -0,0 +1,133 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+This provides a sphinx extension able to list the implemented versioned
+notifications into the developer documentation.
+
+It is used via a single directive in the .rst file
+
+  .. versioned_notifications::
+
+"""
+
+from sphinx.util.compat import Directive
+from docutils import nodes
+
+from watcher.notifications import base as notification
+from watcher.objects import base
+
+
+class VersionedNotificationDirective(Directive):
+
+    SAMPLE_ROOT = 'doc/notification_samples/'
+    TOGGLE_SCRIPT = """
+<script>
+jQuery(document).ready(function(){
+    jQuery('#%s-div').toggle('show');
+    jQuery('#%s-hideshow').on('click', function(event) {
+        jQuery('#%s-div').toggle('show');
+    });
+});
+</script>
+"""
+
+    def run(self):
+        notifications = self._collect_notifications()
+        return self._build_markup(notifications)
+
+    def _collect_notifications(self):
+        base.WatcherObjectRegistry.register_notification_objects()
+        notifications = []
+        ovos = base.WatcherObjectRegistry.obj_classes()
+        for name, cls in ovos.items():
+            cls = cls[0]
+            if (issubclass(cls, notification.NotificationBase) and
+                    cls != notification.NotificationBase):
+
+                payload_name = cls.fields['payload'].objname
+                payload_cls = ovos[payload_name][0]
+                for sample in cls.samples:
+                    notifications.append((cls.__name__,
+                                          payload_cls.__name__,
+                                          sample))
+        return sorted(notifications)
+
+    def _build_markup(self, notifications):
+        content = []
+        cols = ['Event type', 'Notification class', 'Payload class', 'Sample']
+        table = nodes.table()
+        content.append(table)
+        group = nodes.tgroup(cols=len(cols))
+        table.append(group)
+
+        head = nodes.thead()
+        group.append(head)
+
+        for _ in cols:
+            group.append(nodes.colspec(colwidth=1))
+
+        body = nodes.tbody()
+        group.append(body)
+
+        # fill the table header
+        row = nodes.row()
+        body.append(row)
+        for col_name in cols:
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.strong(text=col_name)
+            col.append(text)
+
+        # fill the table content, one notification per row
+        for name, payload, sample_file in notifications:
+            event_type = sample_file[0: -5].replace('-', '.')
+
+            row = nodes.row()
+            body.append(row)
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=event_type)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=name)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=payload)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+
+            with open(self.SAMPLE_ROOT + sample_file, 'r') as f:
+                sample_content = f.read()
+
+            event_type = sample_file[0: -5]
+            html_str = self.TOGGLE_SCRIPT % ((event_type, ) * 3)
+            html_str += ("<input type='button' id='%s-hideshow' "
+                         "value='hide/show sample'>" % event_type)
+            html_str += ("<div id='%s-div'><pre>%s</pre></div>"
+                         % (event_type, sample_content))
+
+            raw = nodes.raw('', html_str, format="html")
+            col.append(raw)
+
+        return content
+
+
+def setup(app):
+    app.add_directive('versioned_notifications',
+                      VersionedNotificationDirective)

doc/notification_samples/audit-create.json

@@ -0,0 +1,69 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "PENDING",
+      "updated_at": null,
+      "deleted_at": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditCreatePayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.create",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-delete.json

@@ -0,0 +1,69 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "DELETED",
+      "updated_at": null,
+      "deleted_at": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditDeletePayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.delete",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-end.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.end",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-error.json

@@ -0,0 +1,80 @@
+{
+  "priority": "ERROR",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": {
+        "watcher_object.data": {
+          "exception": "WatcherException",
+          "exception_message": "TEST",
+          "function_name": "test_send_audit_action_with_error",
+          "module_name": "watcher.tests.notifications.test_audit_notification"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.error",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-start.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.start",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-end.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.end",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-error.json

@@ -0,0 +1,80 @@
+{
+  "priority": "ERROR",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": {
+        "watcher_object.data": {
+          "exception": "WatcherException",
+          "exception_message": "TEST",
+          "function_name": "test_send_audit_action_with_error",
+          "module_name": "watcher.tests.notifications.test_audit_notification"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.error",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-start.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.start",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-update.json

@@ -0,0 +1,78 @@
+{
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:51:38.722986   ",
+  "payload": {
+    "watcher_object.name": "AuditUpdatePayload",
+    "watcher_object.data": {
+      "strategy": {
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.data": {
+          "name": "dummy",
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "default": "hello",
+                "type": "string",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "maximum": 10.2,
+                "default": 3.2,
+                "minimum": 1.0,
+                "description": "number parameter example",
+                "type": "number"
+              }
+            }
+          },
+          "updated_at": null,
+          "display_name": "Dummy strategy",
+          "deleted_at": null,
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "created_at": "2016-11-04T16:25:35Z"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "scope": [],
+      "created_at": "2016-11-04T16:51:21Z",
+      "uuid": "f1e0d912-afd9-4bf2-91ef-c99cd08cc1ef",
+      "goal": {
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.data": {
+          "efficacy_specification": [],
+          "updated_at": null,
+          "name": "dummy",
+          "display_name": "Dummy goal",
+          "deleted_at": null,
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "created_at": "2016-11-04T16:25:35Z"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "deleted_at": null,
+      "state_update": {
+        "watcher_object.name": "AuditStateUpdatePayload",
+        "watcher_object.data": {
+          "state": "ONGOING",
+          "old_state": "PENDING"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "interval": null,
+      "updated_at": null,
+      "state": "ONGOING",
+      "audit_type": "ONESHOT"
+    },
+    "watcher_object.namespace": "watcher",
+    "watcher_object.version": "1.0"
+  },
+  "priority": "INFO",
+  "event_type": "audit.update",
+  "message_id": "697fdf55-7252-4b6c-a2c2-5b9e85f6342c"
+}

doc/notification_samples/infra-optim-exception.json

@@ -0,0 +1,16 @@
+{
+    "event_type": "infra-optim.exception",
+    "payload": {
+        "watcher_object.data": {
+            "exception": "NoAvailableStrategyForGoal",
+            "exception_message": "No strategy could be found to achieve the server_consolidation goal.",
+            "function_name": "_aggregate_create_in_db",
+            "module_name": "watcher.objects.aggregate"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+    },
+    "priority": "ERROR",
+    "publisher_id": "watcher-api:fake-mini"
+}

doc/source/architecture.rst

@@ -171,12 +171,12 @@ This component is responsible for computing a set of potential optimization
 :ref:`Actions <action_definition>` in order to fulfill
 the :ref:`Goal <goal_definition>` of an :ref:`Audit <audit_definition>`.
 
-It first reads the parameters of the :ref:`Audit <audit_definition>` from the
-associated :ref:`Audit Template <audit_template_definition>` and knows the
-:ref:`Goal <goal_definition>` to achieve.
+It first reads the parameters of the :ref:`Audit <audit_definition>` to know
+the :ref:`Goal <goal_definition>` to achieve.
 
-It then selects the most appropriate :ref:`Strategy <strategy_definition>`
-from the list of available strategies achieving this goal.
+Unless specified, it then selects the most appropriate :ref:`strategy
+<strategy_definition>` from the list of available strategies achieving this
+goal.
 
 The :ref:`Strategy <strategy_definition>` is then dynamically loaded (via
 `stevedore <http://docs.openstack.org/developer/stevedore/>`_). The
@@ -290,7 +290,7 @@ the Audit parameters from the
 :ref:`Watcher Database <watcher_database_definition>`. It instantiates the
 appropriate :ref:`strategy <strategy_definition>` (using entry points)
 given both the :ref:`goal <goal_definition>` and the strategy associated to the
-parent :ref:`audit template <audit_template_definition>` of the :ref:`Audit
+parent :ref:`audit template <audit_template_definition>` of the :ref:`audit
 <audit_definition>`. If no strategy is associated to the audit template, the
 strategy is dynamically selected by the Decision Engine.
 
@@ -298,7 +298,7 @@ The :ref:`Watcher Decision Engine <watcher_decision_engine_definition>` also
 builds the :ref:`Cluster Data Model <cluster_data_model_definition>`. This
 data model is needed by the :ref:`Strategy <strategy_definition>` to know the
 current state and topology of the audited
-:ref:`Openstack cluster <cluster_definition>`.
+:ref:`OpenStack cluster <cluster_definition>`.
 
 The :ref:`Watcher Decision Engine <watcher_decision_engine_definition>` calls
 the **execute()** method of the instantiated
@@ -316,7 +316,7 @@ This method finds an appropriate scheduling of
 :ref:`Actions <action_definition>` taking into account some scheduling rules
 (such as priorities between actions).
 It generates a new :ref:`Action Plan <action_plan_definition>` with status
-**RECOMMENDED** and saves it into the:ref:`Watcher Database
+**RECOMMENDED** and saves it into the :ref:`Watcher Database
 <watcher_database_definition>`. The saved action plan is now a scheduled flow
 of actions to which a global efficacy is associated alongside a number of
 :ref:`Efficacy Indicators <efficacy_indicator_definition>` as specified by the
@@ -450,7 +450,7 @@ state may be one of the following:
    stored in the :ref:`Watcher database <watcher_database_definition>` but is
    not returned any more through the Watcher APIs.
 -  **CANCELLED** : the :ref:`Action Plan <action_plan_definition>` was in
-   **PENDING** or **ONGOING** state and was cancelled by the
+   **RECOMMENDED**, **PENDING** or **ONGOING** state and was cancelled by the
    :ref:`Administrator <administrator_definition>`
 
 

doc/source/conf.py

@@ -11,7 +11,21 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+import sys
+
 from watcher import version as watcher_version
+from watcher import objects
+
+objects.register_all()
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../'))
+sys.path.insert(0, os.path.abspath('./'))
 
 # -- General configuration ----------------------------------------------------
 
@@ -26,7 +40,8 @@ extensions = [
     'sphinxcontrib.pecanwsme.rest',
     'stevedore.sphinxext',
     'wsmeext.sphinxext',
-    'watcher.doc',
+    'ext.term',
+    'ext.versioned_notifications',
 ]
 
 wsme_protocols = ['restjson']
@@ -67,6 +82,8 @@ exclude_patterns = [
     # them when scanning for input files.
     'man/footer.rst',
     'man/general-options.rst',
+    'strategies/strategy-template.rst',
+    'image_src/plantuml/README.rst',
 ]
 
 # If true, '()' will be appended to :func: etc. cross-reference text.

doc/source/deploy/user-guide.rst

@@ -93,11 +93,11 @@ following command:
 
 .. code:: bash
 
-  $ watcher strategy list --goal-uuid <your_goal_uuid>
+  $ watcher strategy list --goal <your_goal_uuid_or_name>
 
 or::
 
-  $ openstack optimize strategy list --goal-uuid <your_goal_uuid>
+  $ openstack optimize strategy list --goal <your_goal_uuid_or_name>
 
 You can use the following command to check strategy details including which
 parameters of which format it supports:

doc/source/dev/notifications.rst

@@ -0,0 +1,13 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+.. _watcher_notifications:
+
+========================
+Notifications in Watcher
+========================
+
+.. versioned_notifications::

doc/source/dev/plugin/cdmc-plugin.rst

@@ -22,7 +22,7 @@ cluster data model collectors within Watcher.
 Creating a new plugin
 =====================
 
-In order to create a new model, you have to:
+In order to create a new cluster data model collector, you have to:
 
 - Extend the :py:class:`~.base.BaseClusterDataModelCollector` class.
 - Implement its :py:meth:`~.BaseClusterDataModelCollector.execute` abstract
@@ -65,6 +65,49 @@ This implementation is the most basic one. So in order to get a better
 understanding on how to implement a more advanced cluster data model collector,
 have a look at the :py:class:`~.NovaClusterDataModelCollector` class.
 
+Define a custom model
+=====================
+
+As you may have noticed in the above example, we are reusing an existing model
+provided by Watcher. However, this model can be easily customized by
+implementing a new class that would implement the :py:class:`~.Model` abstract
+base class. Here below is simple example on how to proceed in implementing a
+custom Model:
+
+.. code-block:: python
+
+    # Filepath = <PROJECT_DIR>/thirdparty/dummy.py
+    # Import path = thirdparty.dummy
+
+    from watcher.decision_engine.model import base as modelbase
+    from watcher.decision_engine.model.collector import base
+
+
+    class MyModel(modelbase.Model):
+
+        def to_string(self):
+            return 'MyModel'
+
+
+    class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
+
+        def execute(self):
+            model = MyModel()
+            # Do something here...
+            return model
+
+        @property
+        def notification_endpoints(self):
+            return []
+
+Here below is the abstract ``Model`` class that every single cluster data model
+should implement:
+
+.. autoclass:: watcher.decision_engine.model.base.Model
+    :members:
+    :special-members: __init__
+    :noindex:
+
 Define configuration parameters
 ===============================
 

doc/source/dev/plugin/goal-plugin.rst

@@ -60,8 +60,8 @@ Here is an example showing how you can define a new ``NewGoal`` goal plugin:
     # import path: thirdparty.new
 
     from watcher._i18n import _
+    from watcher.decision_engine.goal import base
     from watcher.decision_engine.goal.efficacy import specs
-    from watcher.decision_engine.strategy.strategies import base
 
     class NewGoal(base.Goal):
 
@@ -79,11 +79,11 @@ Here is an example showing how you can define a new ``NewGoal`` goal plugin:
 
         @classmethod
         def get_efficacy_specification(cls):
-            return specs.UnclassifiedStrategySpecification()
+            return specs.Unclassified()
 
 
 As you may have noticed, the :py:meth:`~.Goal.get_efficacy_specification`
-method returns an :py:meth:`~.UnclassifiedStrategySpecification` instance which
+method returns an :py:meth:`~.Unclassified` instance which
 is provided by Watcher. This efficacy specification is useful during the
 development process of your goal as it corresponds to an empty specification.
 If you want to learn more about what efficacy specifications are used for or to

doc/source/dev/plugin/planner-plugin.rst

@@ -39,7 +39,7 @@ Here is an example showing how you can write a planner plugin called
 
     # Filepath = third-party/third_party/dummy.py
     # Import path = third_party.dummy
-    import uuid
+    from oslo_utils import uuidutils
     from watcher.decision_engine.planner import base
 
 
@@ -47,7 +47,7 @@ Here is an example showing how you can write a planner plugin called
 
         def _create_action_plan(self, context, audit_id):
             action_plan_dict = {
-                'uuid': uuid.uuid4(),
+                'uuid': uuidutils.generate_uuid(),
                 'audit_id': audit_id,
                 'first_action_id': None,
                 'state': objects.action_plan.State.RECOMMENDED

doc/source/dev/plugin/scoring-engine-plugin.rst

@@ -0,0 +1,210 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+.. _implement_scoring_engine_plugin:
+
+==========================
+Build a new scoring engine
+==========================
+
+Watcher Decision Engine has an external :ref:`scoring engine
+<scoring_engine_definition>` plugin interface which gives anyone the ability
+to integrate an external scoring engine in order to make use of it in a
+:ref:`strategy <strategy_definition>`.
+
+This section gives some guidelines on how to implement and integrate custom
+scoring engines with Watcher. If you wish to create a third-party package for
+your plugin, you can refer to our :ref:`documentation for third-party package
+creation <plugin-base_setup>`.
+
+
+Pre-requisites
+==============
+
+Because scoring engines execute a purely mathematical tasks, they typically do
+not have any additional dependencies. Additional requirements might be defined
+by specific scoring engine implementations. For example, some scoring engines
+might require to prepare learning data, which has to be loaded during the
+scoring engine startup. Some other might require some external services to be
+available (e.g. if the scoring infrastructure is running in the cloud).
+
+
+Create a new scoring engine plugin
+==================================
+
+In order to create a new scoring engine you have to:
+
+- Extend the :py:class:`~.ScoringEngine` class
+- Implement its :py:meth:`~.ScoringEngine.get_name` method to return the
+  **unique** ID of the new scoring engine you want to create. This unique ID
+  should be the same as the name of :ref:`the entry point we will declare later
+  on <scoring_engine_plugin_add_entrypoint>`.
+- Implement its :py:meth:`~.ScoringEngine.get_description` method to return the
+  user-friendly description of the implemented scoring engine. It might contain
+  information about algorithm used, learning data etc.
+- Implement its :py:meth:`~.ScoringEngine.get_metainfo` method to return the
+  machine-friendly metadata about this scoring engine. For example, it could be
+  a JSON formatted text with information about the data model used, its input
+  and output data format, column names, etc.
+- Implement its :py:meth:`~.ScoringEngine.calculate_score` method to return the
+  result calculated by this scoring engine.
+
+Here is an example showing how you can write a plugin called ``NewScorer``:
+
+.. code-block:: python
+
+    # filepath: thirdparty/new.py
+    # import path: thirdparty.new
+    from watcher.decision_engine.scoring import base
+
+
+    class NewScorer(base.ScoringEngine):
+
+        def get_name(self):
+            return 'new_scorer'
+
+        def get_description(self):
+            return ''
+
+        def get_metainfo(self):
+            return """{
+                "feature_columns": [
+                    "column1",
+                    "column2",
+                    "column3"],
+                "result_columns": [
+                    "value",
+                    "probability"]
+                }"""
+
+        def calculate_score(self, features):
+            return '[12, 0.83]'
+
+As you can see in the above example, the
+:py:meth:`~.ScoringEngine.calculate_score` method returns a string. Both this
+class and the client (caller) should perform all the necessary serialization
+or deserialization.
+
+
+(Optional) Create a new scoring engine container plugin
+=======================================================
+
+Optionally, it's possible to implement a container plugin, which can return a
+list of scoring engines. This list can be re-evaluated multiple times during
+the lifecycle of :ref:`Watcher Decision Engine
+<watcher_decision_engine_definition>` and synchronized with :ref:`Watcher
+Database <watcher_database_definition>` using the ``watcher-sync`` command line
+tool.
+
+Below is an example of a container using some scoring engine implementation
+that is simply made of a client responsible for communicating with a real
+scoring engine deployed as a web service on external servers:
+
+.. code-block:: python
+
+    class NewScoringContainer(base.ScoringEngineContainer):
+
+        @classmethod
+        def get_scoring_engine_list(self):
+            return [
+                RemoteScoringEngine(
+                    name='scoring_engine1',
+                    description='Some remote Scoring Engine 1',
+                    remote_url='http://engine1.example.com/score'),
+                RemoteScoringEngine(
+                    name='scoring_engine2',
+                    description='Some remote Scoring Engine 2',
+                    remote_url='http://engine2.example.com/score'),
+            ]
+
+
+Abstract Plugin Class
+=====================
+
+Here below is the abstract :py:class:`~.ScoringEngine` class:
+
+.. autoclass:: watcher.decision_engine.scoring.base.ScoringEngine
+    :members:
+    :special-members: __init__
+    :noindex:
+
+
+Abstract Plugin Container Class
+===============================
+
+Here below is the abstract :py:class:`~.ScoringContainer` class:
+
+.. autoclass:: watcher.decision_engine.scoring.base.ScoringEngineContainer
+    :members:
+    :special-members: __init__
+    :noindex:
+
+
+.. _scoring_engine_plugin_add_entrypoint:
+
+Add a new entry point
+=====================
+
+In order for the Watcher Decision Engine to load your new scoring engine, it
+must be registered as a named entry point under the ``watcher_scoring_engines``
+entry point of your ``setup.py`` file. If you are using pbr_, this entry point
+should be placed in your ``setup.cfg`` file.
+
+The name you give to your entry point has to be unique and should be the same
+as the value returned by the :py:meth:`~.ScoringEngine.get_name` method of your
+strategy.
+
+Here below is how you would proceed to register ``NewScorer`` using pbr_:
+
+.. code-block:: ini
+
+    [entry_points]
+    watcher_scoring_engines =
+        new_scorer = thirdparty.new:NewScorer
+
+
+To get a better understanding on how to implement a more advanced scoring
+engine, have a look at the :py:class:`~.DummyScorer` class. This implementation
+is not really using machine learning, but other than that it contains all the
+pieces which the "real" implementation would have.
+
+In addition, for some use cases there is a need to register a list (possibly
+dynamic, depending on the implementation and configuration) of scoring engines
+in a single plugin, so there is no need to restart :ref:`Watcher Decision
+Engine <watcher_decision_engine_definition>` every time such list changes. For
+these cases, an additional ``watcher_scoring_engine_containers`` entry point
+can be used.
+
+For the example how to use scoring engine containers, please have a look at
+the :py:class:`~.DummyScoringContainer` and the way it is configured in
+``setup.cfg``. For new containers it could be done like this:
+
+.. code-block:: ini
+
+    [entry_points]
+    watcher_scoring_engine_containers =
+        new_scoring_container = thirdparty.new:NewContainer
+
+.. _pbr: http://docs.openstack.org/developer/pbr/
+
+
+Using scoring engine plugins
+============================
+
+The Watcher Decision Engine service will automatically discover any installed
+plugins when it is restarted. If a Python package containing a custom plugin is
+installed within the same environment as Watcher, Watcher will automatically
+make that plugin available for use.
+
+At this point, Watcher will scan and register inside the :ref:`Watcher Database
+<watcher_database_definition>` all the scoring engines you implemented upon
+restarting the :ref:`Watcher Decision Engine
+<watcher_decision_engine_definition>`.
+
+In addition, ``watcher-sync`` tool can be used to trigger :ref:`Watcher
+Database <watcher_database_definition>` synchronization. This might be used for
+"dynamic" scoring containers, which can return different scoring engines based
+on some external configuration (if they support that).

doc/source/dev/plugin/strategy-plugin.rst

@@ -43,7 +43,7 @@ In order to create a new strategy, you have to:
   Note: Do not use a variable to return the translated string so it can be
   automatically collected by the translation tool.
 - Implement its :py:meth:`~.BaseStrategy.get_translatable_display_name`
-  class method to return the translation key (actually the english display
+  class method to return the translation key (actually the English display
   name) of your new strategy. The value return should be the same as the
   string translated in :py:meth:`~.BaseStrategy.get_display_name`.
 - Implement its :py:meth:`~.BaseStrategy.execute` method to return the

doc/source/dev/plugins.rst

@@ -21,6 +21,22 @@ Goals
 .. list-plugins:: watcher_goals
    :detailed:
 
+.. _watcher_scoring_engines:
+
+Scoring Engines
+===============
+
+.. list-plugins:: watcher_scoring_engines
+   :detailed:
+
+.. _watcher_scoring_engine_containers:
+
+Scoring Engine Containers
+=========================
+
+.. list-plugins:: watcher_scoring_engine_containers
+   :detailed:
+
 .. _watcher_strategies:
 
 Strategies

doc/source/dev/rally_link.rst

@@ -0,0 +1 @@
+.. include:: ../../../rally-jobs/README.rst

doc/source/glossary.rst

@@ -96,8 +96,8 @@ The :ref:`Cluster <cluster_definition>` may be divided in one or several
 
 .. _cluster_data_model_definition:
 
-Cluster Data Model
-==================
+Cluster Data Model (CDM)
+========================
 
 .. watcher-term:: watcher.decision_engine.model.collector.base
 
@@ -132,7 +132,7 @@ Compute node
 ============
 
 Please, read `the official OpenStack definition of a Compute Node
-<http://docs.openstack.org/openstack-ops/content/compute_nodes.html>`_.
+<http://docs.openstack.org/ops-guide/arch-compute-nodes.html>`_.
 
 .. _customer_definition:
 
@@ -164,7 +164,8 @@ Goal
 Host Aggregate
 ==============
 
-Please, read `the official OpenStack definition of a Host Aggregate <http://docs.openstack.org/developer/nova/aggregates.html>`_.
+Please, read `the official OpenStack definition of a Host Aggregate
+<http://docs.openstack.org/developer/nova/aggregates.html>`_.
 
 .. _instance_definition:
 
@@ -280,6 +281,12 @@ specific domain.
 Please, read `the official OpenStack definition of a Project
 <http://docs.openstack.org/glossary/content/glossary.html>`_.
 
+.. _scoring_engine_definition:
+
+Scoring Engine
+==============
+
+.. watcher-term::  watcher.api.controllers.v1.scoring_engine
 
 .. _sla_definition:
 

doc/source/image_src/plantuml/sequence_overview_watcher_usage.txt

@@ -4,7 +4,7 @@ actor Administrator
 
 == Create some Audit settings ==
 
-Administrator -> Watcher : create new Audit Template (i.e. Audit settings : goal, scope, deadline,...)
+Administrator -> Watcher : create new Audit Template (i.e. Audit settings : goal, scope, ...)
 Watcher -> Watcher : save Audit Template in database
 Administrator <-- Watcher : Audit Template UUID
 

doc/source/image_src/plantuml/watcher_db_schema_diagram.txt

@@ -5,11 +5,12 @@
 hide methods
 hide stereotypes
 
-table(goal) {
+table(goals) {
     primary_key(id: Integer)
     uuid : String[36]
     name : String[63]
     display_name : String[63]
+    efficacy_specification : JSONEncodedList, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -18,12 +19,13 @@ table(goal) {
 }
 
 
-table(strategy) {
+table(strategies) {
     primary_key(id: Integer)
     foreign_key(goal_id : Integer)
     uuid : String[36]
     name : String[63]
     display_name : String[63]
+    parameters_spec : JSONEncodedDict, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -32,16 +34,14 @@ table(strategy) {
 }
 
 
-table(audit_template) {
+table(audit_templates) {
     primary_key(id: Integer)
     foreign_key("goal_id : Integer")
     foreign_key("strategy_id : Integer, nullable")
     uuid : String[36]
     name : String[63], nullable
     description : String[255], nullable
-    host_aggregate : Integer, nullable
-    extra : JSONEncodedDict
-    version : String[15], nullable
+    scope : JSONEncodedList
 
     created_at : DateTime
     updated_at : DateTime
@@ -50,14 +50,16 @@ table(audit_template) {
 }
 
 
-table(audit) {
+table(audits) {
     primary_key(id: Integer)
-    foreign_key("audit_template_id : Integer")
+    foreign_key("goal_id : Integer")
+    foreign_key("strategy_id : Integer, nullable")
     uuid : String[36]
     audit_type : String[20]
     state : String[20], nullable
-    deadline  :DateTime, nullable
     interval : Integer, nullable
+    parameters : JSONEncodedDict, nullable
+    scope : JSONEncodedList, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -66,9 +68,10 @@ table(audit) {
 }
 
 
-table(action_plan) {
+table(action_plans) {
     primary_key(id: Integer)
     foreign_key("audit_id : Integer, nullable")
+    foreign_key("strategy_id : Integer")
     uuid : String[36]
     first_action_id : Integer
     state : String[20], nullable
@@ -81,7 +84,7 @@ table(action_plan) {
 }
 
 
-table(action) {
+table(actions) {
     primary_key(id: Integer)
     foreign_key("action_plan_id : Integer")
     uuid : String[36]
@@ -97,7 +100,7 @@ table(action) {
 }
 
 
-table(efficacy_indicator) {
+table(efficacy_indicators) {
     primary_key(id: Integer)
     foreign_key("action_plan_id : Integer")
     uuid : String[36]
@@ -112,12 +115,39 @@ table(efficacy_indicator) {
     deleted : Integer
 }
 
- "goal" <.. "strategy" : Foreign Key
- "goal" <.. "audit_template" : Foreign Key
- "strategy" <.. "audit_template" : Foreign Key
- "audit_template" <.. "audit" : Foreign Key
- "action_plan" <.. "action" : Foreign Key
- "action_plan" <.. "efficacy_indicator" : Foreign Key
- "audit" <.. "action_plan" : Foreign Key
+table(scoring_engines) {
+    primary_key(id: Integer)
+    uuid : String[36]
+    name : String[63]
+    description : String[255], nullable
+    metainfo : Text, nullable
+
+    created_at : DateTime
+    updated_at : DateTime
+    deleted_at : DateTime
+    deleted : Integer
+}
+
+table(service) {
+    primary_key(id: Integer)
+    name: String[255]
+    host: String[255]
+    last_seen_up: DateTime
+
+    created_at : DateTime
+    updated_at : DateTime
+    deleted_at : DateTime
+    deleted : Integer
+}
+
+ "goals" <.. "strategies" : Foreign Key
+ "goals" <.. "audit_templates" : Foreign Key
+ "strategies" <.. "audit_templates" : Foreign Key
+ "goals" <.. "audits" : Foreign Key
+ "strategies" <.. "audits" : Foreign Key
+ "action_plans" <.. "actions" : Foreign Key
+ "action_plans" <.. "efficacy_indicators" : Foreign Key
+ "strategies" <.. "action_plans" : Foreign Key
+ "audits" <.. "action_plans" : Foreign Key
 
 @enduml

doc/source/index.rst

@@ -56,7 +56,9 @@ Getting Started
   dev/devstack
   deploy/configuration
   deploy/conf-files
+  dev/notifications
   dev/testing
+  dev/rally_link
 
 API References
 --------------
@@ -74,6 +76,7 @@ Plugins
 
   dev/plugin/base-setup
   dev/plugin/goal-plugin
+  dev/plugin/scoring-engine-plugin
   dev/plugin/strategy-plugin
   dev/plugin/cdmc-plugin
   dev/plugin/action-plugin
@@ -94,6 +97,7 @@ Introduction
   deploy/user-guide
   deploy/policy
   deploy/gmr
+  strategies/strategies
 
 Watcher Manual Pages
 ====================

doc/source/strategies/basic-server-consolidation.rst

@@ -0,0 +1,96 @@
+==================================
+Basic Offline Server Consolidation
+==================================
+
+Synopsis
+--------
+
+**display name**: ``basic``
+
+**goal**: ``server_consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.basic_consolidation
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *basic* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameter is:
+
+====================== ====== ============= ===================================
+parameter              type   default Value description
+====================== ====== ============= ===================================
+``migration_attempts`` Number 0             Maximum number of combinations to
+                                            be tried by the strategy while
+                                            searching for potential candidates.
+                                            To remove the limit, set it to 0
+====================== ====== ============= ===================================
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 server_consolidation --strategy basic
+
+    $ openstack optimize audit create -a at1 -p migration_attempts=4
+
+External Links
+--------------
+None.

doc/source/strategies/outlet_temp_control.rst

@@ -0,0 +1,101 @@
+=================================
+Outlet Temperature Based Strategy
+=================================
+
+Synopsis
+--------
+
+**display name**: ``outlet_temperature``
+
+**goal**: ``thermal_optimization``
+
+Outlet (Exhaust Air) temperature is a new thermal telemetry which can be
+used to measure the host's thermal/workload status. This strategy makes
+decisions to migrate workloads to the hosts with good thermal condition
+(lowest outlet temperature) when the outlet temperature of source hosts
+reach a configurable threshold.
+
+Requirements
+------------
+
+This strategy has a dependency on the host having Intel's Power
+Node Manager 3.0 or later enabled.
+
+
+Metrics
+*******
+
+The *outlet_temperature* strategy requires the following metrics:
+
+========================================= ============ ======= =======
+metric                                    service name plugins comment
+========================================= ============ ======= =======
+``hardware.ipmi.node.outlet_temperature`` ceilometer_  IPMI
+========================================= ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#ipmi-based-meters
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameter is:
+
+============== ====== ============= ====================================
+parameter      type   default Value description
+============== ====== ============= ====================================
+``threshold``  Number 35.0          Temperature threshold for migration
+============== ====== ============= ====================================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Outlet Temperature Based Strategy please refer to:
+https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/outlet-temperature-based-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 thermal_optimization --strategy outlet_temperature
+
+    $ openstack optimize audit create -a at1 -p threshold=31.0
+
+External Links
+--------------
+
+- `Intel Power Node Manager 3.0 <http://www.intel.com/content/www/us/en/power-management/intelligent-power-node-manager-3-0-specification.html>`_

doc/source/strategies/strategies.rst

@@ -0,0 +1,8 @@
+Strategies
+==========
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   ./*

doc/source/strategies/strategy-template.rst

@@ -0,0 +1,115 @@
+=============
+Strategy name
+=============
+
+Synopsis
+--------
+
+**display name**:
+
+**goal**:
+
+Add here a complete description of your strategy
+
+Requirements
+------------
+
+Metrics
+*******
+
+Write here the list of metrics required by your strategy algorithm (in the form
+ of a table). If these metrics requires specific Telemetry plugin or other
+ additional software, please explain here how to deploy them (and add link to
+ dedicated installation guide).
+
+Example:
+
+======================= ============ ======= =======
+metric                  service name plugins comment
+======================= ============ ======= =======
+compute.node.*          ceilometer_  none    one point every 60s
+vm.cpu.utilization_perc monasca_     none
+power                   ceilometer_  kwapi_  one point every 60s
+======================= ============ ======= =======
+
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+.. _monasca: https://github.com/openstack/monasca-agent/blob/master/docs/Libvirt.md
+.. _kwapi: https://kwapi.readthedocs.io/en/latest/index.html
+
+
+Cluster data model
+******************
+
+Default Watcher's cluster data model.
+
+or
+
+If your strategy implementation requires a new cluster data model, please
+ describe it in this section, with a link to model plugin's installation guide.
+
+Actions
+*******
+
+Default Watcher's actions.
+
+or
+
+If your strategy implementation requires new actions, add the list of Action
+ plugins here (in the form of a table) with a link to the plugin's installation
+ procedure.
+
+======== =================
+action   description
+======== =================
+action1_ This action1 ...
+action2_ This action2 ...
+======== =================
+
+.. _action1 : https://github.com/myrepo/watcher/plugins/action1
+.. _action2 : https://github.com/myrepo/watcher/plugins/action2
+
+Planner
+*******
+
+Default Watcher's planner.
+
+or
+
+If your strategy requires also a new planner to schedule built actions in time,
+ please describe it in this section, with a link to planner plugin's
+ installation guide.
+
+Configuration
+-------------
+
+If your strategy use configurable parameters, explain here how to tune them.
+
+
+Efficacy Indicator
+------------------
+
+Add here the Efficacy indicator computed by your strategy.
+
+Algorithm
+---------
+
+Add here either the description of your algorithm or
+link to the existing description.
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ Write the command line to create an audit with your strategy.
+
+External Links
+--------------
+
+If you have written papers, blog articles .... about your strategy into Watcher,
+ or if your strategy is based from external publication(s), please add HTTP
+ links and references in this section.
+
+- `link1 <http://www.link1.papers.com>`_
+- `link2 <http://www.link2.papers.com>`_

doc/source/strategies/uniform_airflow.rst

@@ -0,0 +1,107 @@
+==================================
+Uniform Airflow Migration Strategy
+==================================
+
+Synopsis
+--------
+
+**display name**: ``uniform_airflow``
+
+**goal**: ``airflow_optimization``
+
+.. watcher-term:: watcher.decision_engine.strategy.strategies.uniform_airflow
+
+Requirements
+------------
+
+This strategy has a dependency on the server having Intel's Power
+Node Manager 3.0 or later enabled.
+
+Metrics
+*******
+
+The *uniform_airflow* strategy requires the following metrics:
+
+================================== ============ ======= =======
+metric                             service name plugins comment
+================================== ============ ======= =======
+``hardware.ipmi.node.airflow``     ceilometer_  IPMI
+``hardware.ipmi.node.temperature`` ceilometer_  IPMI
+``hardware.ipmi.node.power``       ceilometer_  IPMI
+================================== ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#ipmi-based-meters
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+====================== ====== ============= ===========================
+parameter              type   default Value description
+====================== ====== ============= ===========================
+``threshold_airflow``  Number 400.0         Airflow threshold for
+                                            migration Unit is 0.1CFM
+``threshold_inlet_t``  Number 28.0          Inlet temperature threshold
+                                            for migration decision
+``threshold_power``    Number 350.0         System power threshold for
+                                            migration decision
+``period``             Number 300           Aggregate time period of
+                                            ceilometer
+====================== ====== ============= ===========================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Uniform Airflow Migration Strategy please refer to:
+https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/uniform-airflow-migration-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 airflow_optimization --strategy uniform_airflow
+
+    $ openstack optimize audit create -a at1 -p threshold_airflow=410 \
+           -p threshold_inlet_t=29.0 -p threshold_power=355.0 -p period=310
+
+External Links
+--------------
+
+- `Intel Power Node Manager 3.0 <http://www.intel.com/content/www/us/en/power-management/intelligent-power-node-manager-3-0-specification.html>`_

doc/source/strategies/vm_workload_consolidation.rst

@@ -0,0 +1,100 @@
+==================================
+VM Workload Consolidation Strategy
+==================================
+
+Synopsis
+--------
+
+**display name**: ``vm_workload_consolidation``
+
+**goal**: ``vm_consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.vm_workload_consolidation
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *vm_workload_consolidation* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory``             	     ceilometer_  none
+``disk.root.size``           ceilometer_  none
+============================ ============ ======= =======
+
+The following metrics are not required but increase the accuracy of
+the strategy if available:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory.usage``             ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+Algorithm
+---------
+
+For more information on the VM Workload consolidation strategy please refer to: https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/zhaw-load-consolidation.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 vm_consolidation --strategy vm_workload_consolidation
+
+    $ openstack optimize audit create -a at1
+
+External Links
+--------------
+
+*Spec URL*
+https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/zhaw-load-consolidation.html

doc/source/strategies/workload-stabilization.rst

@@ -0,0 +1,141 @@
+=============================================
+Watcher Overload standard deviation algorithm
+=============================================
+
+Synopsis
+--------
+
+**display name**: ``workload_stabilization``
+
+**goal**: ``workload_balancing``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.workload_stabilization
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *workload_stabilization* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``hardware.memory.used``     ceilometer_  SNMP_
+``cpu_util``                 ceilometer_  none
+``memory.resident``          ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+.. _SNMP: http://docs.openstack.org/admin-guide/telemetry-measurements.html
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+==================== ====== ===================== =============================
+parameter            type   default Value         description
+==================== ====== ===================== =============================
+``metrics``          array  |metrics|             Metrics used as rates of
+                                                  cluster loads.
+``thresholds``       object |thresholds|          Dict where key is a metric
+                                                  and value is a trigger value.
+
+``weights``          object |weights|             These weights used to
+                                                  calculate common standard
+                                                  deviation. Name of weight
+                                                  contains meter name and
+                                                  _weight suffix.
+``instance_metrics`` object |instance_metrics|    Mapping to get hardware
+                                                  statistics using instance
+                                                  metrics.
+``host_choice``      string retry                 Method of host's choice.
+                                                  There are cycle, retry and
+                                                  fullsearch methods. Cycle
+                                                  will iterate hosts in cycle.
+                                                  Retry will get some hosts
+                                                  random (count defined in
+                                                  retry_count option).
+                                                  Fullsearch will return each
+                                                  host from list.
+``retry_count``      number 1                     Count of random returned
+                                                  hosts.
+``periods``          object |periods|             These periods are used to get
+                                                  statistic aggregation for
+                                                  instance and host metrics.
+                                                  The period is simply a
+                                                  repeating interval of time
+                                                  into which the samples are
+                                                  grouped for aggregation.
+                                                  Watcher uses only the last
+                                                  period of all recieved ones.
+==================== ====== ===================== =============================
+
+.. |metrics| replace:: ["cpu_util", "memory.resident"]
+.. |thresholds| replace:: {"cpu_util": 0.2, "memory.resident": 0.2}
+.. |weights| replace:: {"cpu_util_weight": 1.0, "memory.resident_weight": 1.0}
+.. |instance_metrics| replace:: {"cpu_util": "compute.node.cpu.percent", "memory.resident": "hardware.memory.used"}
+.. |periods| replace:: {"instance": 720, "node": 600}
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+Algorithm
+---------
+
+You can find description of overload algorithm and role of standard deviation
+here: https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 workload_balancing --strategy workload_stabilization
+
+    $ openstack optimize audit create -a at1 \
+      -p thresholds='{"memory.resident": 0.05}' \
+      -p metrics='["memory.resident"]'
+
+External Links
+--------------
+
+- `Watcher Overload standard deviation algorithm spec <https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html>`_

doc/source/strategies/workload_balance.rst

@@ -0,0 +1,98 @@
+===================================
+Workload Balance Migration Strategy
+===================================
+
+Synopsis
+--------
+
+**display name**: ``workload_balance``
+
+**goal**: ``workload_balancing``
+
+.. watcher-term:: watcher.decision_engine.strategy.strategies.workload_balance
+
+Requirements
+------------
+
+None.
+
+Metrics
+*******
+
+The *workload_balance* strategy requires the following metrics:
+
+======================= ============ ======= =======
+metric                  service name plugins comment
+======================= ============ ======= =======
+``cpu_util``            ceilometer_  none
+======================= ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+============== ====== ============= ====================================
+parameter      type   default Value description
+============== ====== ============= ====================================
+``threshold``  Number 25.0          Workload threshold for migration
+``period``     Number 300           Aggregate time period of ceilometer
+============== ====== ============= ====================================
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the Workload Balance Migration Strategy please refer
+to: https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/workload-balance-migration-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 workload_balancing --strategy workload_balance
+
+    $ openstack optimize audit create -a at1 -p threshold=26.0 \
+            -p period=310
+
+External Links
+--------------
+
+None.

etc/watcher/policy.json

@@ -37,5 +37,9 @@
 
     "strategy:detail": "rule:default",
     "strategy:get": "rule:default",
-    "strategy:get_all": "rule:default"
+    "strategy:get_all": "rule:default",
+
+    "service:detail": "rule:default",
+    "service:get": "rule:default",
+    "service:get_all": "rule:default"
 }

rally-jobs/README.rst

@@ -0,0 +1,42 @@
+Rally job
+=========
+
+We provide, with Watcher, a Rally plugin you can use to benchmark the optimization service.
+
+To launch this task with configured Rally you just need to run:
+
+::
+
+  rally task start watcher/rally-jobs/watcher-watcher.yaml
+
+Structure
+---------
+
+* plugins - directory where you can add rally plugins. Almost everything in
+  Rally is a plugin. Benchmark context, Benchmark scenario, SLA checks, Generic
+  cleanup resources, ....
+
+* extra - all files from this directory will be copy pasted to gates, so you
+  are able to use absolute paths in rally tasks.
+  Files will be located in ~/.rally/extra/*
+
+* watcher.yaml is a task that is run in gates against OpenStack
+  deployed by DevStack
+
+
+Useful links
+------------
+
+* How to install: http://docs.openstack.org/developer/rally/install.html
+
+* How to set Rally up and launch your first scenario:  https://rally.readthedocs.io/en/latest/tutorial/step_1_setting_up_env_and_running_benchmark_from_samples.html
+
+* More about Rally: https://rally.readthedocs.org/en/latest/
+
+* Rally release notes: https://rally.readthedocs.org/en/latest/release_notes.html
+
+* How to add rally-gates: https://rally.readthedocs.org/en/latest/gates.html
+
+* About plugins:  https://rally.readthedocs.org/en/latest/plugins.html
+
+* Plugin samples: https://github.com/openstack/rally/tree/master/samples/plugins

rally-jobs/watcher-watcher.yaml

@@ -0,0 +1,63 @@
+---
+  Watcher.create_audit_and_delete:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+        audit_templates:
+          audit_templates_per_admin: 5
+          fill_strategy: "round_robin"
+          params:
+            - goal:
+                  name: "dummy"
+              strategy:
+                  name: "dummy"
+      sla:
+        failure_rate:
+          max: 0
+
+  Watcher.create_audit_template_and_delete:
+    -
+      args:
+        goal:
+          name: "dummy"
+        strategy:
+          name: "dummy"
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      sla:
+        failure_rate:
+          max: 0
+
+  Watcher.list_audit_templates:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+        audit_templates:
+          audit_templates_per_admin: 5
+          fill_strategy: "random"
+          params:
+            - goal:
+                  name: "workload_balancing"
+              strategy:
+                  name: "workload_stabilization"
+            - goal:
+                  name: "dummy"
+              strategy:
+                  name: "dummy"
+      sla:
+        failure_rate:
+          max: 0

releasenotes/notes/add-scoring-module-fa00d013ed2d614e.yaml

@@ -1,7 +1,7 @@
 ---
 features:
   - Added a generic scoring engine module, which
-    will standarize interactions with scoring engines
+    will standardize interactions with scoring engines
     through the common API. It is possible to use the
     scoring engine by different Strategies, which
     improve the code and data model re-use.

releasenotes/notes/get-goal-from-strategy-396c9b13a38bb650.yaml

@@ -2,4 +2,4 @@
 features:
   - Added a way to return the of available goals depending
     on which strategies have been deployed on the node
-    where the decison engine is running.
+    where the decision engine is running.

releasenotes/source/conf.py

@@ -242,3 +242,6 @@ texinfo_documents = [
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
+
+# -- Options for Internationalization output ------------------------------
+locale_dirs = ['locale/']

releasenotes/source/index.rst

@@ -6,5 +6,6 @@ Contents:
 .. toctree::
    :maxdepth: 1
 
-   unreleased.rst
+   unreleased
+   newton
 

releasenotes/source/newton.rst

@@ -0,0 +1,6 @@
+===================================
+ Newton Series Release Notes
+===================================
+
+.. release-notes::
+   :branch: origin/stable/newton

requirements.txt

@@ -5,36 +5,38 @@
 apscheduler # MIT License
 enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
 jsonpatch>=1.1 # BSD
-keystoneauth1>=2.10.0 # Apache-2.0
-keystonemiddleware!=4.1.0,!=4.5.0,>=4.0.0 # Apache-2.0
+keystoneauth1>=2.14.0 # Apache-2.0
+keystonemiddleware!=4.5.0,>=4.2.0 # Apache-2.0
+lxml>=2.3 # BSD
 oslo.concurrency>=3.8.0 # Apache-2.0
 oslo.cache>=1.5.0 # Apache-2.0
-oslo.config>=3.14.0 # Apache-2.0
+oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
 oslo.context>=2.9.0 # Apache-2.0
-oslo.db>=4.10.0 # Apache-2.0
+oslo.db!=4.13.1,!=4.13.2,>=4.11.0 # Apache-2.0
 oslo.i18n>=2.1.0 # Apache-2.0
-oslo.log>=1.14.0 # Apache-2.0
-oslo.messaging>=5.2.0 # Apache-2.0
-oslo.policy>=1.9.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
+oslo.messaging>=5.14.0 # Apache-2.0
+oslo.policy>=1.17.0 # Apache-2.0
 oslo.reports>=0.6.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
 oslo.service>=1.10.0 # Apache-2.0
-oslo.utils>=3.16.0 # Apache-2.0
+oslo.utils>=3.18.0 # Apache-2.0
+oslo.versionedobjects>=1.17.0 # Apache-2.0
 PasteDeploy>=1.5.0 # MIT
-pbr>=1.6 # Apache-2.0
-pecan!=1.0.2,!=1.0.3,!=1.0.4,>=1.0.0 # BSD
-PrettyTable<0.8,>=0.7 # BSD
+pbr>=1.8 # Apache-2.0
+pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
+PrettyTable<0.8,>=0.7.1 # BSD
 voluptuous>=0.8.9 # BSD License
 python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0
-python-glanceclient!=2.4.0,>=2.3.0 # Apache-2.0
-python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
+python-glanceclient>=2.5.0 # Apache-2.0
+python-keystoneclient>=3.8.0 # Apache-2.0
 python-neutronclient>=5.1.0 # Apache-2.0
 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
-python-openstackclient>=2.1.0 # Apache-2.0
+python-openstackclient>=3.3.0 # Apache-2.0
 six>=1.9.0 # MIT
 SQLAlchemy<1.1.0,>=1.0.10 # MIT
-stevedore>=1.16.0 # Apache-2.0
+stevedore>=1.17.1 # Apache-2.0
 taskflow>=1.26.0 # Apache-2.0
-WebOb>=1.2.3 # MIT
+WebOb>=1.6.0 # MIT
 WSME>=0.8 # MIT

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = python-watcher
-summary = Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physical resources usage through better VM placement. Watcher can improve your cloud optimization by reducing energy footprint and increasing profits.
+summary = OpenStack Watcher provides a flexible and scalable resource optimization service for multi-tenant OpenStack-based clouds.
 description-file =
     README.rst
 author = OpenStack
@@ -32,7 +32,7 @@ setup-hooks =
 
 [entry_points]
 oslo.config.opts =
-    watcher = watcher.opts:list_opts
+    watcher = watcher.conf.opts:list_opts
 
 console_scripts =
     watcher-api = watcher.cmd.api:main

test-requirements.txt

@@ -2,25 +2,25 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-coverage>=3.6 # Apache-2.0
+coverage>=4.0 # Apache-2.0
 doc8 # Apache-2.0
-freezegun # Apache-2.0
+freezegun>=0.3.6 # Apache-2.0
 hacking<0.11,>=0.10.2
 mock>=2.0 # BSD
 oslotest>=1.10.0 # Apache-2.0
-os-testr>=0.7.0 # Apache-2.0
+os-testr>=0.8.0 # Apache-2.0
 python-subunit>=0.0.18 # Apache-2.0/BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
 
 # Doc requirements
-oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
-sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+oslosphinx>=4.7.0 # Apache-2.0
+sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
 sphinxcontrib-pecanwsme>=0.8 # Apache-2.0
 
 # releasenotes
-reno>=1.8.0 # Apache2
+reno>=1.8.0 # Apache-2.0
 
 # bandit
 bandit>=1.1.0 # Apache-2.0

tox.ini

@@ -1,26 +1,26 @@
 [tox]
-minversion = 1.6
+minversion = 1.8
 envlist = py35,py34,py27,pep8
 skipsdist = True
 
 [testenv]
 usedevelop = True
 whitelist_externals = find
-install_command = pip install -U {opts} {packages}
+install_command =
+                   pip install -U --force-reinstall -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
+
 setenv =
    VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
+deps = -r{toxinidir}/test-requirements.txt
 commands =
-  find . -type f -name "*.pyc" -delete
-  find . -type d -name "__pycache__" -delete
+  find . -type f -name "*.py[c|o]" -delete
   ostestr --concurrency=6 {posargs}
 
 [testenv:pep8]
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    bandit -r watcher -x tests -n5 -ll
+    bandit -r watcher -x tests -n5 -ll -s B320
 
 [testenv:venv]
 setenv = PYTHONHASHSEED=0
@@ -66,4 +66,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen
 
 [testenv:bandit]
 deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r watcher -x tests -n5 -ll
+commands = bandit -r watcher -x tests -n5 -ll -s B320

watcher/api/acl.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 #
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
+# Copyright (c) 2016 Intel Corp
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -17,32 +18,23 @@
 
 """Access Control Lists (ACL's) control access the API server."""
 
-from oslo_config import cfg
 from watcher.api.middleware import auth_token
+from watcher import conf
 
-
-AUTH_OPTS = [
-    cfg.BoolOpt('enable_authentication',
-                default=True,
-                help='This option enables or disables user authentication '
-                'via keystone. Default value is True.'),
-]
-
-CONF = cfg.CONF
-CONF.register_opts(AUTH_OPTS)
+CONF = conf.CONF
 
 
 def install(app, conf, public_routes):
     """Install ACL check on application.
 
-    :param app: A WSGI applicatin.
+    :param app: A WSGI application.
     :param conf: Settings. Dict'ified and passed to keystonemiddleware
     :param public_routes: The list of the routes which will be allowed to
                           access without authentication.
     :return: The same WSGI application with ACL installed.
 
     """
-    if not cfg.CONF.get('enable_authentication'):
+    if not CONF.get('enable_authentication'):
         return app
     return auth_token.AuthTokenMiddleware(app,
                                           conf=dict(conf),

watcher/api/app.py

@@ -2,6 +2,7 @@
 
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
 # All Rights Reserved.
+# Copyright (c) 2016 Intel Corp
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -16,49 +17,14 @@
 #    under the License.
 
 
-from oslo_config import cfg
 import pecan
 
-from watcher._i18n import _
 from watcher.api import acl
 from watcher.api import config as api_config
 from watcher.api import middleware
+from watcher import conf
 
-# Register options for the service
-API_SERVICE_OPTS = [
-    cfg.PortOpt('port',
-                default=9322,
-                help=_('The port for the watcher API server')),
-    cfg.StrOpt('host',
-               default='127.0.0.1',
-               help=_('The listen IP for the watcher API server')),
-    cfg.IntOpt('max_limit',
-               default=1000,
-               help=_('The maximum number of items returned in a single '
-                      'response from a collection resource')),
-    cfg.IntOpt('workers',
-               min=1,
-               help=_('Number of workers for Watcher API service. '
-                      'The default is equal to the number of CPUs available '
-                      'if that can be determined, else a default worker '
-                      'count of 1 is returned.')),
-
-    cfg.BoolOpt('enable_ssl_api',
-                default=False,
-                help=_("Enable the integrated stand-alone API to service "
-                       "requests via HTTPS instead of HTTP. If there is a "
-                       "front-end service performing HTTPS offloading from "
-                       "the service, this option should be False; note, you "
-                       "will want to change public API endpoint to represent "
-                       "SSL termination URL with 'public_endpoint' option.")),
-]
-
-CONF = cfg.CONF
-opt_group = cfg.OptGroup(name='api',
-                         title='Options for the watcher-api service')
-
-CONF.register_group(opt_group)
-CONF.register_opts(API_SERVICE_OPTS, opt_group)
+CONF = conf.CONF
 
 
 def get_pecan_config():

watcher/api/controllers/root.py

@@ -61,7 +61,7 @@ class Root(base.APIBase):
         root = Root()
         root.name = "OpenStack Watcher API"
         root.description = ("Watcher is an OpenStack project which aims to "
-                            "to improve physical resources usage through "
+                            "improve physical resources usage through "
                             "better VM placement.")
         root.versions = [Version.convert('v1')]
         root.default_version = Version.convert('v1')

watcher/api/controllers/v1/__init__.py

@@ -35,6 +35,7 @@ from watcher.api.controllers.v1 import audit
 from watcher.api.controllers.v1 import audit_template
 from watcher.api.controllers.v1 import goal
 from watcher.api.controllers.v1 import scoring_engine
+from watcher.api.controllers.v1 import service
 from watcher.api.controllers.v1 import strategy
 
 
@@ -105,6 +106,9 @@ class V1(APIBase):
     scoring_engines = [link.Link]
     """Links to the Scoring Engines resource"""
 
+    services = [link.Link]
+    """Links to the services resource"""
+
     links = [link.Link]
     """Links that point to a specific URL for this version and documentation"""
 
@@ -159,6 +163,14 @@ class V1(APIBase):
                                 'scoring_engines', '',
                                 bookmark=True)
             ]
+
+        v1.services = [link.Link.make_link(
+            'self', pecan.request.host_url, 'services', ''),
+            link.Link.make_link('bookmark',
+                                pecan.request.host_url,
+                                'services', '',
+                                bookmark=True)
+            ]
         return v1
 
 
@@ -171,6 +183,7 @@ class Controller(rest.RestController):
     action_plans = action_plan.ActionPlansController()
     goals = goal.GoalsController()
     scoring_engines = scoring_engine.ScoringEngineController()
+    services = service.ServicesController()
     strategies = strategy.StrategiesController()
 
     @wsme_pecan.wsexpose(V1)

watcher/api/controllers/v1/action.py

@@ -41,7 +41,7 @@ be one of the following:
    processed by the :ref:`Watcher Applier <watcher_applier_definition>`
 -  **SUCCEEDED** : the :ref:`Action <action_definition>` has been executed
    successfully
--  **FAILED** : an error occured while trying to execute the
+-  **FAILED** : an error occurred while trying to execute the
    :ref:`Action <action_definition>`
 -  **DELETED** : the :ref:`Action <action_definition>` is still stored in the
    :ref:`Watcher database <watcher_database_definition>` but is not returned
@@ -379,7 +379,7 @@ class ActionsController(rest.RestController):
         action_dict = action.as_dict()
         context = pecan.request.context
         new_action = objects.Action(context, **action_dict)
-        new_action.create(context)
+        new_action.create()
 
         # Set the HTTP Location Header
         pecan.response.location = link.build_url('actions', new_action.uuid)

watcher/api/controllers/v1/action_plan.py

@@ -496,7 +496,6 @@ class ActionPlansController(rest.RestController):
         :param action_plan_uuid: UUID of a action plan.
         :param patch: a json PATCH document to apply to this action plan.
         """
-        launch_action_plan = True
         if self.from_actionsPlans:
             raise exception.OperationNotPermitted
 

watcher/api/controllers/v1/audit.py

@@ -60,8 +60,6 @@ class AuditPostType(wtypes.Base):
 
     audit_type = wtypes.wsattr(wtypes.text, mandatory=True)
 
-    deadline = wtypes.wsattr(datetime.datetime, mandatory=False)
-
     state = wsme.wsattr(wtypes.text, readonly=True,
                         default=objects.audit.State.PENDING)
 
@@ -69,8 +67,7 @@ class AuditPostType(wtypes.Base):
                                default={})
     interval = wsme.wsattr(int, mandatory=False)
 
-    host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1),
-                                 mandatory=False)
+    scope = wtypes.wsattr(types.jsontype, readonly=True)
 
     def as_audit(self, context):
         audit_type_values = [val.value for val in objects.audit.AuditType]
@@ -82,7 +79,7 @@ class AuditPostType(wtypes.Base):
             raise exception.AuditIntervalNotAllowed(audit_type=self.audit_type)
 
         if (self.audit_type == objects.audit.AuditType.CONTINUOUS.value and
-           self.interval in (wtypes.Unset, None)):
+                self.interval in (wtypes.Unset, None)):
             raise exception.AuditIntervalNotSpecified(
                 audit_type=self.audit_type)
 
@@ -100,7 +97,7 @@ class AuditPostType(wtypes.Base):
             at2a = {
                 'goal': 'goal_id',
                 'strategy': 'strategy_id',
-                'host_aggregate': 'host_aggregate'
+                'scope': 'scope',
             }
             to_string_fields = set(['goal', 'strategy'])
             for k in at2a:
@@ -114,12 +111,11 @@ class AuditPostType(wtypes.Base):
                         pass
         return Audit(
             audit_type=self.audit_type,
-            deadline=self.deadline,
             parameters=self.parameters,
             goal_id=self.goal,
-            host_aggregate=self.host_aggregate,
             strategy_id=self.strategy,
-            interval=self.interval)
+            interval=self.interval,
+            scope=self.scope,)
 
 
 class AuditPatchType(types.JsonPatchType):
@@ -230,9 +226,6 @@ class Audit(base.APIBase):
     audit_type = wtypes.text
     """Type of this audit"""
 
-    deadline = datetime.datetime
-    """deadline of the audit"""
-
     state = wtypes.text
     """This audit state"""
 
@@ -261,8 +254,8 @@ class Audit(base.APIBase):
     interval = wsme.wsattr(int, mandatory=False)
     """Launch audit periodically (in seconds)"""
 
-    host_aggregate = wtypes.IntegerType(minimum=1)
-    """ID of the Nova host aggregate targeted by the audit template"""
+    scope = wsme.wsattr(types.jsontype, mandatory=False)
+    """Audit Scope"""
 
     def __init__(self, **kwargs):
         self.fields = []
@@ -292,10 +285,10 @@ class Audit(base.APIBase):
     @staticmethod
     def _convert_with_links(audit, url, expand=True):
         if not expand:
-            audit.unset_fields_except(['uuid', 'audit_type', 'deadline',
-                                       'state', 'goal_uuid', 'interval',
-                                       'strategy_uuid', 'host_aggregate',
-                                       'goal_name', 'strategy_name'])
+            audit.unset_fields_except(['uuid', 'audit_type', 'state',
+                                       'goal_uuid', 'interval', 'scope',
+                                       'strategy_uuid', 'goal_name',
+                                       'strategy_name'])
 
         audit.links = [link.Link.make_link('self', url,
                                            'audits', audit.uuid),
@@ -316,15 +309,15 @@ class Audit(base.APIBase):
         sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
                      audit_type='ONESHOT',
                      state='PENDING',
-                     deadline=None,
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
                      updated_at=datetime.datetime.utcnow(),
-                     interval=7200)
+                     interval=7200,
+                     scope=[])
 
         sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
         sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff'
-        sample.host_aggregate = 1
+
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -381,7 +374,7 @@ class AuditsController(rest.RestController):
     def _get_audits_collection(self, marker, limit,
                                sort_key, sort_dir, expand=False,
                                resource_url=None, goal=None,
-                               strategy=None, host_aggregate=None):
+                               strategy=None):
         limit = api_utils.validate_limit(limit)
         api_utils.validate_sort_dir(sort_dir)
         marker_obj = None
@@ -424,19 +417,16 @@ class AuditsController(rest.RestController):
 
     @wsme_pecan.wsexpose(AuditCollection, types.uuid, int, wtypes.text,
                          wtypes.text, wtypes.text, wtypes.text, int)
-    def get_all(self, marker=None, limit=None,
-                sort_key='id', sort_dir='asc', goal=None,
-                strategy=None, host_aggregate=None):
+    def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc',
+                goal=None, strategy=None):
         """Retrieve a list of audits.
 
         :param marker: pagination marker for large data sets.
         :param limit: maximum number of resources to return in a single result.
         :param sort_key: column to sort results by. Default: id.
         :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
-         id.
         :param goal: goal UUID or name to filter by
         :param strategy: strategy UUID or name to filter by
-        :param host_aggregate: Optional host_aggregate
         """
 
         context = pecan.request.context
@@ -445,8 +435,7 @@ class AuditsController(rest.RestController):
 
         return self._get_audits_collection(marker, limit, sort_key,
                                            sort_dir, goal=goal,
-                                           strategy=strategy,
-                                           host_aggregate=host_aggregate)
+                                           strategy=strategy)
 
     @wsme_pecan.wsexpose(AuditCollection, wtypes.text, types.uuid, int,
                          wtypes.text, wtypes.text)
@@ -530,13 +519,12 @@ class AuditsController(rest.RestController):
         audit_dict = audit.as_dict()
 
         new_audit = objects.Audit(context, **audit_dict)
-        new_audit.create(context)
+        new_audit.create()
 
         # Set the HTTP Location Header
         pecan.response.location = link.build_url('audits', new_audit.uuid)
 
         # trigger decision-engine to run the audit
-
         if new_audit.audit_type == objects.audit.AuditType.ONESHOT.value:
             dc_client = rpcapi.DecisionEngineAPI()
             dc_client.trigger_audit(context, new_audit.uuid)
@@ -555,14 +543,11 @@ class AuditsController(rest.RestController):
             raise exception.OperationNotPermitted
 
         context = pecan.request.context
-        audit_to_update = api_utils.get_resource('Audit',
-                                                 audit_uuid)
+        audit_to_update = api_utils.get_resource(
+            'Audit', audit_uuid, eager=True)
         policy.enforce(context, 'audit:update', audit_to_update,
                        action='audit:update')
 
-        audit_to_update = objects.Audit.get_by_uuid(pecan.request.context,
-                                                    audit_uuid)
-
         try:
             audit_dict = audit_to_update.as_dict()
             audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch))
@@ -591,7 +576,8 @@ class AuditsController(rest.RestController):
         :param audit_uuid: UUID of a audit.
         """
         context = pecan.request.context
-        audit_to_delete = api_utils.get_resource('Audit', audit_uuid)
+        audit_to_delete = api_utils.get_resource(
+            'Audit', audit_uuid, eager=True)
         policy.enforce(context, 'audit:update', audit_to_delete,
                        action='audit:update')
 

watcher/api/controllers/v1/audit_template.py

@@ -41,11 +41,6 @@ settings related to the level of automation for the
 A flag will indicate whether the :ref:`Action Plan <action_plan_definition>`
 will be launched automatically or will need a manual confirmation from the
 :ref:`Administrator <administrator_definition>`.
-
-Last but not least, an :ref:`Audit Template <audit_template_definition>` may
-contain a list of extra parameters related to the
-:ref:`Strategy <strategy_definition>` configuration. These parameters can be
-provided as a list of key-value pairs.
 """
 
 import datetime
@@ -66,6 +61,7 @@ from watcher.common import context as context_utils
 from watcher.common import exception
 from watcher.common import policy
 from watcher.common import utils as common_utils
+from watcher.decision_engine.scope import default
 from watcher import objects
 
 
@@ -78,37 +74,24 @@ class AuditTemplatePostType(wtypes.Base):
     description = wtypes.wsattr(wtypes.text, mandatory=False)
     """Short description of this audit template"""
 
-    deadline = wsme.wsattr(datetime.datetime, mandatory=False)
-    """deadline of the audit template"""
-
-    host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1),
-                                 mandatory=False)
-    """ID of the Nova host aggregate targeted by the audit template"""
-
-    extra = wtypes.wsattr({wtypes.text: types.jsontype}, mandatory=False)
-    """The metadata of the audit template"""
-
     goal = wtypes.wsattr(wtypes.text, mandatory=True)
     """Goal UUID or name of the audit template"""
 
     strategy = wtypes.wsattr(wtypes.text, mandatory=False)
     """Strategy UUID or name of the audit template"""
 
-    version = wtypes.text
-    """Internal version of the audit template"""
+    scope = wtypes.wsattr(types.jsontype, mandatory=False, default=[])
+    """Audit Scope"""
 
     def as_audit_template(self):
         return AuditTemplate(
             name=self.name,
             description=self.description,
-            deadline=self.deadline,
-            host_aggregate=self.host_aggregate,
-            extra=self.extra,
             goal_id=self.goal,  # Dirty trick ...
             goal=self.goal,
             strategy_id=self.strategy,  # Dirty trick ...
             strategy_uuid=self.strategy,
-            version=self.version,
+            scope=self.scope,
         )
 
     @staticmethod
@@ -123,6 +106,9 @@ class AuditTemplatePostType(wtypes.Base):
         else:
             raise exception.InvalidGoal(goal=audit_template.goal)
 
+        common_utils.Draft4Validator(
+            default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope)
+
         if audit_template.strategy:
             available_strategies = objects.Strategy.list(
                 AuditTemplatePostType._ctx)
@@ -305,18 +291,9 @@ class AuditTemplate(base.APIBase):
     name = wtypes.text
     """Name of this audit template"""
 
-    description = wtypes.text
+    description = wtypes.wsattr(wtypes.text, mandatory=False)
     """Short description of this audit template"""
 
-    deadline = datetime.datetime
-    """deadline of the audit template"""
-
-    host_aggregate = wtypes.IntegerType(minimum=1)
-    """ID of the Nova host aggregate targeted by the audit template"""
-
-    extra = {wtypes.text: types.jsontype}
-    """The metadata of the audit template"""
-
     goal_uuid = wsme.wsproperty(
         wtypes.text, _get_goal_uuid, _set_goal_uuid, mandatory=True)
     """Goal UUID the audit template refers to"""
@@ -333,15 +310,15 @@ class AuditTemplate(base.APIBase):
         wtypes.text, _get_strategy_name, _set_strategy_name, mandatory=False)
     """The name of the strategy this audit template refers to"""
 
-    version = wtypes.text
-    """Internal version of the audit template"""
-
     audits = wsme.wsattr([link.Link], readonly=True)
     """Links to the collection of audits contained in this audit template"""
 
     links = wsme.wsattr([link.Link], readonly=True)
     """A list containing a self link and associated audit template links"""
 
+    scope = wsme.wsattr(types.jsontype, mandatory=False)
+    """Audit Scope"""
+
     def __init__(self, **kwargs):
         super(AuditTemplate, self).__init__()
         self.fields = []
@@ -374,8 +351,8 @@ class AuditTemplate(base.APIBase):
     def _convert_with_links(audit_template, url, expand=True):
         if not expand:
             audit_template.unset_fields_except(
-                ['uuid', 'name', 'host_aggregate', 'goal_uuid', 'goal_name',
-                 'strategy_uuid', 'strategy_name'])
+                ['uuid', 'name', 'goal_uuid', 'goal_name',
+                 'scope', 'strategy_uuid', 'strategy_name'])
 
         # The numeric ID should not be exposed to
         # the user, it's internal only.
@@ -402,13 +379,12 @@ class AuditTemplate(base.APIBase):
         sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
                      name='My Audit Template',
                      description='Description of my audit template',
-                     host_aggregate=5,
                      goal_uuid='83e44733-b640-40e2-8d8a-7dd3be7134e6',
                      strategy_uuid='367d826e-b6a4-4b70-bc44-c3f6fe1c9986',
-                     extra={'automatic': True},
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
-                     updated_at=datetime.datetime.utcnow())
+                     updated_at=datetime.datetime.utcnow(),
+                     scope=[],)
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -592,11 +568,11 @@ class AuditTemplatesController(rest.RestController):
         audit_template_dict = audit_template.as_dict()
         new_audit_template = objects.AuditTemplate(context,
                                                    **audit_template_dict)
-        new_audit_template.create(context)
+        new_audit_template.create()
 
         # Set the HTTP Location Header
-        pecan.response.location = link.build_url('audit_templates',
-                                                 new_audit_template.uuid)
+        pecan.response.location = link.build_url(
+            'audit_templates', new_audit_template.uuid)
         return AuditTemplate.convert_with_links(new_audit_template)
 
     @wsme.validate(types.uuid, [AuditTemplatePatchType])

watcher/api/controllers/v1/efficacy_indicator.py

@@ -19,7 +19,7 @@ An efficacy indicator is a single value that gives an indication on how the
 :ref:`solution <solution_definition>` produced by a given :ref:`strategy
 <strategy_definition>` performed. These efficacy indicators are specific to a
 given :ref:`goal <goal_definition>` and are usually used to compute the
-:ref:`gobal efficacy <efficacy_definition>` of the resulting :ref:`action plan
+:ref:`global efficacy <efficacy_definition>` of the resulting :ref:`action plan
 <action_plan_definition>`.
 
 In Watcher, these efficacy indicators are specified alongside the goal they

watcher/api/controllers/v1/scoring_engine.py

@@ -16,8 +16,10 @@
 # limitations under the License.
 
 """
-A :ref:`Scoring Engine <scoring_engine_definition>` is an instance of a data
-model, to which a learning data was applied.
+A :ref:`Scoring Engine <scoring_engine_definition>` is an executable that has
+a well-defined input, a well-defined output, and performs a purely mathematical
+task. That is, the calculation does not depend on the environment in which it
+is running - it would produce the same result anywhere.
 
 Because there might be multiple algorithms used to build a particular data
 model (and therefore a scoring engine), the usage of scoring engine might

watcher/api/controllers/v1/service.py

@@ -0,0 +1,263 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 Servionica
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Service mechanism provides ability to monitor Watcher services state.
+"""
+
+import datetime
+import six
+
+from oslo_config import cfg
+from oslo_log import log
+from oslo_utils import timeutils
+import pecan
+from pecan import rest
+import wsme
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from watcher._i18n import _LW
+from watcher.api.controllers import base
+from watcher.api.controllers import link
+from watcher.api.controllers.v1 import collection
+from watcher.api.controllers.v1 import utils as api_utils
+from watcher.common import exception
+from watcher.common import policy
+from watcher import objects
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+class Service(base.APIBase):
+    """API representation of a service.
+
+    This class enforces type checking and value constraints, and converts
+    between the internal object model and the API representation of a service.
+    """
+
+    _status = None
+
+    def _get_status(self):
+        return self._status
+
+    def _set_status(self, name):
+        service = objects.Service.get_by_name(pecan.request.context, name)
+        last_heartbeat = (service.last_seen_up or service.updated_at
+                          or service.created_at)
+        if isinstance(last_heartbeat, six.string_types):
+            # NOTE(russellb) If this service came in over rpc via
+            # conductor, then the timestamp will be a string and needs to be
+            # converted back to a datetime.
+            last_heartbeat = timeutils.parse_strtime(last_heartbeat)
+        else:
+            # Objects have proper UTC timezones, but the timeutils comparison
+            # below does not (and will fail)
+            last_heartbeat = last_heartbeat.replace(tzinfo=None)
+        elapsed = timeutils.delta_seconds(last_heartbeat, timeutils.utcnow())
+        is_up = abs(elapsed) <= CONF.service_down_time
+        if not is_up:
+            LOG.warning(_LW('Seems service %(name)s on host %(host)s is down. '
+                            'Last heartbeat was %(lhb)s.'
+                            'Elapsed time is %(el)s'),
+                        {'name': service.name,
+                         'host': service.host,
+                         'lhb': str(last_heartbeat), 'el': str(elapsed)})
+            self._status = objects.service.ServiceStatus.FAILED
+        else:
+            self._status = objects.service.ServiceStatus.ACTIVE
+
+    id = wsme.wsattr(int, readonly=True)
+    """ID for this service."""
+
+    name = wtypes.text
+    """Name of the service."""
+
+    host = wtypes.text
+    """Host where service is placed on."""
+
+    last_seen_up = wsme.wsattr(datetime.datetime, readonly=True)
+    """Time when Watcher service sent latest heartbeat."""
+
+    status = wsme.wsproperty(wtypes.text, _get_status, _set_status,
+                             mandatory=True)
+
+    links = wsme.wsattr([link.Link], readonly=True)
+    """A list containing a self link."""
+
+    def __init__(self, **kwargs):
+        super(Service, self).__init__()
+
+        fields = list(objects.Service.fields.keys()) + ['status']
+        self.fields = []
+        for field in fields:
+            self.fields.append(field)
+            setattr(self, field, kwargs.get(
+                field if field != 'status' else 'name', wtypes.Unset))
+
+    @staticmethod
+    def _convert_with_links(service, url, expand=True):
+        if not expand:
+            service.unset_fields_except(
+                ['id', 'name', 'host', 'status'])
+
+        service.links = [
+            link.Link.make_link('self', url, 'services', str(service.id)),
+            link.Link.make_link('bookmark', url, 'services', str(service.id),
+                                bookmark=True)]
+        return service
+
+    @classmethod
+    def convert_with_links(cls, service, expand=True):
+        service = Service(**service.as_dict())
+        return cls._convert_with_links(
+            service, pecan.request.host_url, expand)
+
+    @classmethod
+    def sample(cls, expand=True):
+        sample = cls(id=1,
+                     name='watcher-applier',
+                     host='Controller',
+                     last_seen_up=datetime.datetime(2016, 1, 1))
+        return cls._convert_with_links(sample, 'http://localhost:9322', expand)
+
+
+class ServiceCollection(collection.Collection):
+    """API representation of a collection of services."""
+
+    services = [Service]
+    """A list containing services objects"""
+
+    def __init__(self, **kwargs):
+        super(ServiceCollection, self).__init__()
+        self._type = 'services'
+
+    @staticmethod
+    def convert_with_links(services, limit, url=None, expand=False,
+                           **kwargs):
+        service_collection = ServiceCollection()
+        service_collection.services = [
+            Service.convert_with_links(g, expand) for g in services]
+
+        if 'sort_key' in kwargs:
+            reverse = False
+            if kwargs['sort_key'] == 'service':
+                if 'sort_dir' in kwargs:
+                    reverse = True if kwargs['sort_dir'] == 'desc' else False
+                service_collection.services = sorted(
+                    service_collection.services,
+                    key=lambda service: service.id,
+                    reverse=reverse)
+
+        service_collection.next = service_collection.get_next(
+            limit, url=url, marker_field='id', **kwargs)
+        return service_collection
+
+    @classmethod
+    def sample(cls):
+        sample = cls()
+        sample.services = [Service.sample(expand=False)]
+        return sample
+
+
+class ServicesController(rest.RestController):
+    """REST controller for Services."""
+    def __init__(self):
+        super(ServicesController, self).__init__()
+
+    from_services = False
+    """A flag to indicate if the requests to this controller are coming
+    from the top-level resource Services."""
+
+    _custom_actions = {
+        'detail': ['GET'],
+    }
+
+    def _get_services_collection(self, marker, limit, sort_key, sort_dir,
+                                 expand=False, resource_url=None):
+        limit = api_utils.validate_limit(limit)
+        api_utils.validate_sort_dir(sort_dir)
+
+        sort_db_key = (sort_key if sort_key in objects.Service.fields.keys()
+                       else None)
+
+        marker_obj = None
+        if marker:
+            marker_obj = objects.Service.get(
+                pecan.request.context, marker)
+
+        services = objects.Service.list(
+            pecan.request.context, limit, marker_obj,
+            sort_key=sort_db_key, sort_dir=sort_dir)
+
+        return ServiceCollection.convert_with_links(
+            services, limit, url=resource_url, expand=expand,
+            sort_key=sort_key, sort_dir=sort_dir)
+
+    @wsme_pecan.wsexpose(ServiceCollection, int, int, wtypes.text, wtypes.text)
+    def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc'):
+        """Retrieve a list of services.
+
+        :param marker: pagination marker for large data sets.
+        :param limit: maximum number of resources to return in a single result.
+        :param sort_key: column to sort results by. Default: id.
+        :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
+        """
+        context = pecan.request.context
+        policy.enforce(context, 'service:get_all',
+                       action='service:get_all')
+
+        return self._get_services_collection(marker, limit, sort_key, sort_dir)
+
+    @wsme_pecan.wsexpose(ServiceCollection, int, int, wtypes.text, wtypes.text)
+    def detail(self, marker=None, limit=None, sort_key='id', sort_dir='asc'):
+        """Retrieve a list of services with detail.
+
+        :param marker: pagination marker for large data sets.
+        :param limit: maximum number of resources to return in a single result.
+        :param sort_key: column to sort results by. Default: id.
+        :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
+        """
+        context = pecan.request.context
+        policy.enforce(context, 'service:detail',
+                       action='service:detail')
+        # NOTE(lucasagomes): /detail should only work agaist collections
+        parent = pecan.request.path.split('/')[:-1][-1]
+        if parent != "services":
+            raise exception.HTTPNotFound
+        expand = True
+        resource_url = '/'.join(['services', 'detail'])
+
+        return self._get_services_collection(
+            marker, limit, sort_key, sort_dir, expand, resource_url)
+
+    @wsme_pecan.wsexpose(Service, wtypes.text)
+    def get_one(self, service):
+        """Retrieve information about the given service.
+
+        :param service: ID or name of the service.
+        """
+        if self.from_services:
+            raise exception.OperationNotPermitted
+
+        context = pecan.request.context
+        rpc_service = api_utils.get_resource('Service', service)
+        policy.enforce(context, 'service:get', rpc_service,
+                       action='service:get')
+
+        return Service.convert_with_links(rpc_service)

watcher/api/controllers/v1/strategy.py

@@ -110,7 +110,7 @@ class Strategy(base.APIBase):
     """The name of the goal this audit refers to"""
 
     parameters_spec = {wtypes.text: types.jsontype}
-    """ Parameters spec dict"""
+    """Parameters spec dict"""
 
     def __init__(self, **kwargs):
         super(Strategy, self).__init__()

watcher/api/controllers/v1/utils.py

@@ -15,11 +15,13 @@
 
 import jsonpatch
 from oslo_config import cfg
+from oslo_utils import reflection
 from oslo_utils import uuidutils
 import pecan
 import wsme
 
 from watcher._i18n import _
+from watcher.common import utils
 from watcher import objects
 
 CONF = cfg.CONF
@@ -80,17 +82,27 @@ def as_filters_dict(**filters):
     return filters_dict
 
 
-def get_resource(resource, resource_ident):
-    """Get the resource from the uuid or logical name.
+def get_resource(resource, resource_id, eager=False):
+    """Get the resource from the uuid, id or logical name.
 
     :param resource: the resource type.
-    :param resource_ident: the UUID or logical name of the resource.
+    :param resource_id: the UUID, ID or logical name of the resource.
 
     :returns: The resource.
     """
     resource = getattr(objects, resource)
 
-    if uuidutils.is_uuid_like(resource_ident):
-        return resource.get_by_uuid(pecan.request.context, resource_ident)
+    _get = None
+    if utils.is_int_like(resource_id):
+        resource_id = int(resource_id)
+        _get = resource.get
+    elif uuidutils.is_uuid_like(resource_id):
+        _get = resource.get_by_uuid
+    else:
+        _get = resource.get_by_name
 
-    return resource.get_by_name(pecan.request.context, resource_ident)
+    method_signature = reflection.get_signature(_get)
+    if 'eager' in method_signature.parameters:
+        return _get(pecan.request.context, resource_id, eager=eager)
+
+    return _get(pecan.request.context, resource_id)

watcher/api/hooks.py

@@ -18,6 +18,7 @@
 from oslo_config import cfg
 from oslo_utils import importutils
 from pecan import hooks
+from six.moves import http_client
 
 from watcher.common import context
 
@@ -95,22 +96,24 @@ class NoExceptionTracebackHook(hooks.PecanHook):
             return
 
         # Do nothing if there is no error.
-        if 200 <= state.response.status_int < 400:
+        # Status codes in the range 200 (OK) to 399 (400 = BAD_REQUEST) are not
+        # an error.
+        if (http_client.OK <= state.response.status_int <
+                http_client.BAD_REQUEST):
             return
 
         json_body = state.response.json
-        # Do not remove traceback when server in debug mode (except 'Server'
-        # errors when 'debuginfo' will be used for traces).
-        if cfg.CONF.debug and json_body.get('faultcode') != 'Server':
+        # Do not remove traceback when traceback config is set
+        if cfg.CONF.debug:
             return
 
         faultstring = json_body.get('faultstring')
         traceback_marker = 'Traceback (most recent call last):'
-        if faultstring and (traceback_marker in faultstring):
+        if faultstring and traceback_marker in faultstring:
             # Cut-off traceback.
             faultstring = faultstring.split(traceback_marker, 1)[0]
             # Remove trailing newlines and spaces if any.
             json_body['faultstring'] = faultstring.rstrip()
-            # Replace the whole json. Cannot change original one beacause it's
+            # Replace the whole json. Cannot change original one because it's
             # generated on the fly.
             state.response.json = json_body

watcher/api/middleware/auth_token.py

@@ -33,7 +33,7 @@ class AuthTokenMiddleware(auth_token.AuthProtocol):
     for public routes in the API.
 
     """
-    def __init__(self, app, conf, public_api_routes=[]):
+    def __init__(self, app, conf, public_api_routes=()):
         route_pattern_tpl = '%s(\.json|\.xml)?$'
 
         try:

watcher/api/middleware/parsable_error.py

@@ -27,14 +27,14 @@ from oslo_serialization import jsonutils
 import six
 import webob
 
-from watcher._i18n import _
-from watcher._i18n import _LE
+from watcher._i18n import _, _LE
 
 LOG = log.getLogger(__name__)
 
 
 class ParsableErrorMiddleware(object):
     """Replace error body with something the client can parse."""
+
     def __init__(self, app):
         self.app = app
 
@@ -59,8 +59,7 @@ class ParsableErrorMiddleware(object):
                     # compute the length.
                     headers = [(h, v)
                                for (h, v) in headers
-                               if h not in ('Content-Length', 'Content-Type')
-                               ]
+                               if h not in ('Content-Length', 'Content-Type')]
                 # Save the headers in case we need to modify them.
                 state['headers'] = headers
                 return start_response(status, headers, exc_info)
@@ -68,24 +67,27 @@ class ParsableErrorMiddleware(object):
         app_iter = self.app(environ, replacement_start_response)
         if (state['status_code'] // 100) not in (2, 3):
             req = webob.Request(environ)
-            if (req.accept.best_match(['application/json', 'application/xml']
-                                      ) == 'application/xml'):
+            if (
+                    req.accept.best_match(
+                        ['application/json',
+                         'application/xml']) == 'application/xml'
+            ):
                 try:
                     # simple check xml is valid
-                    body = [et.ElementTree.tostring(
-                            et.ElementTree.Element('error_message',
-                                                   text='\n'.join(app_iter)))]
+                    body = [
+                        et.ElementTree.tostring(
+                            et.ElementTree.Element(
+                                'error_message', text='\n'.join(app_iter)))]
                 except et.ElementTree.ParseError as err:
                     LOG.error(_LE('Error parsing HTTP response: %s'), err)
-                    body = [et.ElementTree.tostring(
-                            et.ElementTree.Element('error_message',
-                                                   text=state['status_code']))]
+                    body = ['<error_message>%s'
+                            '</error_message>' % state['status_code']]
                 state['headers'].append(('Content-Type', 'application/xml'))
             else:
                 if six.PY3:
                     app_iter = [i.decode('utf-8') for i in app_iter]
                 body = [jsonutils.dumps(
-                        {'error_message': '\n'.join(app_iter)})]
+                    {'error_message': '\n'.join(app_iter)})]
                 if six.PY3:
                     body = [item.encode('utf-8') for item in body]
                 state['headers'].append(('Content-Type', 'application/json'))

watcher/applier/action_plan/default.py

@@ -20,9 +20,7 @@ from oslo_log import log
 
 from watcher.applier.action_plan import base
 from watcher.applier import default
-from watcher.applier.messaging import event_types
-from watcher.common.messaging.events import event
-from watcher.objects import action_plan as ap_objects
+from watcher import objects
 
 LOG = log.getLogger(__name__)
 
@@ -34,32 +32,20 @@ class DefaultActionPlanHandler(base.BaseActionPlanHandler):
         self.service = service
         self.action_plan_uuid = action_plan_uuid
 
-    def notify(self, uuid, event_type, state):
-        action_plan = ap_objects.ActionPlan.get_by_uuid(self.ctx, uuid)
+    def update_action_plan(self, uuid, state):
+        action_plan = objects.ActionPlan.get_by_uuid(self.ctx, uuid)
         action_plan.state = state
         action_plan.save()
-        ev = event.Event()
-        ev.type = event_type
-        ev.data = {}
-        payload = {'action_plan__uuid': uuid,
-                   'action_plan_state': state}
-        self.service.publish_status_event(ev.type.name, payload)
 
     def execute(self):
         try:
-            # update state
-            self.notify(self.action_plan_uuid,
-                        event_types.EventTypes.LAUNCH_ACTION_PLAN,
-                        ap_objects.State.ONGOING)
+            self.update_action_plan(self.action_plan_uuid,
+                                    objects.action_plan.State.ONGOING)
             applier = default.DefaultApplier(self.ctx, self.service)
             applier.execute(self.action_plan_uuid)
-            state = ap_objects.State.SUCCEEDED
+            state = objects.action_plan.State.SUCCEEDED
         except Exception as e:
             LOG.exception(e)
-            state = ap_objects.State.FAILED
-
+            state = objects.action_plan.State.FAILED
         finally:
-            # update state
-            self.notify(self.action_plan_uuid,
-                        event_types.EventTypes.LAUNCH_ACTION_PLAN,
-                        state)
+            self.update_action_plan(self.action_plan_uuid, state)

watcher/applier/manager.py

@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Copyright (c) 2015 b<>com
+# Copyright (c) 2016 Intel Corp
 #
 # Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
 #
@@ -17,60 +18,40 @@
 # limitations under the License.
 #
 
-from oslo_config import cfg
-
 from watcher.applier.messaging import trigger
+from watcher.common import service_manager
 
-CONF = cfg.CONF
+from watcher import conf
+
+CONF = conf.CONF
 
 
-# Register options
-APPLIER_MANAGER_OPTS = [
-    cfg.IntOpt('workers',
-               default='1',
-               min=1,
-               required=True,
-               help='Number of workers for applier, default value is 1.'),
-    cfg.StrOpt('conductor_topic',
-               default='watcher.applier.control',
-               help='The topic name used for'
-                    'control events, this topic '
-                    'used for rpc call '),
-    cfg.StrOpt('status_topic',
-               default='watcher.applier.status',
-               help='The topic name used for '
-                    'status events, this topic '
-                    'is used so as to notify'
-                    'the others components '
-                    'of the system'),
-    cfg.StrOpt('publisher_id',
-               default='watcher.applier.api',
-               help='The identifier used by watcher '
-                    'module on the message broker'),
-    cfg.StrOpt('workflow_engine',
-               default='taskflow',
-               required=True,
-               help='Select the engine to use to execute the workflow')
-]
+class ApplierManager(service_manager.ServiceManager):
 
-opt_group = cfg.OptGroup(name='watcher_applier',
-                         title='Options for the Applier messaging'
-                               'core')
-CONF.register_group(opt_group)
-CONF.register_opts(APPLIER_MANAGER_OPTS, opt_group)
+    @property
+    def service_name(self):
+        return 'watcher-applier'
 
+    @property
+    def api_version(self):
+        return '1.0'
 
-class ApplierManager(object):
+    @property
+    def publisher_id(self):
+        return CONF.watcher_applier.publisher_id
 
-    API_VERSION = '1.0'
+    @property
+    def conductor_topic(self):
+        return CONF.watcher_applier.conductor_topic
 
-    conductor_endpoints = [trigger.TriggerActionPlan]
-    status_endpoints = []
-    notification_endpoints = []
-    notification_topics = []
+    @property
+    def notification_topics(self):
+        return []
 
-    def __init__(self):
-        self.publisher_id = CONF.watcher_applier.publisher_id
-        self.conductor_topic = CONF.watcher_applier.conductor_topic
-        self.status_topic = CONF.watcher_applier.status_topic
-        self.api_version = self.API_VERSION
+    @property
+    def conductor_endpoints(self):
+        return [trigger.TriggerActionPlan]
+
+    @property
+    def notification_endpoints(self):
+        return []

watcher/applier/rpcapi.py

@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Copyright (c) 2015 b<>com
+# Copyright (c) 2016 Intel Corp
 #
 # Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
 #
@@ -15,18 +16,15 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-from oslo_config import cfg
 
-from watcher.applier import manager
 from watcher.common import exception
 from watcher.common import service
+from watcher.common import service_manager
 from watcher.common import utils
 
+from watcher import conf
 
-CONF = cfg.CONF
-CONF.register_group(manager.opt_group)
-CONF.register_opts(manager.APPLIER_MANAGER_OPTS, manager.opt_group)
+CONF = conf.CONF
 
 
 class ApplierAPI(service.Service):
@@ -39,21 +37,35 @@ class ApplierAPI(service.Service):
             raise exception.InvalidUuidOrName(name=action_plan_uuid)
 
         return self.conductor_client.call(
-            context.to_dict(), 'launch_action_plan',
-            action_plan_uuid=action_plan_uuid)
+            context, 'launch_action_plan', action_plan_uuid=action_plan_uuid)
 
 
-class ApplierAPIManager(object):
+class ApplierAPIManager(service_manager.ServiceManager):
 
-    API_VERSION = '1.0'
+    @property
+    def service_name(self):
+        return None
 
-    conductor_endpoints = []
-    status_endpoints = []
-    notification_endpoints = []
-    notification_topics = []
+    @property
+    def api_version(self):
+        return '1.0'
 
-    def __init__(self):
-        self.publisher_id = CONF.watcher_applier.publisher_id
-        self.conductor_topic = CONF.watcher_applier.conductor_topic
-        self.status_topic = CONF.watcher_applier.status_topic
-        self.api_version = self.API_VERSION
+    @property
+    def publisher_id(self):
+        return CONF.watcher_applier.publisher_id
+
+    @property
+    def conductor_topic(self):
+        return CONF.watcher_applier.conductor_topic
+
+    @property
+    def notification_topics(self):
+        return []
+
+    @property
+    def conductor_endpoints(self):
+        return []
+
+    @property
+    def notification_endpoints(self):
+        return []

watcher/applier/workflow_engine/base.py

@@ -21,10 +21,8 @@ import abc
 import six
 
 from watcher.applier.actions import factory
-from watcher.applier.messaging import event_types
 from watcher.common import clients
 from watcher.common.loader import loadable
-from watcher.common.messaging.events import event
 from watcher import objects
 
 
@@ -77,12 +75,7 @@ class BaseWorkFlowEngine(loadable.Loadable):
         db_action = objects.Action.get_by_uuid(self.context, action.uuid)
         db_action.state = state
         db_action.save()
-        ev = event.Event()
-        ev.type = event_types.EventTypes.LAUNCH_ACTION
-        ev.data = {}
-        payload = {'action_uuid': action.uuid,
-                   'action_state': state}
-        self.applier_manager.publish_status_event(ev.type.name, payload)
+        # NOTE(v-francoise): Implement notifications for action
 
     @abc.abstractmethod
     def execute(self, actions):

watcher/applier/workflow_engine/default.py

@@ -23,7 +23,7 @@ from taskflow import task
 from watcher._i18n import _LE, _LW, _LC
 from watcher.applier.workflow_engine import base
 from watcher.common import exception
-from watcher.objects import action as obj_action
+from watcher import objects
 
 LOG = log.getLogger(__name__)
 
@@ -107,14 +107,12 @@ class TaskFlowActionContainer(task.Task):
 
     def pre_execute(self):
         try:
-            self.engine.notify(self._db_action,
-                               obj_action.State.ONGOING)
+            self.engine.notify(self._db_action, objects.action.State.ONGOING)
             LOG.debug("Pre-condition action: %s", self.name)
             self.action.pre_condition()
         except Exception as e:
             LOG.exception(e)
-            self.engine.notify(self._db_action,
-                               obj_action.State.FAILED)
+            self.engine.notify(self._db_action, objects.action.State.FAILED)
             raise
 
     def execute(self, *args, **kwargs):
@@ -122,15 +120,13 @@ class TaskFlowActionContainer(task.Task):
             LOG.debug("Running action: %s", self.name)
 
             self.action.execute()
-            self.engine.notify(self._db_action,
-                               obj_action.State.SUCCEEDED)
+            self.engine.notify(self._db_action, objects.action.State.SUCCEEDED)
         except Exception as e:
             LOG.exception(e)
             LOG.error(_LE('The workflow engine has failed '
                           'to execute the action: %s'), self.name)
 
-            self.engine.notify(self._db_action,
-                               obj_action.State.FAILED)
+            self.engine.notify(self._db_action, objects.action.State.FAILED)
             raise
 
     def post_execute(self):
@@ -139,8 +135,7 @@ class TaskFlowActionContainer(task.Task):
             self.action.post_condition()
         except Exception as e:
             LOG.exception(e)
-            self.engine.notify(self._db_action,
-                               obj_action.State.FAILED)
+            self.engine.notify(self._db_action, objects.action.State.FAILED)
             raise
 
     def revert(self, *args, **kwargs):

watcher/cmd/api.py

@@ -24,19 +24,19 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service
+from watcher import conf
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
 
     host, port = cfg.CONF.api.host, cfg.CONF.api.port
     protocol = "http" if not CONF.api.enable_ssl_api else "https"
     # Build and start the WSGI app
-    server = service.WSGIService(
-        'watcher-api', CONF.api.enable_ssl_api)
+    server = service.WSGIService('watcher-api', CONF.api.enable_ssl_api)
 
     if host == '127.0.0.1':
         LOG.info(_LI('serving on 127.0.0.1:%(port)s, '

watcher/cmd/applier.py

@@ -20,19 +20,19 @@
 import os
 import sys
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.applier import manager
 from watcher.common import service as watcher_service
+from watcher import conf
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    watcher_service.prepare_service(sys.argv)
+    watcher_service.prepare_service(sys.argv, CONF)
 
     LOG.info(_LI('Starting Watcher Applier service in PID %s'), os.getpid())
 

watcher/cmd/dbmanage.py

@@ -24,10 +24,11 @@ import sys
 from oslo_config import cfg
 
 from watcher.common import service
+from watcher import conf
 from watcher.db import migration
 from watcher.db import purge
 
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 class DBCommand(object):
@@ -152,5 +153,5 @@ def main():
     if not set(sys.argv).intersection(valid_commands):
         sys.argv.append('upgrade')
 
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
     CONF.command.func()

watcher/cmd/decisionengine.py

@@ -20,21 +20,23 @@
 import os
 import sys
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as watcher_service
+from watcher import conf
+from watcher.decision_engine import gmr
 from watcher.decision_engine import manager
 from watcher.decision_engine import scheduling
 from watcher.decision_engine import sync
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    watcher_service.prepare_service(sys.argv)
+    watcher_service.prepare_service(sys.argv, CONF)
+    gmr.register_gmr_plugins()
 
     LOG.info(_LI('Starting Watcher Decision Engine service in PID %s'),
              os.getpid())

watcher/cmd/sync.py

@@ -24,15 +24,17 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as service
+from watcher import conf
 from watcher.decision_engine import sync
 
 LOG = logging.getLogger(__name__)
+CONF = conf.CONF
 
 
 def main():
     LOG.info(_LI('Watcher sync started.'))
 
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
     syncer = sync.Syncer()
     syncer.sync()
 

watcher/common/ceilometer_helper.py

@@ -134,27 +134,24 @@ class CeilometerHelper(object):
                               aggregate='avg'):
         """Representing a statistic aggregate by operators
 
-        :param resource_id: id
-        :param meter_name: meter names of which we want the statistics
-        :param period: `period`: In seconds. If no period is given, only one
-                       aggregate statistic is returned. If given, a faceted
-                       result will be returned, divided into given periods.
-                       Periods with no data are ignored.
-        :param aggregate:
-        :return:
+        :param resource_id: id of resource to list statistics for.
+        :param meter_name: Name of meter to list statistics for.
+        :param period: Period in seconds over which to group samples.
+        :param aggregate: Available aggregates are: count, cardinality,
+                           min, max, sum, stddev, avg. Defaults to avg.
+        :return: Return the latest statistical data, None if no data.
         """
 
-        start_time = (datetime.datetime.utcnow() -
-                      datetime.timedelta(seconds=int(period)))
+        end_time = datetime.datetime.utcnow()
+        start_time = end_time - datetime.timedelta(seconds=int(period))
         query = self.build_query(
-            resource_id=resource_id, start_time=start_time)
+            resource_id=resource_id, start_time=start_time, end_time=end_time)
         statistic = self.query_retry(f=self.ceilometer.statistics.list,
                                      meter_name=meter_name,
                                      q=query,
                                      period=period,
                                      aggregates=[
-                                         {'func': aggregate}],
-                                     groupby=['resource_id'])
+                                         {'func': aggregate}])
 
         item_value = None
         if statistic:

watcher/common/clients.py

@@ -19,47 +19,14 @@ from neutronclient.neutron import client as netclient
 from novaclient import client as nvclient
 from oslo_config import cfg
 
-from watcher._i18n import _
 from watcher.common import exception
 
+from watcher import conf
 
-NOVA_CLIENT_OPTS = [
-    cfg.StrOpt('api_version',
-               default='2',
-               help=_('Version of Nova API to use in novaclient.'))]
-
-GLANCE_CLIENT_OPTS = [
-    cfg.StrOpt('api_version',
-               default='2',
-               help=_('Version of Glance API to use in glanceclient.'))]
-
-CINDER_CLIENT_OPTS = [
-    cfg.StrOpt('api_version',
-               default='2',
-               help=_('Version of Cinder API to use in cinderclient.'))]
-
-CEILOMETER_CLIENT_OPTS = [
-    cfg.StrOpt('api_version',
-               default='2',
-               help=_('Version of Ceilometer API to use in '
-                      'ceilometerclient.'))]
-
-NEUTRON_CLIENT_OPTS = [
-    cfg.StrOpt('api_version',
-               default='2.0',
-               help=_('Version of Neutron API to use in neutronclient.'))]
-
-cfg.CONF.register_opts(NOVA_CLIENT_OPTS, group='nova_client')
-cfg.CONF.register_opts(GLANCE_CLIENT_OPTS, group='glance_client')
-cfg.CONF.register_opts(CINDER_CLIENT_OPTS, group='cinder_client')
-cfg.CONF.register_opts(CEILOMETER_CLIENT_OPTS, group='ceilometer_client')
-cfg.CONF.register_opts(NEUTRON_CLIENT_OPTS, group='neutron_client')
+CONF = conf.CONF
 
 _CLIENTS_AUTH_GROUP = 'watcher_clients_auth'
 
-ka_loading.register_auth_conf_options(cfg.CONF, _CLIENTS_AUTH_GROUP)
-ka_loading.register_session_conf_options(cfg.CONF, _CLIENTS_AUTH_GROUP)
-
 
 class OpenStackClients(object):
     """Convenience class to create and cache client instances."""

watcher/common/context.py

@@ -51,7 +51,7 @@ class RequestContext(context.RequestContext):
             show_deleted=kwargs.pop('show_deleted', False),
             request_id=request_id,
             resource_uuid=kwargs.pop('resource_uuid', None),
-            is_admin_project=kwargs.pop('is_admin_project', None),
+            is_admin_project=kwargs.pop('is_admin_project', True),
             overwrite=overwrite,
             roles=roles)
 

watcher/common/exception.py

@@ -26,22 +26,16 @@ import functools
 import sys
 
 from keystoneclient import exceptions as keystone_exceptions
-from oslo_config import cfg
 from oslo_log import log as logging
 import six
 
 from watcher._i18n import _, _LE
 
+from watcher import conf
+
 LOG = logging.getLogger(__name__)
 
-exc_log_opts = [
-    cfg.BoolOpt('fatal_exception_format_errors',
-                default=False,
-                help='Make exception message format errors fatal.'),
-]
-
-CONF = cfg.CONF
-CONF.register_opts(exc_log_opts)
+CONF = conf.CONF
 
 
 def wrap_keystone_exception(func):
@@ -119,6 +113,10 @@ class WatcherException(Exception):
             return six.text_type(self)
 
 
+class UnsupportedError(WatcherException):
+    msg_fmt = _("Not supported")
+
+
 class NotAuthorized(WatcherException):
     msg_fmt = _("Not authorized")
     code = 403
@@ -168,6 +166,14 @@ class InvalidStrategy(Invalid):
     msg_fmt = _("Strategy %(strategy)s is invalid")
 
 
+class InvalidAudit(Invalid):
+    msg_fmt = _("Audit %(audit)s is invalid")
+
+
+class EagerlyLoadedAuditRequired(InvalidAudit):
+    msg_fmt = _("Audit %(audit)s was not eagerly loaded")
+
+
 class InvalidUUID(Invalid):
     msg_fmt = _("Expected a uuid but received %(uuid)s")
 
@@ -207,7 +213,7 @@ class AuditTemplateAlreadyExists(Conflict):
 
 class AuditTemplateReferenced(Invalid):
     msg_fmt = _("AuditTemplate %(audit_template)s is referenced by one or "
-                "multiple audit")
+                "multiple audits")
 
 
 class AuditTypeNotFound(Invalid):
@@ -317,7 +323,7 @@ class AuthorizationFailure(WatcherException):
 
 
 class KeystoneFailure(WatcherException):
-    msg_fmt = _("'Keystone API endpoint is missing''")
+    msg_fmt = _("Keystone API endpoint is missing")
 
 
 class ClusterEmpty(WatcherException):
@@ -328,6 +334,10 @@ class MetricCollectorNotDefined(WatcherException):
     msg_fmt = _("The metrics resource collector is not defined")
 
 
+class ClusterStateStale(WatcherException):
+    msg_fmt = _("The cluster state is stale")
+
+
 class ClusterDataModelCollectionError(WatcherException):
     msg_fmt = _("The cluster data model '%(cdm)s' could not be built")
 
@@ -336,6 +346,10 @@ class ClusterStateNotDefined(WatcherException):
     msg_fmt = _("The cluster state is not defined")
 
 
+class CapacityNotDefined(WatcherException):
+    msg_fmt = _("The capacity %(capacity)s is not defined for '%(resource)s'")
+
+
 class NoAvailableStrategyForGoal(WatcherException):
     msg_fmt = _("No strategy could be found to achieve the '%(goal)s' goal.")
 
@@ -358,6 +372,19 @@ class NoSuchMetricForHost(WatcherException):
     msg_fmt = _("No %(metric)s metric for %(host)s found.")
 
 
+class ServiceAlreadyExists(Conflict):
+    msg_fmt = _("A service with name %(name)s is already working on %(host)s.")
+
+
+class ServiceNotFound(ResourceNotFound):
+    msg_fmt = _("The service %(service)s cannot be found.")
+
+
+class WildcardCharacterIsUsed(WatcherException):
+    msg_fmt = _("You shouldn't use any other IDs of %(resource)s if you use "
+                "wildcard character.")
+
+
 # Model
 
 class InstanceNotFound(WatcherException):
@@ -382,3 +409,8 @@ class NotSoftDeletedStateError(WatcherException):
 
 class NegativeLimitError(WatcherException):
     msg_fmt = _("Limit should be positive")
+
+
+class NotificationPayloadError(WatcherException):
+    _msg_fmt = _("Payload not populated when trying to send notification "
+                 "\"%(class_name)s\"")

watcher/common/messaging/events/event.py

@@ -1,54 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-class Event(object):
-    """Generic event to use with EventDispatcher"""
-
-    def __init__(self, event_type=None, data=None, request_id=None):
-        """Default constructor
-
-        :param event_type: the type of the event
-        :param data: a dictionary which contains data
-        :param request_id: a string which represent the uuid of the request
-        """
-        self._type = event_type
-        self._data = data
-        self._request_id = request_id
-
-    @property
-    def type(self):
-        return self._type
-
-    @type.setter
-    def type(self, type):
-        self._type = type
-
-    @property
-    def data(self):
-        return self._data
-
-    @data.setter
-    def data(self, data):
-        self._data = data
-
-    @property
-    def request_id(self):
-        return self._request_id
-
-    @request_id.setter
-    def request_id(self, id):
-        self._request_id = id

watcher/common/messaging/events/event_dispatcher.py

@@ -1,78 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from oslo_log import log
-
-from watcher.decision_engine.messaging import events as messaging_events
-
-LOG = log.getLogger(__name__)
-
-
-class EventDispatcher(object):
-    """Generic event dispatcher which listen and dispatch events"""
-
-    def __init__(self):
-        self._events = dict()
-
-    def __del__(self):
-        self._events = None
-
-    def has_listener(self, event_type, listener):
-        """Return true if listener is register to event_type """
-        # Check for event type and for the listener
-        if event_type in self._events.keys():
-            return listener in self._events[event_type]
-        else:
-            return False
-
-    def dispatch_event(self, event):
-        LOG.debug("dispatch evt : %s" % str(event.type))
-        """
-        Dispatch an instance of Event class
-        """
-        if messaging_events.Events.ALL in self._events.keys():
-            listeners = self._events[messaging_events.Events.ALL]
-            for listener in listeners:
-                listener(event)
-
-        # Dispatch the event to all the associated listeners
-        if event.type in self._events.keys():
-            listeners = self._events[event.type]
-            for listener in listeners:
-                listener(event)
-
-    def add_event_listener(self, event_type, listener):
-        """Add an event listener for an event type"""
-        # Add listener to the event type
-        if not self.has_listener(event_type, listener):
-            listeners = self._events.get(event_type, [])
-            listeners.append(listener)
-            self._events[event_type] = listeners
-
-    def remove_event_listener(self, event_type, listener):
-        """Remove event listener. """
-        # Remove the listener from the event type
-        if self.has_listener(event_type, listener):
-            listeners = self._events[event_type]
-
-            if len(listeners) == 1:
-                # Only this listener remains so remove the key
-                del self._events[event_type]
-
-            else:
-                # Update listeners chain
-                listeners.remove(listener)
-                self._events[event_type] = listeners

watcher/common/messaging/messaging_handler.py

@@ -1,120 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import socket
-import threading
-
-import eventlet
-from oslo_config import cfg
-from oslo_log import log
-import oslo_messaging as om
-
-from watcher.common import rpc
-from watcher._i18n import _LE, _LW
-
-# NOTE:
-# Ubuntu 14.04 forces librabbitmq when kombu is used
-# Unfortunately it forces a version that has a crash
-# bug.  Calling eventlet.monkey_patch() tells kombu
-# to use libamqp instead.
-eventlet.monkey_patch()
-
-LOG = log.getLogger(__name__)
-CONF = cfg.CONF
-
-
-class MessagingHandler(threading.Thread):
-
-    def __init__(self, publisher_id, topic_name, endpoints, version,
-                 serializer=None):
-        super(MessagingHandler, self).__init__()
-        self.publisher_id = publisher_id
-        self.topic_name = topic_name
-        self.__endpoints = []
-        self.__serializer = serializer
-        self.__version = version
-
-        self.__server = None
-        self.__notifier = None
-        self.__transport = None
-        self.add_endpoints(endpoints)
-
-    def add_endpoints(self, endpoints):
-        self.__endpoints.extend(endpoints)
-
-    def remove_endpoint(self, endpoint):
-        if endpoint in self.__endpoints:
-            self.__endpoints.remove(endpoint)
-
-    @property
-    def endpoints(self):
-        return self.__endpoints
-
-    @property
-    def transport(self):
-        return self.__transport
-
-    def build_notifier(self):
-        serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
-        return om.Notifier(
-            self.__transport,
-            publisher_id=self.publisher_id,
-            topic=self.topic_name,
-            serializer=serializer
-        )
-
-    def build_server(self, target):
-        return om.get_rpc_server(self.__transport, target,
-                                 self.__endpoints,
-                                 executor='eventlet',
-                                 serializer=self.__serializer)
-
-    def _configure(self):
-        try:
-            self.__transport = om.get_transport(CONF)
-            self.__notifier = self.build_notifier()
-            if len(self.__endpoints):
-                target = om.Target(
-                    topic=self.topic_name,
-                    # For compatibility, we can override it with 'host' opt
-                    server=CONF.host or socket.getfqdn(),
-                    version=self.__version,
-                )
-                self.__server = self.build_server(target)
-            else:
-                LOG.warning(
-                    _LW("No endpoint defined; can only publish events"))
-        except Exception as e:
-            LOG.exception(e)
-            LOG.error(_LE("Messaging configuration error"))
-
-    def run(self):
-        LOG.debug("configure MessagingHandler for %s" % self.topic_name)
-        self._configure()
-        if len(self.__endpoints) > 0:
-            LOG.debug("Starting up server")
-            self.__server.start()
-
-    def stop(self):
-        LOG.debug('Stopped server')
-        self.__server.stop()
-
-    def publish_event(self, event_type, payload, request_id=None):
-        self.__notifier.info(
-            {'version_api': self.__version,
-             'request_id': request_id},
-            {'event_id': event_type}, payload
-        )

watcher/common/messaging/notification_handler.py

@@ -1,47 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import eventlet
-import oslo_messaging as messaging
-
-from watcher.common.messaging.utils import observable
-
-
-eventlet.monkey_patch()
-
-
-class NotificationHandler(observable.Observable):
-    def __init__(self, publisher_id):
-        super(NotificationHandler, self).__init__()
-        self.publisher_id = publisher_id
-
-    def info(self, ctx, publisher_id, event_type, payload, metadata):
-        if publisher_id == self.publisher_id:
-            self.set_changed()
-            self.notify(ctx, publisher_id, event_type, metadata, payload)
-            return messaging.NotificationResult.HANDLED
-
-    def warn(self, ctx, publisher_id, event_type, payload, metadata):
-        if publisher_id == self.publisher_id:
-            self.set_changed()
-            self.notify(ctx, publisher_id, event_type, metadata, payload)
-            return messaging.NotificationResult.HANDLED
-
-    def error(self, ctx, publisher_id, event_type, payload, metadata):
-        if publisher_id == self.publisher_id:
-            self.set_changed()
-            self.notify(ctx, publisher_id, event_type, metadata, payload)
-            return messaging.NotificationResult.HANDLED

watcher/common/nova_helper.py

@@ -26,6 +26,8 @@ import cinderclient.exceptions as ciexceptions
 import novaclient.exceptions as nvexceptions
 
 from watcher.common import clients
+from watcher.common import exception
+from watcher.common import utils
 
 LOG = log.getLogger(__name__)
 
@@ -43,16 +45,35 @@ class NovaHelper(object):
     def get_compute_node_list(self):
         return self.nova.hypervisors.list()
 
+    def get_compute_node_by_id(self, node_id):
+        """Get compute node by ID (*not* UUID)"""
+        # We need to pass an object with an 'id' attribute to make it work
+        return self.nova.hypervisors.get(utils.Struct(id=node_id))
+
+    def get_compute_node_by_hostname(self, node_hostname):
+        """Get compute node by ID (*not* UUID)"""
+        # We need to pass an object with an 'id' attribute to make it work
+        try:
+            compute_nodes = self.nova.hypervisors.search(node_hostname)
+            if len(compute_nodes) != 1:
+                raise exception.ComputeNodeNotFound(name=node_hostname)
+
+            return self.get_compute_node_by_id(compute_nodes[0].id)
+        except Exception as exc:
+            LOG.exception(exc)
+            raise exception.ComputeNodeNotFound(name=node_hostname)
+
+    def get_aggregate_list(self):
+        return self.nova.aggregates.list()
+
+    def get_aggregate_detail(self, aggregate_id):
+        return self.nova.aggregates.get(aggregate_id)
+
+    def get_availability_zone_list(self):
+        return self.nova.availability_zones.list(detailed=True)
+
     def find_instance(self, instance_id):
-        search_opts = {'all_tenants': True}
-        instances = self.nova.servers.list(detailed=True,
-                                           search_opts=search_opts)
-        instance = None
-        for _instance in instances:
-            if _instance.id == instance_id:
-                instance = _instance
-                break
-        return instance
+        return self.nova.servers.get(instance_id)
 
     def wait_for_volume_status(self, volume, status, timeout=60,
                                poll_interval=1):
@@ -643,7 +664,7 @@ class NovaHelper(object):
             cache[fid] = flavor
         attr_defaults = [('name', 'unknown-id-%s' % fid),
                          ('vcpus', 0), ('ram', 0), ('disk', 0),
-                         ('ephemeral', 0)]
+                         ('ephemeral', 0), ('extra_specs', {})]
         for attr, default in attr_defaults:
             if not flavor:
                 instance.flavor[attr] = default

watcher/common/observable.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from watcher.common.messaging.utils import synchronization
+from watcher.common import synchronization
 
 
 class Observable(synchronization.Synchronization):

watcher/common/paths.py

@@ -17,38 +17,9 @@
 
 import os
 
-from oslo_config import cfg
+from watcher import conf
 
-PATH_OPTS = [
-    cfg.StrOpt('pybasedir',
-               default=os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                                    '../')),
-               help='Directory where the watcher python module is installed.'),
-    cfg.StrOpt('bindir',
-               default='$pybasedir/bin',
-               help='Directory where watcher binaries are installed.'),
-    cfg.StrOpt('state_path',
-               default='$pybasedir',
-               help="Top-level directory for maintaining watcher's state."),
-]
-
-CONF = cfg.CONF
-CONF.register_opts(PATH_OPTS)
-
-
-def basedir_def(*args):
-    """Return an uninterpolated path relative to $pybasedir."""
-    return os.path.join('$pybasedir', *args)
-
-
-def bindir_def(*args):
-    """Return an uninterpolated path relative to $bindir."""
-    return os.path.join('$bindir', *args)
-
-
-def state_path_def(*args):
-    """Return an uninterpolated path relative to $state_path."""
-    return os.path.join('$state_path', *args)
+CONF = conf.CONF
 
 
 def basedir_rel(*args):

watcher/common/rpc.py

@@ -72,8 +72,12 @@ def init(conf):
         aliases=TRANSPORT_ALIASES)
 
     serializer = RequestContextSerializer(JsonPayloadSerializer())
-    NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT,
-                                  serializer=serializer)
+    if not conf.notification_level:
+        NOTIFIER = messaging.Notifier(
+            NOTIFICATION_TRANSPORT, serializer=serializer, driver='noop')
+    else:
+        NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT,
+                                      serializer=serializer)
 
 
 def initialized():
@@ -121,7 +125,7 @@ class RequestContextSerializer(messaging.Serializer):
         return self._base.deserialize_entity(context, entity)
 
     def serialize_context(self, context):
-        return context
+        return context.to_dict()
 
     def deserialize_context(self, context):
         return watcher_context.RequestContext.from_dict(context)
@@ -146,8 +150,6 @@ def get_server(target, endpoints, serializer=None):
                                     serializer=serializer)
 
 
-def get_notifier(service=None, host=None, publisher_id=None):
+def get_notifier(publisher_id):
     assert NOTIFIER is not None
-    if not publisher_id:
-        publisher_id = "%s.%s" % (service, host or CONF.host)
     return NOTIFIER.prepare(publisher_id=publisher_id)

watcher/common/service.py

@@ -14,9 +14,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import logging
+import datetime
 import socket
 
+import eventlet
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import _options
@@ -27,30 +28,36 @@ from oslo_reports import opts as gmr_opts
 from oslo_service import service
 from oslo_service import wsgi
 
-from watcher._i18n import _, _LI
+from watcher._i18n import _
 from watcher.api import app
 from watcher.common import config
-from watcher.common.messaging.events import event_dispatcher as dispatcher
-from watcher.common.messaging import messaging_handler
+from watcher.common import context
 from watcher.common import rpc
+from watcher.common import scheduling
+from watcher.conf import plugins as plugins_conf
+from watcher import objects
 from watcher.objects import base
-from watcher import opts
+from watcher.objects import fields as wfields
 from watcher import version
 
-service_opts = [
-    cfg.IntOpt('periodic_interval',
-               default=60,
-               help=_('Seconds between running periodic tasks.')),
-    cfg.StrOpt('host',
-               default=socket.getfqdn(),
-               help=_('Name of this node. This can be an opaque identifier.  '
-                      'It is not necessarily a hostname, FQDN, or IP address. '
-                      'However, the node name must be valid within '
-                      'an AMQP key, and if using ZeroMQ, a valid '
-                      'hostname, FQDN, or IP address.')),
-]
+# NOTE:
+# Ubuntu 14.04 forces librabbitmq when kombu is used
+# Unfortunately it forces a version that has a crash
+# bug.  Calling eventlet.monkey_patch() tells kombu
+# to use libamqp instead.
+eventlet.monkey_patch()
+
+NOTIFICATION_OPTS = [
+    cfg.StrOpt('notification_level',
+               choices=[''] + list(wfields.NotificationPriority.ALL),
+               default=wfields.NotificationPriority.INFO,
+               help=_('Specifies the minimum level for which to send '
+                      'notifications. If not set, no notifications will '
+                      'be sent. The default is for this option to be at the '
+                      '`INFO` level.'))
+]
+cfg.CONF.register_opts(NOTIFICATION_OPTS)
 
-cfg.CONF.register_opts(service_opts)
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
@@ -68,21 +75,21 @@ Singleton = service.Singleton
 class WSGIService(service.ServiceBase):
     """Provides ability to launch Watcher API from wsgi app."""
 
-    def __init__(self, name, use_ssl=False):
+    def __init__(self, service_name, use_ssl=False):
         """Initialize, but do not start the WSGI server.
 
-        :param name: The name of the WSGI server given to the loader.
+        :param service_name: The service name of the WSGI server.
         :param use_ssl: Wraps the socket in an SSL context if True.
         """
-        self.name = name
+        self.service_name = service_name
         self.app = app.VersionSelectorApplication()
         self.workers = (CONF.api.workers or
                         processutils.get_worker_count())
-        self.server = wsgi.Server(CONF, name, self.app,
+        self.server = wsgi.Server(CONF, self.service_name, self.app,
                                   host=CONF.api.host,
                                   port=CONF.api.port,
                                   use_ssl=use_ssl,
-                                  logger_name=name)
+                                  logger_name=self.service_name)
 
     def start(self):
         """Start serving this service using loaded configuration"""
@@ -101,7 +108,53 @@ class WSGIService(service.ServiceBase):
         self.server.reset()
 
 
-class Service(service.ServiceBase, dispatcher.EventDispatcher):
+class ServiceHeartbeat(scheduling.BackgroundSchedulerService):
+
+    def __init__(self, gconfig=None, service_name=None, **kwargs):
+        gconfig = None or {}
+        super(ServiceHeartbeat, self).__init__(gconfig, **kwargs)
+        self.service_name = service_name
+        self.context = context.make_context()
+
+    def send_beat(self):
+        host = CONF.host
+        watcher_list = objects.Service.list(
+            self.context, filters={'name': self.service_name,
+                                   'host': host})
+        if watcher_list:
+            watcher_service = watcher_list[0]
+            watcher_service.last_seen_up = datetime.datetime.utcnow()
+            watcher_service.save()
+        else:
+            watcher_service = objects.Service(self.context)
+            watcher_service.name = self.service_name
+            watcher_service.host = host
+            watcher_service.create()
+
+    def add_heartbeat_job(self):
+        self.add_job(self.send_beat, 'interval', seconds=60,
+                     next_run_time=datetime.datetime.now())
+
+    def start(self):
+        """Start service."""
+        self.add_heartbeat_job()
+        super(ServiceHeartbeat, self).start()
+
+    def stop(self):
+        """Stop service."""
+        self.shutdown()
+
+    def wait(self):
+        """Wait for service to complete."""
+
+    def reset(self):
+        """Reset service.
+
+        Called in case service running in daemon mode receives SIGHUP.
+        """
+
+
+class Service(service.ServiceBase):
 
     API_VERSION = '1.0'
 
@@ -110,18 +163,14 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
         self.manager = manager_class()
 
         self.publisher_id = self.manager.publisher_id
-        self.api_version = self.manager.API_VERSION
+        self.api_version = self.manager.api_version
 
         self.conductor_topic = self.manager.conductor_topic
-        self.status_topic = self.manager.status_topic
         self.notification_topics = self.manager.notification_topics
 
         self.conductor_endpoints = [
             ep(self) for ep in self.manager.conductor_endpoints
         ]
-        self.status_endpoints = [
-            ep(self.publisher_id) for ep in self.manager.status_endpoints
-        ]
         self.notification_endpoints = self.manager.notification_endpoints
 
         self.serializer = rpc.RequestContextSerializer(
@@ -130,22 +179,23 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
         self._transport = None
         self._notification_transport = None
         self._conductor_client = None
-        self._status_client = None
 
         self.conductor_topic_handler = None
-        self.status_topic_handler = None
         self.notification_handler = None
 
+        self.heartbeat = None
+
         if self.conductor_topic and self.conductor_endpoints:
             self.conductor_topic_handler = self.build_topic_handler(
                 self.conductor_topic, self.conductor_endpoints)
-        if self.status_topic and self.status_endpoints:
-            self.status_topic_handler = self.build_topic_handler(
-                self.status_topic, self.status_endpoints)
         if self.notification_topics and self.notification_endpoints:
             self.notification_handler = self.build_notification_handler(
                 self.notification_topics, self.notification_endpoints
             )
+        self.service_name = self.manager.service_name
+        if self.service_name:
+            self.heartbeat = ServiceHeartbeat(
+                service_name=self.manager.service_name)
 
     @property
     def transport(self):
@@ -174,25 +224,17 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
     def conductor_client(self, c):
         self.conductor_client = c
 
-    @property
-    def status_client(self):
-        if self._status_client is None:
-            target = om.Target(
-                topic=self.status_topic,
-                version=self.API_VERSION,
-            )
-            self._status_client = om.RPCClient(
-                self.transport, target, serializer=self.serializer)
-        return self._status_client
-
-    @status_client.setter
-    def status_client(self, c):
-        self.status_client = c
-
     def build_topic_handler(self, topic_name, endpoints=()):
-        return messaging_handler.MessagingHandler(
-            self.publisher_id, topic_name, [self.manager] + list(endpoints),
-            self.api_version, self.serializer)
+        serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
+        target = om.Target(
+            topic=topic_name,
+            # For compatibility, we can override it with 'host' opt
+            server=CONF.host or socket.gethostname(),
+            version=self.api_version,
+        )
+        return om.get_rpc_server(
+            self.transport, target, endpoints,
+            executor='eventlet', serializer=serializer)
 
     def build_notification_handler(self, topic_names, endpoints=()):
         serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
@@ -207,20 +249,20 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
                   CONF.transport_url, CONF.rpc_backend)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.start()
-        if self.status_topic_handler:
-            self.status_topic_handler.start()
         if self.notification_handler:
             self.notification_handler.start()
+        if self.heartbeat:
+            self.heartbeat.start()
 
     def stop(self):
         LOG.debug("Disconnecting from '%s' (%s)",
                   CONF.transport_url, CONF.rpc_backend)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.stop()
-        if self.status_topic_handler:
-            self.status_topic_handler.stop()
         if self.notification_handler:
             self.notification_handler.stop()
+        if self.heartbeat:
+            self.heartbeat.stop()
 
     def reset(self):
         """Reset a service in case it received a SIGHUP."""
@@ -228,34 +270,11 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
     def wait(self):
         """Wait for service to complete."""
 
-    def publish_control(self, event, payload):
-        return self.conductor_topic_handler.publish_event(event, payload)
-
-    def publish_status_event(self, event, payload, request_id=None):
-        if self.status_topic_handler:
-            return self.status_topic_handler.publish_event(
-                event, payload, request_id)
-        else:
-            LOG.info(
-                _LI("No status notifier declared: notification '%s' not sent"),
-                event)
-
-    def get_version(self):
-        return self.api_version
-
-    def check_api_version(self, context):
+    def check_api_version(self, ctx):
         api_manager_version = self.conductor_client.call(
-            context.to_dict(), 'check_api_version',
-            api_version=self.api_version)
+            ctx, 'check_api_version', api_version=self.api_version)
         return api_manager_version
 
-    def response(self, evt, ctx, message):
-        payload = {
-            'request_id': ctx['request_id'],
-            'msg': message
-        }
-        self.publish_status_event(evt, payload)
-
 
 def launch(conf, service_, workers=1, restart_method='reload'):
     return service.launch(conf, service_, workers, restart_method)
@@ -269,7 +288,9 @@ def prepare_service(argv=(), conf=cfg.CONF):
     cfg.set_defaults(_options.log_opts,
                      default_log_levels=_DEFAULT_LOG_LEVELS)
     log.setup(conf, 'python-watcher')
-    conf.log_opt_values(LOG, logging.DEBUG)
+    conf.log_opt_values(LOG, log.DEBUG)
+    objects.register_all()
 
-    gmr.TextGuruMeditation.register_section(_('Plugins'), opts.show_plugins)
-    gmr.TextGuruMeditation.setup_autorun(version)
+    gmr.TextGuruMeditation.register_section(
+        _('Plugins'), plugins_conf.show_plugins)
+    gmr.TextGuruMeditation.setup_autorun(version, conf=conf)

watcher/common/service_manager.py

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2016 Servionica
+##
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ServiceManager(object):
+
+    @abc.abstractproperty
+    def service_name(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def api_version(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def publisher_id(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def conductor_topic(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def notification_topics(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def conductor_endpoints(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def notification_endpoints(self):
+        raise NotImplementedError()

watcher/common/utils.py

@@ -16,29 +16,21 @@
 
 """Utilities and helper functions."""
 
-from jsonschema import validators
-from oslo_config import cfg
-from oslo_log import log as logging
-from watcher.common import exception
-
 import re
-import six
-import uuid
 
+from jsonschema import validators
+from oslo_log import log as logging
+from oslo_utils import strutils
+from oslo_utils import timeutils
+from oslo_utils import uuidutils
+import six
 
 from watcher._i18n import _LW
+from watcher.common import exception
 
-UTILS_OPTS = [
-    cfg.StrOpt('rootwrap_config',
-               default="/etc/watcher/rootwrap.conf",
-               help='Path to the rootwrap configuration file to use for '
-                    'running commands as root.'),
-    cfg.StrOpt('tempdir',
-               help='Explicitly specify the temporary working directory.'),
-]
+from watcher import conf
 
-CONF = cfg.CONF
-CONF.register_opts(UTILS_OPTS)
+CONF = conf.CONF
 
 LOG = logging.getLogger(__name__)
 
@@ -66,6 +58,12 @@ class Struct(dict):
             raise AttributeError(name)
 
 
+generate_uuid = uuidutils.generate_uuid
+is_uuid_like = uuidutils.is_uuid_like
+is_int_like = strutils.is_int_like
+strtime = timeutils.strtime
+
+
 def safe_rstrip(value, chars=None):
     """Removes trailing characters from a string if that does not make it empty
 
@@ -83,31 +81,6 @@ def safe_rstrip(value, chars=None):
     return value.rstrip(chars) or value
 
 
-def generate_uuid():
-    return str(uuid.uuid4())
-
-
-def is_int_like(val):
-    """Check if a value looks like an int."""
-    try:
-        return str(int(val)) == str(val)
-    except Exception:
-        return False
-
-
-def is_uuid_like(val):
-    """Returns validation of a value as a UUID.
-
-    For our purposes, a UUID is a canonical form string:
-    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
-
-    """
-    try:
-        return str(uuid.UUID(val)) == val
-    except (TypeError, ValueError, AttributeError):
-        return False
-
-
 def is_hostname_safe(hostname):
     """Determine if the supplied hostname is RFC compliant.
 
@@ -133,10 +106,6 @@ def get_cls_import_path(cls):
     return module + '.' + cls.__name__
 
 
-def strtime(at):
-    return at.strftime("%Y-%m-%dT%H:%M:%S.%f")
-
-
 # Default value feedback extension as jsonschema doesn't support it
 def extend_with_default(validator_class):
     validate_properties = validator_class.VALIDATORS["properties"]
@@ -165,7 +134,7 @@ def extend_with_strict_schema(validator_class):
                 raise exception.AuditParameterNotAllowed(parameter=para)
 
             for error in validate_properties(
-                    validator, properties, instance, schema
+                validator, properties, instance, schema
             ):
                 yield error
 
@@ -173,3 +142,5 @@ def extend_with_strict_schema(validator_class):
 
 StrictDefaultValidatingDraft4Validator = extend_with_default(
     extend_with_strict_schema(validators.Draft4Validator))
+
+Draft4Validator = validators.Draft4Validator

watcher/conf/__init__.py

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+# Copyright (c) 2016 Intel Corp
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+from watcher.conf import api
+from watcher.conf import applier
+from watcher.conf import ceilometer_client
+from watcher.conf import cinder_client
+from watcher.conf import clients_auth
+from watcher.conf import db
+from watcher.conf import decision_engine
+from watcher.conf import exception
+from watcher.conf import glance_client
+from watcher.conf import neutron_client
+from watcher.conf import nova_client
+from watcher.conf import paths
+from watcher.conf import planner
+from watcher.conf import service
+from watcher.conf import utils
+
+CONF = cfg.CONF
+
+service.register_opts(CONF)
+api.register_opts(CONF)
+utils.register_opts(CONF)
+paths.register_opts(CONF)
+exception.register_opts(CONF)
+db.register_opts(CONF)
+planner.register_opts(CONF)
+applier.register_opts(CONF)
+decision_engine.register_opts(CONF)
+nova_client.register_opts(CONF)
+glance_client.register_opts(CONF)
+cinder_client.register_opts(CONF)
+ceilometer_client.register_opts(CONF)
+neutron_client.register_opts(CONF)
+clients_auth.register_opts(CONF)

watcher/conf/_opts.py

@@ -0,0 +1,60 @@
+# -*- encoding: utf-8 -*-
+# Copyright 2014
+# The Cloudscaling Group, Inc.
+# Copyright (c) 2016 Intel Corp
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneauth1 import loading as ka_loading
+
+from watcher.conf import api as conf_api
+from watcher.conf import applier as conf_applier
+from watcher.conf import ceilometer_client as conf_ceilometer_client
+from watcher.conf import cinder_client as conf_cinder_client
+from watcher.conf import db
+from watcher.conf import decision_engine as conf_de
+from watcher.conf import exception
+from watcher.conf import glance_client as conf_glance_client
+from watcher.conf import neutron_client as conf_neutron_client
+from watcher.conf import nova_client as conf_nova_client
+from watcher.conf import paths
+from watcher.conf import planner as conf_planner
+from watcher.conf import utils
+
+
+def list_opts():
+    """Legacy aggregation of all the watcher config options"""
+    return [
+        ('DEFAULT',
+         (conf_api.AUTH_OPTS +
+          exception.EXC_LOG_OPTS +
+          paths.PATH_OPTS +
+          utils.UTILS_OPTS)),
+        ('api', conf_api.API_SERVICE_OPTS),
+        ('database', db.SQL_OPTS),
+        ('watcher_planner', conf_planner.WATCHER_PLANNER_OPTS),
+        ('watcher_applier', conf_applier.APPLIER_MANAGER_OPTS),
+        ('watcher_decision_engine',
+         (conf_de.WATCHER_DECISION_ENGINE_OPTS +
+          conf_de.WATCHER_CONTINUOUS_OPTS)),
+        ('nova_client', conf_nova_client.NOVA_CLIENT_OPTS),
+        ('glance_client', conf_glance_client.GLANCE_CLIENT_OPTS),
+        ('cinder_client', conf_cinder_client.CINDER_CLIENT_OPTS),
+        ('ceilometer_client', conf_ceilometer_client.CEILOMETER_CLIENT_OPTS),
+        ('neutron_client', conf_neutron_client.NEUTRON_CLIENT_OPTS),
+        ('watcher_clients_auth',
+         (ka_loading.get_auth_common_conf_options() +
+          ka_loading.get_auth_plugin_conf_options('password') +
+          ka_loading.get_session_conf_options()))
+    ]

watcher/conf/api.py

@@ -0,0 +1,67 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 Intel Corp
+#
+# Authors: Prudhvi Rao Shedimbi <prudhvi.rao.shedimbi@intel.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+api = cfg.OptGroup(name='api',
+                   title='Options for the Watcher API service')
+
+AUTH_OPTS = [
+    cfg.BoolOpt('enable_authentication',
+                default=True,
+                help='This option enables or disables user authentication '
+                'via keystone. Default value is True.'),
+]
+
+API_SERVICE_OPTS = [
+    cfg.PortOpt('port',
+                default=9322,
+                help='The port for the watcher API server'),
+    cfg.StrOpt('host',
+               default='127.0.0.1',
+               help='The listen IP address for the watcher API server'),
+    cfg.IntOpt('max_limit',
+               default=1000,
+               help='The maximum number of items returned in a single '
+                    'response from a collection resource'),
+    cfg.IntOpt('workers',
+               min=1,
+               help='Number of workers for Watcher API service. '
+                    'The default is equal to the number of CPUs available '
+                    'if that can be determined, else a default worker '
+                    'count of 1 is returned.'),
+
+    cfg.BoolOpt('enable_ssl_api',
+                default=False,
+                help="Enable the integrated stand-alone API to service "
+                     "requests via HTTPS instead of HTTP. If there is a "
+                     "front-end service performing HTTPS offloading from "
+                     "the service, this option should be False; note, you "
+                     "will want to change public API endpoint to represent "
+                     "SSL termination URL with 'public_endpoint' option."),
+]
+
+
+def register_opts(conf):
+    conf.register_group(api)
+    conf.register_opts(API_SERVICE_OPTS, group=api)
+    conf.register_opts(AUTH_OPTS)
+
+
+def list_opts():
+    return [('api', API_SERVICE_OPTS), ('DEFAULT', AUTH_OPTS)]

watcher/conf/applier.py

@@ -0,0 +1,53 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 Intel Corp
+#
+# Authors: Prudhvi Rao Shedimbi <prudhvi.rao.shedimbi@intel.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+watcher_applier = cfg.OptGroup(name='watcher_applier',
+                               title='Options for the Applier messaging'
+                               'core')
+
+APPLIER_MANAGER_OPTS = [
+    cfg.IntOpt('workers',
+               default='1',
+               min=1,
+               required=True,
+               help='Number of workers for applier, default value is 1.'),
+    cfg.StrOpt('conductor_topic',
+               default='watcher.applier.control',
+               help='The topic name used for'
+                    'control events, this topic '
+                    'used for rpc call '),
+    cfg.StrOpt('publisher_id',
+               default='watcher.applier.api',
+               help='The identifier used by watcher '
+                    'module on the message broker'),
+    cfg.StrOpt('workflow_engine',
+               default='taskflow',
+               required=True,
+               help='Select the engine to use to execute the workflow')
+]
+
+
+def register_opts(conf):
+    conf.register_group(watcher_applier)
+    conf.register_opts(APPLIER_MANAGER_OPTS, group=watcher_applier)
+
+
+def list_opts():
+    return [('watcher_applier', APPLIER_MANAGER_OPTS)]