Merge "Added Tempest API tests for /scoring_engines"

In this changeset, I implemented the main logic although this is
mainly a shameful copy/paste of Nova's blueprint
https://blueprints.launchpad.net/nova/+spec/centralize-config-options

Partially Implements: blueprint centralise-config-opts

Change-Id: Ib645ad5da5c706336bb6ac37e85b027d05665c32

MANIFEST.in

@@ -1,6 +0,0 @@
-include AUTHORS
-include ChangeLog
-exclude .gitignore
-exclude .gitreview
-
-global-exclude *.pyc

README.rst

@@ -10,10 +10,8 @@ Watcher
 
 OpenStack Watcher provides a flexible and scalable resource optimization
 service for multi-tenant OpenStack-based clouds.
-Watcher provides a complete optimization loop-including everything from a
-metrics receiver, complex event processor and profiler, optimization processor
-and an action plan applier. This provides a robust framework to realize a wide
-range of cloud optimization goals, including the reduction of data center
+Watcher provides a robust framework to realize a wide range of cloud
+optimization goals, including the reduction of data center
 operating costs, increased system performance via intelligent virtual machine
 migration, increased energy efficiency-and more!
 

devstack/lib/watcher

@@ -126,6 +126,8 @@ function create_watcher_conf {
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
 
+    iniset $WATCHER_CONF oslo_messaging_notifications driver "messaging"
+
     iniset $NOVA_CONF oslo_messaging_notifications topics "notifications,watcher_notifications"
 
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR

devstack/local.conf.controller

@@ -28,7 +28,7 @@ ENABLED_SERVICES+=,q-svc,q-dhcp,q-meta,q-agt,q-l3,neutron
 enable_service n-cauth
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+# enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
 enable_plugin watcher git://git.openstack.org/openstack/watcher

doc/ext/term.py

@@ -75,7 +75,7 @@ class WatcherTerm(BaseWatcherDirective):
     # Inside your .rst file
     .. watcher-term:: import.path.to.your.DocumentedObject
 
-    This directive will then import the docstring and then interprete it.
+    This directive will then import the docstring and then interpret it.
     """
 
     # You need to put an import path as an argument for this directive to work
@@ -85,7 +85,12 @@ class WatcherTerm(BaseWatcherDirective):
         cls_path = self.arguments[0]
 
         try:
-            cls = importlib.import_module(cls_path)
+            try:
+                cls = importlib.import_module(cls_path)
+            except ImportError:
+                module_name, cls_name = cls_path.rsplit('.', 1)
+                mod = importlib.import_module(module_name)
+                cls = getattr(mod, cls_name)
         except Exception as exc:
             raise self.error(exc)
 
@@ -97,6 +102,72 @@ class WatcherTerm(BaseWatcherDirective):
         return node.children
 
 
+class WatcherFunc(BaseWatcherDirective):
+    """Directive to import a value returned by a func into the Watcher doc
+
+    **How to use it**
+
+    # inside your .py file
+    class Bar(object):
+
+        def foo(object):
+            return foo_string
+
+
+    # Inside your .rst file
+    .. watcher-func:: import.path.to.your.Bar.foo node_classname
+
+    node_classname is decumented here:
+    http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html
+
+    This directive will then import the value and then interpret it.
+    """
+
+    # You need to put an import path as an argument for this directive to work
+    # required_arguments = 1
+    # optional_arguments = 1
+
+    option_spec = {'format': rst.directives.unchanged}
+    has_content = True
+
+    def run(self):
+        if not self.content:
+            error = self.state_machine.reporter.error(
+                'The "%s" directive is empty; content required.' % self.name,
+                nodes.literal_block(self.block_text, self.block_text),
+                line=self.lineno)
+            return [error]
+
+        func_path = self.content[0]
+        try:
+            cls_path, func_name = func_path.rsplit('.', 1)
+            module_name, cls_name = cls_path.rsplit('.', 1)
+            mod = importlib.import_module(module_name)
+            cls = getattr(mod, cls_name)
+        except Exception as exc:
+            raise self.error(exc)
+
+        cls_obj = cls()
+        func = getattr(cls_obj, func_name)
+        textblock = func()
+        if not isinstance(textblock, str):
+            textblock = str(textblock)
+
+        self.add_textblock(textblock)
+
+        try:
+            node_class = getattr(nodes,
+                                 self.options.get('format', 'paragraph'))
+        except Exception as exc:
+            raise self.error(exc)
+
+        node = node_class()
+        node.document = self.state.document
+        self.state.nested_parse(self.result, 0, node)
+        return [node]
+
+
 def setup(app):
     app.add_directive('watcher-term', WatcherTerm)
+    app.add_directive('watcher-func', WatcherFunc)
     return {'version': version_info.version_string()}

doc/ext/versioned_notifications.py

@@ -0,0 +1,133 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+This provides a sphinx extension able to list the implemented versioned
+notifications into the developer documentation.
+
+It is used via a single directive in the .rst file
+
+  .. versioned_notifications::
+
+"""
+
+from sphinx.util.compat import Directive
+from docutils import nodes
+
+from watcher.notifications import base as notification
+from watcher.objects import base
+
+
+class VersionedNotificationDirective(Directive):
+
+    SAMPLE_ROOT = 'doc/notification_samples/'
+    TOGGLE_SCRIPT = """
+<script>
+jQuery(document).ready(function(){
+    jQuery('#%s-div').toggle('show');
+    jQuery('#%s-hideshow').on('click', function(event) {
+        jQuery('#%s-div').toggle('show');
+    });
+});
+</script>
+"""
+
+    def run(self):
+        notifications = self._collect_notifications()
+        return self._build_markup(notifications)
+
+    def _collect_notifications(self):
+        base.WatcherObjectRegistry.register_notification_objects()
+        notifications = []
+        ovos = base.WatcherObjectRegistry.obj_classes()
+        for name, cls in ovos.items():
+            cls = cls[0]
+            if (issubclass(cls, notification.NotificationBase) and
+                    cls != notification.NotificationBase):
+
+                payload_name = cls.fields['payload'].objname
+                payload_cls = ovos[payload_name][0]
+                for sample in cls.samples:
+                    notifications.append((cls.__name__,
+                                          payload_cls.__name__,
+                                          sample))
+        return sorted(notifications)
+
+    def _build_markup(self, notifications):
+        content = []
+        cols = ['Event type', 'Notification class', 'Payload class', 'Sample']
+        table = nodes.table()
+        content.append(table)
+        group = nodes.tgroup(cols=len(cols))
+        table.append(group)
+
+        head = nodes.thead()
+        group.append(head)
+
+        for _ in cols:
+            group.append(nodes.colspec(colwidth=1))
+
+        body = nodes.tbody()
+        group.append(body)
+
+        # fill the table header
+        row = nodes.row()
+        body.append(row)
+        for col_name in cols:
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.strong(text=col_name)
+            col.append(text)
+
+        # fill the table content, one notification per row
+        for name, payload, sample_file in notifications:
+            event_type = sample_file[0: -5].replace('-', '.')
+
+            row = nodes.row()
+            body.append(row)
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=event_type)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=name)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+            text = nodes.literal(text=payload)
+            col.append(text)
+
+            col = nodes.entry()
+            row.append(col)
+
+            with open(self.SAMPLE_ROOT + sample_file, 'r') as f:
+                sample_content = f.read()
+
+            event_type = sample_file[0: -5]
+            html_str = self.TOGGLE_SCRIPT % ((event_type, ) * 3)
+            html_str += ("<input type='button' id='%s-hideshow' "
+                         "value='hide/show sample'>" % event_type)
+            html_str += ("<div id='%s-div'><pre>%s</pre></div>"
+                         % (event_type, sample_content))
+
+            raw = nodes.raw('', html_str, format="html")
+            col.append(raw)
+
+        return content
+
+
+def setup(app):
+    app.add_directive('versioned_notifications',
+                      VersionedNotificationDirective)

doc/notification_samples/audit-create.json

@@ -0,0 +1,69 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "PENDING",
+      "updated_at": null,
+      "deleted_at": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditCreatePayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.create",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-delete.json

@@ -0,0 +1,69 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "DELETED",
+      "updated_at": null,
+      "deleted_at": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditDeletePayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.delete",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-end.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.end",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-error.json

@@ -0,0 +1,80 @@
+{
+  "priority": "ERROR",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": {
+        "watcher_object.data": {
+          "exception": "WatcherException",
+          "exception_message": "TEST",
+          "function_name": "test_send_audit_action_with_error",
+          "module_name": "watcher.tests.notifications.test_audit_notification"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.error",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-planner-start.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.planner.start",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-end.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.end",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-error.json

@@ -0,0 +1,80 @@
+{
+  "priority": "ERROR",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": {
+        "watcher_object.data": {
+          "exception": "WatcherException",
+          "exception_message": "TEST",
+          "function_name": "test_send_audit_action_with_error",
+          "module_name": "watcher.tests.notifications.test_audit_notification"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.error",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-strategy-start.json

@@ -0,0 +1,70 @@
+{
+  "priority": "INFO",
+  "payload": {
+    "watcher_object.data": {
+      "audit_type": "ONESHOT",
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "state": "ONGOING",
+      "updated_at": null,
+      "deleted_at": null,
+      "fault": null,
+      "goal": {
+        "watcher_object.data": {
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "name": "dummy",
+          "updated_at": null,
+          "deleted_at": null,
+          "efficacy_specification": [],
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy goal"
+        },
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "interval": null,
+      "scope": [],
+      "strategy": {
+        "watcher_object.data": {
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "type": "string",
+                "default": "hello",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "description": "number parameter example",
+                "maximum": 10.2,
+                "type": "number",
+                "default": 3.2,
+                "minimum": 1.0
+              }
+            }
+          },
+          "name": "dummy",
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "updated_at": null,
+          "deleted_at": null,
+          "created_at": "2016-11-04T16:25:35Z",
+          "display_name": "Dummy strategy"
+        },
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.version": "1.0",
+        "watcher_object.namespace": "watcher"
+      },
+      "created_at": "2016-11-04T16:29:20Z",
+      "uuid": "4a97b9dd-2023-43dc-b713-815bdd94d4d6"
+    },
+    "watcher_object.name": "AuditActionPayload",
+    "watcher_object.version": "1.0",
+    "watcher_object.namespace": "watcher"
+  },
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:31:36.264673   ",
+  "event_type": "audit.strategy.start",
+  "message_id": "cbcf9f2c-7c53-4b4d-91ec-db49cca024b6"
+}

doc/notification_samples/audit-update.json

@@ -0,0 +1,78 @@
+{
+  "publisher_id": "infra-optim:localhost",
+  "timestamp": "2016-11-04 16:51:38.722986   ",
+  "payload": {
+    "watcher_object.name": "AuditUpdatePayload",
+    "watcher_object.data": {
+      "strategy": {
+        "watcher_object.name": "StrategyPayload",
+        "watcher_object.data": {
+          "name": "dummy",
+          "parameters_spec": {
+            "properties": {
+              "para2": {
+                "default": "hello",
+                "type": "string",
+                "description": "string parameter example"
+              },
+              "para1": {
+                "maximum": 10.2,
+                "default": 3.2,
+                "minimum": 1.0,
+                "description": "number parameter example",
+                "type": "number"
+              }
+            }
+          },
+          "updated_at": null,
+          "display_name": "Dummy strategy",
+          "deleted_at": null,
+          "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39",
+          "created_at": "2016-11-04T16:25:35Z"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "scope": [],
+      "created_at": "2016-11-04T16:51:21Z",
+      "uuid": "f1e0d912-afd9-4bf2-91ef-c99cd08cc1ef",
+      "goal": {
+        "watcher_object.name": "GoalPayload",
+        "watcher_object.data": {
+          "efficacy_specification": [],
+          "updated_at": null,
+          "name": "dummy",
+          "display_name": "Dummy goal",
+          "deleted_at": null,
+          "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a",
+          "created_at": "2016-11-04T16:25:35Z"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "parameters": {
+        "para2": "hello",
+        "para1": 3.2
+      },
+      "deleted_at": null,
+      "state_update": {
+        "watcher_object.name": "AuditStateUpdatePayload",
+        "watcher_object.data": {
+          "state": "ONGOING",
+          "old_state": "PENDING"
+        },
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+      },
+      "interval": null,
+      "updated_at": null,
+      "state": "ONGOING",
+      "audit_type": "ONESHOT"
+    },
+    "watcher_object.namespace": "watcher",
+    "watcher_object.version": "1.0"
+  },
+  "priority": "INFO",
+  "event_type": "audit.update",
+  "message_id": "697fdf55-7252-4b6c-a2c2-5b9e85f6342c"
+}

doc/notification_samples/infra-optim-exception.json

@@ -0,0 +1,16 @@
+{
+    "event_type": "infra-optim.exception",
+    "payload": {
+        "watcher_object.data": {
+            "exception": "NoAvailableStrategyForGoal",
+            "exception_message": "No strategy could be found to achieve the server_consolidation goal.",
+            "function_name": "_aggregate_create_in_db",
+            "module_name": "watcher.objects.aggregate"
+        },
+        "watcher_object.name": "ExceptionPayload",
+        "watcher_object.namespace": "watcher",
+        "watcher_object.version": "1.0"
+    },
+    "priority": "ERROR",
+    "publisher_id": "watcher-api:fake-mini"
+}

doc/source/architecture.rst

@@ -171,12 +171,12 @@ This component is responsible for computing a set of potential optimization
 :ref:`Actions <action_definition>` in order to fulfill
 the :ref:`Goal <goal_definition>` of an :ref:`Audit <audit_definition>`.
 
-It first reads the parameters of the :ref:`Audit <audit_definition>` from the
-associated :ref:`Audit Template <audit_template_definition>` and knows the
-:ref:`Goal <goal_definition>` to achieve.
+It first reads the parameters of the :ref:`Audit <audit_definition>` to know
+the :ref:`Goal <goal_definition>` to achieve.
 
-It then selects the most appropriate :ref:`Strategy <strategy_definition>`
-from the list of available strategies achieving this goal.
+Unless specified, it then selects the most appropriate :ref:`strategy
+<strategy_definition>` from the list of available strategies achieving this
+goal.
 
 The :ref:`Strategy <strategy_definition>` is then dynamically loaded (via
 `stevedore <http://docs.openstack.org/developer/stevedore/>`_). The
@@ -290,7 +290,7 @@ the Audit parameters from the
 :ref:`Watcher Database <watcher_database_definition>`. It instantiates the
 appropriate :ref:`strategy <strategy_definition>` (using entry points)
 given both the :ref:`goal <goal_definition>` and the strategy associated to the
-parent :ref:`audit template <audit_template_definition>` of the :ref:`Audit
+parent :ref:`audit template <audit_template_definition>` of the :ref:`audit
 <audit_definition>`. If no strategy is associated to the audit template, the
 strategy is dynamically selected by the Decision Engine.
 
@@ -298,7 +298,7 @@ The :ref:`Watcher Decision Engine <watcher_decision_engine_definition>` also
 builds the :ref:`Cluster Data Model <cluster_data_model_definition>`. This
 data model is needed by the :ref:`Strategy <strategy_definition>` to know the
 current state and topology of the audited
-:ref:`Openstack cluster <cluster_definition>`.
+:ref:`OpenStack cluster <cluster_definition>`.
 
 The :ref:`Watcher Decision Engine <watcher_decision_engine_definition>` calls
 the **execute()** method of the instantiated
@@ -450,7 +450,7 @@ state may be one of the following:
    stored in the :ref:`Watcher database <watcher_database_definition>` but is
    not returned any more through the Watcher APIs.
 -  **CANCELLED** : the :ref:`Action Plan <action_plan_definition>` was in
-   **PENDING** or **ONGOING** state and was cancelled by the
+   **RECOMMENDED**, **PENDING** or **ONGOING** state and was cancelled by the
    :ref:`Administrator <administrator_definition>`
 
 

doc/source/conf.py

@@ -11,7 +11,21 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+import sys
+
 from watcher import version as watcher_version
+from watcher import objects
+
+objects.register_all()
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../'))
+sys.path.insert(0, os.path.abspath('./'))
 
 # -- General configuration ----------------------------------------------------
 
@@ -26,7 +40,8 @@ extensions = [
     'sphinxcontrib.pecanwsme.rest',
     'stevedore.sphinxext',
     'wsmeext.sphinxext',
-    'watcher.doc',
+    'ext.term',
+    'ext.versioned_notifications',
 ]
 
 wsme_protocols = ['restjson']
@@ -67,6 +82,8 @@ exclude_patterns = [
     # them when scanning for input files.
     'man/footer.rst',
     'man/general-options.rst',
+    'strategies/strategy-template.rst',
+    'image_src/plantuml/README.rst',
 ]
 
 # If true, '()' will be appended to :func: etc. cross-reference text.

doc/source/deploy/user-guide.rst

@@ -93,11 +93,11 @@ following command:
 
 .. code:: bash
 
-  $ watcher strategy list --goal-uuid <your_goal_uuid>
+  $ watcher strategy list --goal <your_goal_uuid_or_name>
 
 or::
 
-  $ openstack optimize strategy list --goal-uuid <your_goal_uuid>
+  $ openstack optimize strategy list --goal <your_goal_uuid_or_name>
 
 You can use the following command to check strategy details including which
 parameters of which format it supports:

doc/source/dev/notifications.rst

@@ -0,0 +1,13 @@
+..
+      Except where otherwise noted, this document is licensed under Creative
+      Commons Attribution 3.0 License.  You can view the license at:
+
+          https://creativecommons.org/licenses/by/3.0/
+
+.. _watcher_notifications:
+
+========================
+Notifications in Watcher
+========================
+
+.. versioned_notifications::

doc/source/dev/plugin/cdmc-plugin.rst

@@ -22,7 +22,7 @@ cluster data model collectors within Watcher.
 Creating a new plugin
 =====================
 
-In order to create a new model, you have to:
+In order to create a new cluster data model collector, you have to:
 
 - Extend the :py:class:`~.base.BaseClusterDataModelCollector` class.
 - Implement its :py:meth:`~.BaseClusterDataModelCollector.execute` abstract
@@ -65,6 +65,49 @@ This implementation is the most basic one. So in order to get a better
 understanding on how to implement a more advanced cluster data model collector,
 have a look at the :py:class:`~.NovaClusterDataModelCollector` class.
 
+Define a custom model
+=====================
+
+As you may have noticed in the above example, we are reusing an existing model
+provided by Watcher. However, this model can be easily customized by
+implementing a new class that would implement the :py:class:`~.Model` abstract
+base class. Here below is simple example on how to proceed in implementing a
+custom Model:
+
+.. code-block:: python
+
+    # Filepath = <PROJECT_DIR>/thirdparty/dummy.py
+    # Import path = thirdparty.dummy
+
+    from watcher.decision_engine.model import base as modelbase
+    from watcher.decision_engine.model.collector import base
+
+
+    class MyModel(modelbase.Model):
+
+        def to_string(self):
+            return 'MyModel'
+
+
+    class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector):
+
+        def execute(self):
+            model = MyModel()
+            # Do something here...
+            return model
+
+        @property
+        def notification_endpoints(self):
+            return []
+
+Here below is the abstract ``Model`` class that every single cluster data model
+should implement:
+
+.. autoclass:: watcher.decision_engine.model.base.Model
+    :members:
+    :special-members: __init__
+    :noindex:
+
 Define configuration parameters
 ===============================
 

doc/source/dev/plugin/strategy-plugin.rst

@@ -43,7 +43,7 @@ In order to create a new strategy, you have to:
   Note: Do not use a variable to return the translated string so it can be
   automatically collected by the translation tool.
 - Implement its :py:meth:`~.BaseStrategy.get_translatable_display_name`
-  class method to return the translation key (actually the english display
+  class method to return the translation key (actually the English display
   name) of your new strategy. The value return should be the same as the
   string translated in :py:meth:`~.BaseStrategy.get_display_name`.
 - Implement its :py:meth:`~.BaseStrategy.execute` method to return the

doc/source/glossary.rst

@@ -96,8 +96,8 @@ The :ref:`Cluster <cluster_definition>` may be divided in one or several
 
 .. _cluster_data_model_definition:
 
-Cluster Data Model
-==================
+Cluster Data Model (CDM)
+========================
 
 .. watcher-term:: watcher.decision_engine.model.collector.base
 
@@ -164,7 +164,8 @@ Goal
 Host Aggregate
 ==============
 
-Please, read `the official OpenStack definition of a Host Aggregate <http://docs.openstack.org/developer/nova/aggregates.html>`_.
+Please, read `the official OpenStack definition of a Host Aggregate
+<http://docs.openstack.org/developer/nova/aggregates.html>`_.
 
 .. _instance_definition:
 

doc/source/image_src/plantuml/sequence_overview_watcher_usage.txt

@@ -4,7 +4,7 @@ actor Administrator
 
 == Create some Audit settings ==
 
-Administrator -> Watcher : create new Audit Template (i.e. Audit settings : goal, scope, deadline,...)
+Administrator -> Watcher : create new Audit Template (i.e. Audit settings : goal, scope, ...)
 Watcher -> Watcher : save Audit Template in database
 Administrator <-- Watcher : Audit Template UUID
 

doc/source/image_src/plantuml/watcher_db_schema_diagram.txt

@@ -5,11 +5,12 @@
 hide methods
 hide stereotypes
 
-table(goal) {
+table(goals) {
     primary_key(id: Integer)
     uuid : String[36]
     name : String[63]
     display_name : String[63]
+    efficacy_specification : JSONEncodedList, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -18,12 +19,13 @@ table(goal) {
 }
 
 
-table(strategy) {
+table(strategies) {
     primary_key(id: Integer)
     foreign_key(goal_id : Integer)
     uuid : String[36]
     name : String[63]
     display_name : String[63]
+    parameters_spec : JSONEncodedDict, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -32,16 +34,14 @@ table(strategy) {
 }
 
 
-table(audit_template) {
+table(audit_templates) {
     primary_key(id: Integer)
     foreign_key("goal_id : Integer")
     foreign_key("strategy_id : Integer, nullable")
     uuid : String[36]
     name : String[63], nullable
     description : String[255], nullable
-    host_aggregate : Integer, nullable
-    extra : JSONEncodedDict
-    version : String[15], nullable
+    scope : JSONEncodedList
 
     created_at : DateTime
     updated_at : DateTime
@@ -50,14 +50,16 @@ table(audit_template) {
 }
 
 
-table(audit) {
+table(audits) {
     primary_key(id: Integer)
-    foreign_key("audit_template_id : Integer")
+    foreign_key("goal_id : Integer")
+    foreign_key("strategy_id : Integer, nullable")
     uuid : String[36]
     audit_type : String[20]
     state : String[20], nullable
-    deadline  :DateTime, nullable
     interval : Integer, nullable
+    parameters : JSONEncodedDict, nullable
+    scope : JSONEncodedList, nullable
 
     created_at : DateTime
     updated_at : DateTime
@@ -66,9 +68,10 @@ table(audit) {
 }
 
 
-table(action_plan) {
+table(action_plans) {
     primary_key(id: Integer)
     foreign_key("audit_id : Integer, nullable")
+    foreign_key("strategy_id : Integer")
     uuid : String[36]
     first_action_id : Integer
     state : String[20], nullable
@@ -81,7 +84,7 @@ table(action_plan) {
 }
 
 
-table(action) {
+table(actions) {
     primary_key(id: Integer)
     foreign_key("action_plan_id : Integer")
     uuid : String[36]
@@ -97,7 +100,7 @@ table(action) {
 }
 
 
-table(efficacy_indicator) {
+table(efficacy_indicators) {
     primary_key(id: Integer)
     foreign_key("action_plan_id : Integer")
     uuid : String[36]
@@ -112,12 +115,39 @@ table(efficacy_indicator) {
     deleted : Integer
 }
 
- "goal" <.. "strategy" : Foreign Key
- "goal" <.. "audit_template" : Foreign Key
- "strategy" <.. "audit_template" : Foreign Key
- "audit_template" <.. "audit" : Foreign Key
- "action_plan" <.. "action" : Foreign Key
- "action_plan" <.. "efficacy_indicator" : Foreign Key
- "audit" <.. "action_plan" : Foreign Key
+table(scoring_engines) {
+    primary_key(id: Integer)
+    uuid : String[36]
+    name : String[63]
+    description : String[255], nullable
+    metainfo : Text, nullable
+
+    created_at : DateTime
+    updated_at : DateTime
+    deleted_at : DateTime
+    deleted : Integer
+}
+
+table(service) {
+    primary_key(id: Integer)
+    name: String[255]
+    host: String[255]
+    last_seen_up: DateTime
+
+    created_at : DateTime
+    updated_at : DateTime
+    deleted_at : DateTime
+    deleted : Integer
+}
+
+ "goals" <.. "strategies" : Foreign Key
+ "goals" <.. "audit_templates" : Foreign Key
+ "strategies" <.. "audit_templates" : Foreign Key
+ "goals" <.. "audits" : Foreign Key
+ "strategies" <.. "audits" : Foreign Key
+ "action_plans" <.. "actions" : Foreign Key
+ "action_plans" <.. "efficacy_indicators" : Foreign Key
+ "strategies" <.. "action_plans" : Foreign Key
+ "audits" <.. "action_plans" : Foreign Key
 
 @enduml

doc/source/index.rst

@@ -56,6 +56,7 @@ Getting Started
   dev/devstack
   deploy/configuration
   deploy/conf-files
+  dev/notifications
   dev/testing
   dev/rally_link
 
@@ -96,6 +97,7 @@ Introduction
   deploy/user-guide
   deploy/policy
   deploy/gmr
+  strategies/strategies
 
 Watcher Manual Pages
 ====================

doc/source/strategies/basic-server-consolidation.rst

@@ -0,0 +1,96 @@
+==================================
+Basic Offline Server Consolidation
+==================================
+
+Synopsis
+--------
+
+**display name**: ``basic``
+
+**goal**: ``server_consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.basic_consolidation
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *basic* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameter is:
+
+====================== ====== ============= ===================================
+parameter              type   default Value description
+====================== ====== ============= ===================================
+``migration_attempts`` Number 0             Maximum number of combinations to
+                                            be tried by the strategy while
+                                            searching for potential candidates.
+                                            To remove the limit, set it to 0
+====================== ====== ============= ===================================
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 server_consolidation --strategy basic
+
+    $ openstack optimize audit create -a at1 -p migration_attempts=4
+
+External Links
+--------------
+None.

doc/source/strategies/strategies.rst

@@ -0,0 +1,8 @@
+Strategies
+==========
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   ./*

doc/source/strategies/strategy-template.rst

@@ -0,0 +1,115 @@
+=============
+Strategy name
+=============
+
+Synopsis
+--------
+
+**display name**:
+
+**goal**:
+
+Add here a complete description of your strategy
+
+Requirements
+------------
+
+Metrics
+*******
+
+Write here the list of metrics required by your strategy algorithm (in the form
+ of a table). If these metrics requires specific Telemetry plugin or other
+ additional software, please explain here how to deploy them (and add link to
+ dedicated installation guide).
+
+Example:
+
+======================= ============ ======= =======
+metric                  service name plugins comment
+======================= ============ ======= =======
+compute.node.*          ceilometer_  none    one point every 60s
+vm.cpu.utilization_perc monasca_     none
+power                   ceilometer_  kwapi_  one point every 60s
+======================= ============ ======= =======
+
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+.. _monasca: https://github.com/openstack/monasca-agent/blob/master/docs/Libvirt.md
+.. _kwapi: https://kwapi.readthedocs.io/en/latest/index.html
+
+
+Cluster data model
+******************
+
+Default Watcher's cluster data model.
+
+or
+
+If your strategy implementation requires a new cluster data model, please
+ describe it in this section, with a link to model plugin's installation guide.
+
+Actions
+*******
+
+Default Watcher's actions.
+
+or
+
+If your strategy implementation requires new actions, add the list of Action
+ plugins here (in the form of a table) with a link to the plugin's installation
+ procedure.
+
+======== =================
+action   description
+======== =================
+action1_ This action1 ...
+action2_ This action2 ...
+======== =================
+
+.. _action1 : https://github.com/myrepo/watcher/plugins/action1
+.. _action2 : https://github.com/myrepo/watcher/plugins/action2
+
+Planner
+*******
+
+Default Watcher's planner.
+
+or
+
+If your strategy requires also a new planner to schedule built actions in time,
+ please describe it in this section, with a link to planner plugin's
+ installation guide.
+
+Configuration
+-------------
+
+If your strategy use configurable parameters, explain here how to tune them.
+
+
+Efficacy Indicator
+------------------
+
+Add here the Efficacy indicator computed by your strategy.
+
+Algorithm
+---------
+
+Add here either the description of your algorithm or
+link to the existing description.
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ Write the command line to create an audit with your strategy.
+
+External Links
+--------------
+
+If you have written papers, blog articles .... about your strategy into Watcher,
+ or if your strategy is based from external publication(s), please add HTTP
+ links and references in this section.
+
+- `link1 <http://www.link1.papers.com>`_
+- `link2 <http://www.link2.papers.com>`_

doc/source/strategies/vm_workload_consolidation.rst

@@ -0,0 +1,100 @@
+==================================
+VM Workload Consolidation Strategy
+==================================
+
+Synopsis
+--------
+
+**display name**: ``vm_workload_consolidation``
+
+**goal**: ``vm_consolidation``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.vm_workload_consolidation
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *vm_workload_consolidation* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory``             	     ceilometer_  none
+``disk.root.size``           ceilometer_  none
+============================ ============ ======= =======
+
+The following metrics are not required but increase the accuracy of
+the strategy if available:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``memory.usage``             ceilometer_  none
+``cpu_util``                 ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+       * - ``change_nova_service_state``
+         - .. watcher-term:: watcher.applier.actions.change_nova_service_state.ChangeNovaServiceState
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+Algorithm
+---------
+
+For more information on the VM Workload consolidation strategy please refer to: https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/zhaw-load-consolidation.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 vm_consolidation --strategy vm_workload_consolidation
+
+    $ openstack optimize audit create -a at1
+
+External Links
+--------------
+
+*Spec URL*
+https://specs.openstack.org/openstack/watcher-specs/specs/mitaka/implemented/zhaw-load-consolidation.html

doc/source/strategies/workload-stabilization.rst

@@ -0,0 +1,131 @@
+=============================================
+Watcher Overload standard deviation algorithm
+=============================================
+
+Synopsis
+--------
+
+**display name**: ``workload_stabilization``
+
+**goal**: ``workload_balancing``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.workload_stabilization
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *workload_stabilization* strategy requires the following metrics:
+
+============================ ============ ======= =======
+metric                       service name plugins comment
+============================ ============ ======= =======
+``compute.node.cpu.percent`` ceilometer_  none
+``hardware.memory.used``     ceilometer_  SNMP_
+``cpu_util``                 ceilometer_  none
+``memory.resident``          ceilometer_  none
+============================ ============ ======= =======
+
+.. _ceilometer: http://docs.openstack.org/admin-guide/telemetry-measurements.html#openstack-compute
+.. _SNMP: http://docs.openstack.org/admin-guide/telemetry-measurements.html
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.default.DefaultPlanner
+
+Configuration
+-------------
+
+Strategy parameters are:
+
+==================== ====== ===================== =============================
+parameter            type   default Value         description
+==================== ====== ===================== =============================
+``metrics``          array  |metrics|             Metrics used as rates of
+                                                  cluster loads.
+``thresholds``       object |thresholds|          Dict where key is a metric
+                                                  and value is a trigger value.
+
+``weights``          object |weights|             These weights used to
+                                                  calculate common standard
+                                                  deviation. Name of weight
+                                                  contains meter name and
+                                                  _weight suffix.
+``instance_metrics`` object |instance_metrics|    Mapping to get hardware
+                                                  statistics using instance
+                                                  metrics.
+``host_choice``      string retry                 Method of host's choice.
+                                                  There are cycle, retry and
+                                                  fullsearch methods. Cycle
+                                                  will iterate hosts in cycle.
+                                                  Retry will get some hosts
+                                                  random (count defined in
+                                                  retry_count option).
+                                                  Fullsearch will return each
+                                                  host from list.
+``retry_count``      number 1                     Count of random returned
+                                                  hosts.
+==================== ====== ===================== =============================
+
+.. |metrics| replace:: ["cpu_util", "memory.resident"]
+.. |thresholds| replace:: {"cpu_util": 0.2, "memory.resident": 0.2}
+.. |weights| replace:: {"cpu_util_weight": 1.0, "memory.resident_weight": 1.0}
+.. |instance_metrics| replace:: {"cpu_util": "hardware.cpu.util", "memory.resident": "hardware.memory.used"}
+
+Efficacy Indicator
+------------------
+
+.. watcher-func::
+  :format: literal_block
+
+  watcher.decision_engine.goal.efficacy.specs.ServerConsolidation.get_global_efficacy_indicator
+
+Algorithm
+---------
+
+You can find description of overload algorithm and role of standard deviation
+here: https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 workload_balancing --strategy workload_stabilization
+
+    $ openstack optimize audit create -a at1 \
+      -p thresholds='{"memory.resident": 0.05}' \
+      -p metrics='["memory.resident"]'
+
+External Links
+--------------
+
+- `Watcher Overload standard deviation algorithm spec <https://specs.openstack.org/openstack/watcher-specs/specs/newton/implemented/sd-strategy.html>`_

etc/watcher/policy.json

@@ -37,5 +37,9 @@
 
     "strategy:detail": "rule:default",
     "strategy:get": "rule:default",
-    "strategy:get_all": "rule:default"
+    "strategy:get_all": "rule:default",
+
+    "service:detail": "rule:default",
+    "service:get": "rule:default",
+    "service:get_all": "rule:default"
 }

releasenotes/source/conf.py

@@ -242,3 +242,6 @@ texinfo_documents = [
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
+
+# -- Options for Internationalization output ------------------------------
+locale_dirs = ['locale/']

releasenotes/source/index.rst

@@ -6,5 +6,6 @@ Contents:
 .. toctree::
    :maxdepth: 1
 
-   unreleased.rst
+   unreleased
+   newton
 

releasenotes/source/newton.rst

@@ -0,0 +1,6 @@
+===================================
+ Newton Series Release Notes
+===================================
+
+.. release-notes::
+   :branch: origin/stable/newton

requirements.txt

@@ -5,37 +5,38 @@
 apscheduler # MIT License
 enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
 jsonpatch>=1.1 # BSD
-keystoneauth1>=2.10.0 # Apache-2.0
-keystonemiddleware!=4.1.0,!=4.5.0,>=4.0.0 # Apache-2.0
+keystoneauth1>=2.14.0 # Apache-2.0
+keystonemiddleware!=4.5.0,>=4.2.0 # Apache-2.0
 lxml>=2.3 # BSD
 oslo.concurrency>=3.8.0 # Apache-2.0
 oslo.cache>=1.5.0 # Apache-2.0
-oslo.config>=3.14.0 # Apache-2.0
+oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
 oslo.context>=2.9.0 # Apache-2.0
-oslo.db>=4.10.0 # Apache-2.0
+oslo.db!=4.13.1,!=4.13.2,>=4.11.0 # Apache-2.0
 oslo.i18n>=2.1.0 # Apache-2.0
-oslo.log>=1.14.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
 oslo.messaging>=5.2.0 # Apache-2.0
-oslo.policy>=1.9.0 # Apache-2.0
+oslo.policy>=1.15.0 # Apache-2.0
 oslo.reports>=0.6.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
 oslo.service>=1.10.0 # Apache-2.0
-oslo.utils>=3.16.0 # Apache-2.0
+oslo.utils>=3.18.0 # Apache-2.0
+oslo.versionedobjects>=1.13.0 # Apache-2.0
 PasteDeploy>=1.5.0 # MIT
-pbr>=1.6 # Apache-2.0
-pecan!=1.0.2,!=1.0.3,!=1.0.4,>=1.0.0 # BSD
-PrettyTable<0.8,>=0.7 # BSD
+pbr>=1.8 # Apache-2.0
+pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
+PrettyTable<0.8,>=0.7.1 # BSD
 voluptuous>=0.8.9 # BSD License
 python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0
-python-glanceclient!=2.4.0,>=2.3.0 # Apache-2.0
-python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
+python-glanceclient>=2.5.0 # Apache-2.0
+python-keystoneclient>=3.6.0 # Apache-2.0
 python-neutronclient>=5.1.0 # Apache-2.0
 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
-python-openstackclient>=2.1.0 # Apache-2.0
+python-openstackclient>=3.3.0 # Apache-2.0
 six>=1.9.0 # MIT
 SQLAlchemy<1.1.0,>=1.0.10 # MIT
-stevedore>=1.16.0 # Apache-2.0
+stevedore>=1.17.1 # Apache-2.0
 taskflow>=1.26.0 # Apache-2.0
-WebOb>=1.2.3 # MIT
+WebOb>=1.6.0 # MIT
 WSME>=0.8 # MIT

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = python-watcher
-summary = Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physical resources usage through better VM placement. Watcher can improve your cloud optimization by reducing energy footprint and increasing profits.
+summary = OpenStack Watcher provides a flexible and scalable resource optimization service for multi-tenant OpenStack-based clouds.
 description-file =
     README.rst
 author = OpenStack
@@ -32,7 +32,7 @@ setup-hooks =
 
 [entry_points]
 oslo.config.opts =
-    watcher = watcher.opts:list_opts
+    watcher = watcher.conf.opts:list_opts
 
 console_scripts =
     watcher-api = watcher.cmd.api:main

test-requirements.txt

@@ -2,25 +2,25 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-coverage>=3.6 # Apache-2.0
+coverage>=4.0 # Apache-2.0
 doc8 # Apache-2.0
-freezegun # Apache-2.0
+freezegun>=0.3.6 # Apache-2.0
 hacking<0.11,>=0.10.2
 mock>=2.0 # BSD
 oslotest>=1.10.0 # Apache-2.0
-os-testr>=0.7.0 # Apache-2.0
+os-testr>=0.8.0 # Apache-2.0
 python-subunit>=0.0.18 # Apache-2.0/BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
 
 # Doc requirements
-oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
-sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+oslosphinx>=4.7.0 # Apache-2.0
+sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
 sphinxcontrib-pecanwsme>=0.8 # Apache-2.0
 
 # releasenotes
-reno>=1.8.0 # Apache2
+reno>=1.8.0 # Apache-2.0
 
 # bandit
 bandit>=1.1.0 # Apache-2.0

tox.ini

@@ -1,26 +1,26 @@
 [tox]
-minversion = 1.6
+minversion = 1.8
 envlist = py35,py34,py27,pep8
 skipsdist = True
 
 [testenv]
 usedevelop = True
 whitelist_externals = find
-install_command = pip install -U {opts} {packages}
+install_command =
+                  constraints: pip install -U --force-reinstall -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
+                  pip install -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
+deps = -r{toxinidir}/test-requirements.txt
 commands =
-  find . -type f -name "*.pyc" -delete
-  find . -type d -name "__pycache__" -delete
+  find . -type f -name "*.py[c|o]" -delete
   ostestr --concurrency=6 {posargs}
 
 [testenv:pep8]
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    bandit -r watcher -x tests -n5 -ll
+    bandit -r watcher -x tests -n5 -ll -s B320
 
 [testenv:venv]
 setenv = PYTHONHASHSEED=0
@@ -66,4 +66,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen
 
 [testenv:bandit]
 deps = -r{toxinidir}/test-requirements.txt
-commands = bandit -r watcher -x tests -n5 -ll
+commands = bandit -r watcher -x tests -n5 -ll -s B320

watcher/api/acl.py

@@ -35,7 +35,7 @@ CONF.register_opts(AUTH_OPTS)
 def install(app, conf, public_routes):
     """Install ACL check on application.
 
-    :param app: A WSGI applicatin.
+    :param app: A WSGI application.
     :param conf: Settings. Dict'ified and passed to keystonemiddleware
     :param public_routes: The list of the routes which will be allowed to
                           access without authentication.

watcher/api/controllers/root.py

@@ -61,7 +61,7 @@ class Root(base.APIBase):
         root = Root()
         root.name = "OpenStack Watcher API"
         root.description = ("Watcher is an OpenStack project which aims to "
-                            "to improve physical resources usage through "
+                            "improve physical resources usage through "
                             "better VM placement.")
         root.versions = [Version.convert('v1')]
         root.default_version = Version.convert('v1')

watcher/api/controllers/v1/__init__.py

@@ -35,6 +35,7 @@ from watcher.api.controllers.v1 import audit
 from watcher.api.controllers.v1 import audit_template
 from watcher.api.controllers.v1 import goal
 from watcher.api.controllers.v1 import scoring_engine
+from watcher.api.controllers.v1 import service
 from watcher.api.controllers.v1 import strategy
 
 
@@ -105,6 +106,9 @@ class V1(APIBase):
     scoring_engines = [link.Link]
     """Links to the Scoring Engines resource"""
 
+    services = [link.Link]
+    """Links to the services resource"""
+
     links = [link.Link]
     """Links that point to a specific URL for this version and documentation"""
 
@@ -159,6 +163,14 @@ class V1(APIBase):
                                 'scoring_engines', '',
                                 bookmark=True)
             ]
+
+        v1.services = [link.Link.make_link(
+            'self', pecan.request.host_url, 'services', ''),
+            link.Link.make_link('bookmark',
+                                pecan.request.host_url,
+                                'services', '',
+                                bookmark=True)
+            ]
         return v1
 
 
@@ -171,6 +183,7 @@ class Controller(rest.RestController):
     action_plans = action_plan.ActionPlansController()
     goals = goal.GoalsController()
     scoring_engines = scoring_engine.ScoringEngineController()
+    services = service.ServicesController()
     strategies = strategy.StrategiesController()
 
     @wsme_pecan.wsexpose(V1)

watcher/api/controllers/v1/action.py

@@ -379,7 +379,7 @@ class ActionsController(rest.RestController):
         action_dict = action.as_dict()
         context = pecan.request.context
         new_action = objects.Action(context, **action_dict)
-        new_action.create(context)
+        new_action.create()
 
         # Set the HTTP Location Header
         pecan.response.location = link.build_url('actions', new_action.uuid)

watcher/api/controllers/v1/audit.py

@@ -60,8 +60,6 @@ class AuditPostType(wtypes.Base):
 
     audit_type = wtypes.wsattr(wtypes.text, mandatory=True)
 
-    deadline = wtypes.wsattr(datetime.datetime, mandatory=False)
-
     state = wsme.wsattr(wtypes.text, readonly=True,
                         default=objects.audit.State.PENDING)
 
@@ -69,8 +67,7 @@ class AuditPostType(wtypes.Base):
                                default={})
     interval = wsme.wsattr(int, mandatory=False)
 
-    host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1),
-                                 mandatory=False)
+    scope = wtypes.wsattr(types.jsontype, readonly=True)
 
     def as_audit(self, context):
         audit_type_values = [val.value for val in objects.audit.AuditType]
@@ -82,7 +79,7 @@ class AuditPostType(wtypes.Base):
             raise exception.AuditIntervalNotAllowed(audit_type=self.audit_type)
 
         if (self.audit_type == objects.audit.AuditType.CONTINUOUS.value and
-           self.interval in (wtypes.Unset, None)):
+                self.interval in (wtypes.Unset, None)):
             raise exception.AuditIntervalNotSpecified(
                 audit_type=self.audit_type)
 
@@ -100,7 +97,7 @@ class AuditPostType(wtypes.Base):
             at2a = {
                 'goal': 'goal_id',
                 'strategy': 'strategy_id',
-                'host_aggregate': 'host_aggregate'
+                'scope': 'scope',
             }
             to_string_fields = set(['goal', 'strategy'])
             for k in at2a:
@@ -114,12 +111,11 @@ class AuditPostType(wtypes.Base):
                         pass
         return Audit(
             audit_type=self.audit_type,
-            deadline=self.deadline,
             parameters=self.parameters,
             goal_id=self.goal,
-            host_aggregate=self.host_aggregate,
             strategy_id=self.strategy,
-            interval=self.interval)
+            interval=self.interval,
+            scope=self.scope,)
 
 
 class AuditPatchType(types.JsonPatchType):
@@ -230,9 +226,6 @@ class Audit(base.APIBase):
     audit_type = wtypes.text
     """Type of this audit"""
 
-    deadline = datetime.datetime
-    """deadline of the audit"""
-
     state = wtypes.text
     """This audit state"""
 
@@ -261,8 +254,8 @@ class Audit(base.APIBase):
     interval = wsme.wsattr(int, mandatory=False)
     """Launch audit periodically (in seconds)"""
 
-    host_aggregate = wtypes.IntegerType(minimum=1)
-    """ID of the Nova host aggregate targeted by the audit template"""
+    scope = wsme.wsattr(types.jsontype, mandatory=False)
+    """Audit Scope"""
 
     def __init__(self, **kwargs):
         self.fields = []
@@ -292,10 +285,10 @@ class Audit(base.APIBase):
     @staticmethod
     def _convert_with_links(audit, url, expand=True):
         if not expand:
-            audit.unset_fields_except(['uuid', 'audit_type', 'deadline',
-                                       'state', 'goal_uuid', 'interval',
-                                       'strategy_uuid', 'host_aggregate',
-                                       'goal_name', 'strategy_name'])
+            audit.unset_fields_except(['uuid', 'audit_type', 'state',
+                                       'goal_uuid', 'interval', 'scope',
+                                       'strategy_uuid', 'goal_name',
+                                       'strategy_name'])
 
         audit.links = [link.Link.make_link('self', url,
                                            'audits', audit.uuid),
@@ -316,15 +309,15 @@ class Audit(base.APIBase):
         sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
                      audit_type='ONESHOT',
                      state='PENDING',
-                     deadline=None,
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
                      updated_at=datetime.datetime.utcnow(),
-                     interval=7200)
+                     interval=7200,
+                     scope=[])
 
         sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
         sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff'
-        sample.host_aggregate = 1
+
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -381,7 +374,7 @@ class AuditsController(rest.RestController):
     def _get_audits_collection(self, marker, limit,
                                sort_key, sort_dir, expand=False,
                                resource_url=None, goal=None,
-                               strategy=None, host_aggregate=None):
+                               strategy=None):
         limit = api_utils.validate_limit(limit)
         api_utils.validate_sort_dir(sort_dir)
         marker_obj = None
@@ -424,19 +417,16 @@ class AuditsController(rest.RestController):
 
     @wsme_pecan.wsexpose(AuditCollection, types.uuid, int, wtypes.text,
                          wtypes.text, wtypes.text, wtypes.text, int)
-    def get_all(self, marker=None, limit=None,
-                sort_key='id', sort_dir='asc', goal=None,
-                strategy=None, host_aggregate=None):
+    def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc',
+                goal=None, strategy=None):
         """Retrieve a list of audits.
 
         :param marker: pagination marker for large data sets.
         :param limit: maximum number of resources to return in a single result.
         :param sort_key: column to sort results by. Default: id.
         :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
-         id.
         :param goal: goal UUID or name to filter by
         :param strategy: strategy UUID or name to filter by
-        :param host_aggregate: Optional host_aggregate
         """
 
         context = pecan.request.context
@@ -445,8 +435,7 @@ class AuditsController(rest.RestController):
 
         return self._get_audits_collection(marker, limit, sort_key,
                                            sort_dir, goal=goal,
-                                           strategy=strategy,
-                                           host_aggregate=host_aggregate)
+                                           strategy=strategy)
 
     @wsme_pecan.wsexpose(AuditCollection, wtypes.text, types.uuid, int,
                          wtypes.text, wtypes.text)
@@ -530,7 +519,7 @@ class AuditsController(rest.RestController):
         audit_dict = audit.as_dict()
 
         new_audit = objects.Audit(context, **audit_dict)
-        new_audit.create(context)
+        new_audit.create()
 
         # Set the HTTP Location Header
         pecan.response.location = link.build_url('audits', new_audit.uuid)
@@ -555,14 +544,11 @@ class AuditsController(rest.RestController):
             raise exception.OperationNotPermitted
 
         context = pecan.request.context
-        audit_to_update = api_utils.get_resource('Audit',
-                                                 audit_uuid)
+        audit_to_update = api_utils.get_resource(
+            'Audit', audit_uuid, eager=True)
         policy.enforce(context, 'audit:update', audit_to_update,
                        action='audit:update')
 
-        audit_to_update = objects.Audit.get_by_uuid(pecan.request.context,
-                                                    audit_uuid)
-
         try:
             audit_dict = audit_to_update.as_dict()
             audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch))
@@ -591,7 +577,8 @@ class AuditsController(rest.RestController):
         :param audit_uuid: UUID of a audit.
         """
         context = pecan.request.context
-        audit_to_delete = api_utils.get_resource('Audit', audit_uuid)
+        audit_to_delete = api_utils.get_resource(
+            'Audit', audit_uuid, eager=True)
         policy.enforce(context, 'audit:update', audit_to_delete,
                        action='audit:update')
 

watcher/api/controllers/v1/audit_template.py

@@ -41,11 +41,6 @@ settings related to the level of automation for the
 A flag will indicate whether the :ref:`Action Plan <action_plan_definition>`
 will be launched automatically or will need a manual confirmation from the
 :ref:`Administrator <administrator_definition>`.
-
-Last but not least, an :ref:`Audit Template <audit_template_definition>` may
-contain a list of extra parameters related to the
-:ref:`Strategy <strategy_definition>` configuration. These parameters can be
-provided as a list of key-value pairs.
 """
 
 import datetime
@@ -66,6 +61,7 @@ from watcher.common import context as context_utils
 from watcher.common import exception
 from watcher.common import policy
 from watcher.common import utils as common_utils
+from watcher.decision_engine.scope import default
 from watcher import objects
 
 
@@ -78,37 +74,24 @@ class AuditTemplatePostType(wtypes.Base):
     description = wtypes.wsattr(wtypes.text, mandatory=False)
     """Short description of this audit template"""
 
-    deadline = wsme.wsattr(datetime.datetime, mandatory=False)
-    """deadline of the audit template"""
-
-    host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1),
-                                 mandatory=False)
-    """ID of the Nova host aggregate targeted by the audit template"""
-
-    extra = wtypes.wsattr({wtypes.text: types.jsontype}, mandatory=False)
-    """The metadata of the audit template"""
-
     goal = wtypes.wsattr(wtypes.text, mandatory=True)
     """Goal UUID or name of the audit template"""
 
     strategy = wtypes.wsattr(wtypes.text, mandatory=False)
     """Strategy UUID or name of the audit template"""
 
-    version = wtypes.text
-    """Internal version of the audit template"""
+    scope = wtypes.wsattr(types.jsontype, mandatory=False, default=[])
+    """Audit Scope"""
 
     def as_audit_template(self):
         return AuditTemplate(
             name=self.name,
             description=self.description,
-            deadline=self.deadline,
-            host_aggregate=self.host_aggregate,
-            extra=self.extra,
             goal_id=self.goal,  # Dirty trick ...
             goal=self.goal,
             strategy_id=self.strategy,  # Dirty trick ...
             strategy_uuid=self.strategy,
-            version=self.version,
+            scope=self.scope,
         )
 
     @staticmethod
@@ -123,6 +106,9 @@ class AuditTemplatePostType(wtypes.Base):
         else:
             raise exception.InvalidGoal(goal=audit_template.goal)
 
+        common_utils.Draft4Validator(
+            default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope)
+
         if audit_template.strategy:
             available_strategies = objects.Strategy.list(
                 AuditTemplatePostType._ctx)
@@ -305,18 +291,9 @@ class AuditTemplate(base.APIBase):
     name = wtypes.text
     """Name of this audit template"""
 
-    description = wtypes.text
+    description = wtypes.wsattr(wtypes.text, mandatory=False)
     """Short description of this audit template"""
 
-    deadline = datetime.datetime
-    """deadline of the audit template"""
-
-    host_aggregate = wtypes.IntegerType(minimum=1)
-    """ID of the Nova host aggregate targeted by the audit template"""
-
-    extra = {wtypes.text: types.jsontype}
-    """The metadata of the audit template"""
-
     goal_uuid = wsme.wsproperty(
         wtypes.text, _get_goal_uuid, _set_goal_uuid, mandatory=True)
     """Goal UUID the audit template refers to"""
@@ -333,15 +310,15 @@ class AuditTemplate(base.APIBase):
         wtypes.text, _get_strategy_name, _set_strategy_name, mandatory=False)
     """The name of the strategy this audit template refers to"""
 
-    version = wtypes.text
-    """Internal version of the audit template"""
-
     audits = wsme.wsattr([link.Link], readonly=True)
     """Links to the collection of audits contained in this audit template"""
 
     links = wsme.wsattr([link.Link], readonly=True)
     """A list containing a self link and associated audit template links"""
 
+    scope = wsme.wsattr(types.jsontype, mandatory=False)
+    """Audit Scope"""
+
     def __init__(self, **kwargs):
         super(AuditTemplate, self).__init__()
         self.fields = []
@@ -374,8 +351,8 @@ class AuditTemplate(base.APIBase):
     def _convert_with_links(audit_template, url, expand=True):
         if not expand:
             audit_template.unset_fields_except(
-                ['uuid', 'name', 'host_aggregate', 'goal_uuid', 'goal_name',
-                 'strategy_uuid', 'strategy_name'])
+                ['uuid', 'name', 'goal_uuid', 'goal_name',
+                 'scope', 'strategy_uuid', 'strategy_name'])
 
         # The numeric ID should not be exposed to
         # the user, it's internal only.
@@ -402,13 +379,12 @@ class AuditTemplate(base.APIBase):
         sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
                      name='My Audit Template',
                      description='Description of my audit template',
-                     host_aggregate=5,
                      goal_uuid='83e44733-b640-40e2-8d8a-7dd3be7134e6',
                      strategy_uuid='367d826e-b6a4-4b70-bc44-c3f6fe1c9986',
-                     extra={'automatic': True},
                      created_at=datetime.datetime.utcnow(),
                      deleted_at=None,
-                     updated_at=datetime.datetime.utcnow())
+                     updated_at=datetime.datetime.utcnow(),
+                     scope=[],)
         return cls._convert_with_links(sample, 'http://localhost:9322', expand)
 
 
@@ -592,11 +568,11 @@ class AuditTemplatesController(rest.RestController):
         audit_template_dict = audit_template.as_dict()
         new_audit_template = objects.AuditTemplate(context,
                                                    **audit_template_dict)
-        new_audit_template.create(context)
+        new_audit_template.create()
 
         # Set the HTTP Location Header
-        pecan.response.location = link.build_url('audit_templates',
-                                                 new_audit_template.uuid)
+        pecan.response.location = link.build_url(
+            'audit_templates', new_audit_template.uuid)
         return AuditTemplate.convert_with_links(new_audit_template)
 
     @wsme.validate(types.uuid, [AuditTemplatePatchType])

watcher/api/controllers/v1/efficacy_indicator.py

@@ -19,7 +19,7 @@ An efficacy indicator is a single value that gives an indication on how the
 :ref:`solution <solution_definition>` produced by a given :ref:`strategy
 <strategy_definition>` performed. These efficacy indicators are specific to a
 given :ref:`goal <goal_definition>` and are usually used to compute the
-:ref:`gobal efficacy <efficacy_definition>` of the resulting :ref:`action plan
+:ref:`global efficacy <efficacy_definition>` of the resulting :ref:`action plan
 <action_plan_definition>`.
 
 In Watcher, these efficacy indicators are specified alongside the goal they

watcher/api/controllers/v1/service.py

@@ -0,0 +1,263 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 Servionica
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Service mechanism provides ability to monitor Watcher services state.
+"""
+
+import datetime
+import six
+
+from oslo_config import cfg
+from oslo_log import log
+from oslo_utils import timeutils
+import pecan
+from pecan import rest
+import wsme
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from watcher._i18n import _LW
+from watcher.api.controllers import base
+from watcher.api.controllers import link
+from watcher.api.controllers.v1 import collection
+from watcher.api.controllers.v1 import utils as api_utils
+from watcher.common import exception
+from watcher.common import policy
+from watcher import objects
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+class Service(base.APIBase):
+    """API representation of a service.
+
+    This class enforces type checking and value constraints, and converts
+    between the internal object model and the API representation of a service.
+    """
+
+    _status = None
+
+    def _get_status(self):
+        return self._status
+
+    def _set_status(self, name):
+        service = objects.Service.get_by_name(pecan.request.context, name)
+        last_heartbeat = (service.last_seen_up or service.updated_at
+                          or service.created_at)
+        if isinstance(last_heartbeat, six.string_types):
+            # NOTE(russellb) If this service came in over rpc via
+            # conductor, then the timestamp will be a string and needs to be
+            # converted back to a datetime.
+            last_heartbeat = timeutils.parse_strtime(last_heartbeat)
+        else:
+            # Objects have proper UTC timezones, but the timeutils comparison
+            # below does not (and will fail)
+            last_heartbeat = last_heartbeat.replace(tzinfo=None)
+        elapsed = timeutils.delta_seconds(last_heartbeat, timeutils.utcnow())
+        is_up = abs(elapsed) <= CONF.service_down_time
+        if not is_up:
+            LOG.warning(_LW('Seems service %(name)s on host %(host)s is down. '
+                            'Last heartbeat was %(lhb)s.'
+                            'Elapsed time is %(el)s'),
+                        {'name': service.name,
+                         'host': service.host,
+                         'lhb': str(last_heartbeat), 'el': str(elapsed)})
+            self._status = objects.service.ServiceStatus.FAILED
+        else:
+            self._status = objects.service.ServiceStatus.ACTIVE
+
+    id = wsme.wsattr(int, readonly=True)
+    """ID for this service."""
+
+    name = wtypes.text
+    """Name of the service."""
+
+    host = wtypes.text
+    """Host where service is placed on."""
+
+    last_seen_up = wsme.wsattr(datetime.datetime, readonly=True)
+    """Time when Watcher service sent latest heartbeat."""
+
+    status = wsme.wsproperty(wtypes.text, _get_status, _set_status,
+                             mandatory=True)
+
+    links = wsme.wsattr([link.Link], readonly=True)
+    """A list containing a self link."""
+
+    def __init__(self, **kwargs):
+        super(Service, self).__init__()
+
+        fields = list(objects.Service.fields.keys()) + ['status']
+        self.fields = []
+        for field in fields:
+            self.fields.append(field)
+            setattr(self, field, kwargs.get(
+                field if field != 'status' else 'name', wtypes.Unset))
+
+    @staticmethod
+    def _convert_with_links(service, url, expand=True):
+        if not expand:
+            service.unset_fields_except(
+                ['id', 'name', 'host', 'status'])
+
+        service.links = [
+            link.Link.make_link('self', url, 'services', str(service.id)),
+            link.Link.make_link('bookmark', url, 'services', str(service.id),
+                                bookmark=True)]
+        return service
+
+    @classmethod
+    def convert_with_links(cls, service, expand=True):
+        service = Service(**service.as_dict())
+        return cls._convert_with_links(
+            service, pecan.request.host_url, expand)
+
+    @classmethod
+    def sample(cls, expand=True):
+        sample = cls(id=1,
+                     name='watcher-applier',
+                     host='Controller',
+                     last_seen_up=datetime.datetime(2016, 1, 1))
+        return cls._convert_with_links(sample, 'http://localhost:9322', expand)
+
+
+class ServiceCollection(collection.Collection):
+    """API representation of a collection of services."""
+
+    services = [Service]
+    """A list containing services objects"""
+
+    def __init__(self, **kwargs):
+        super(ServiceCollection, self).__init__()
+        self._type = 'services'
+
+    @staticmethod
+    def convert_with_links(services, limit, url=None, expand=False,
+                           **kwargs):
+        service_collection = ServiceCollection()
+        service_collection.services = [
+            Service.convert_with_links(g, expand) for g in services]
+
+        if 'sort_key' in kwargs:
+            reverse = False
+            if kwargs['sort_key'] == 'service':
+                if 'sort_dir' in kwargs:
+                    reverse = True if kwargs['sort_dir'] == 'desc' else False
+                service_collection.services = sorted(
+                    service_collection.services,
+                    key=lambda service: service.id,
+                    reverse=reverse)
+
+        service_collection.next = service_collection.get_next(
+            limit, url=url, marker_field='id', **kwargs)
+        return service_collection
+
+    @classmethod
+    def sample(cls):
+        sample = cls()
+        sample.services = [Service.sample(expand=False)]
+        return sample
+
+
+class ServicesController(rest.RestController):
+    """REST controller for Services."""
+    def __init__(self):
+        super(ServicesController, self).__init__()
+
+    from_services = False
+    """A flag to indicate if the requests to this controller are coming
+    from the top-level resource Services."""
+
+    _custom_actions = {
+        'detail': ['GET'],
+    }
+
+    def _get_services_collection(self, marker, limit, sort_key, sort_dir,
+                                 expand=False, resource_url=None):
+        limit = api_utils.validate_limit(limit)
+        api_utils.validate_sort_dir(sort_dir)
+
+        sort_db_key = (sort_key if sort_key in objects.Service.fields.keys()
+                       else None)
+
+        marker_obj = None
+        if marker:
+            marker_obj = objects.Service.get(
+                pecan.request.context, marker)
+
+        services = objects.Service.list(
+            pecan.request.context, limit, marker_obj,
+            sort_key=sort_db_key, sort_dir=sort_dir)
+
+        return ServiceCollection.convert_with_links(
+            services, limit, url=resource_url, expand=expand,
+            sort_key=sort_key, sort_dir=sort_dir)
+
+    @wsme_pecan.wsexpose(ServiceCollection, int, int, wtypes.text, wtypes.text)
+    def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc'):
+        """Retrieve a list of services.
+
+        :param marker: pagination marker for large data sets.
+        :param limit: maximum number of resources to return in a single result.
+        :param sort_key: column to sort results by. Default: id.
+        :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
+        """
+        context = pecan.request.context
+        policy.enforce(context, 'service:get_all',
+                       action='service:get_all')
+
+        return self._get_services_collection(marker, limit, sort_key, sort_dir)
+
+    @wsme_pecan.wsexpose(ServiceCollection, int, int, wtypes.text, wtypes.text)
+    def detail(self, marker=None, limit=None, sort_key='id', sort_dir='asc'):
+        """Retrieve a list of services with detail.
+
+        :param marker: pagination marker for large data sets.
+        :param limit: maximum number of resources to return in a single result.
+        :param sort_key: column to sort results by. Default: id.
+        :param sort_dir: direction to sort. "asc" or "desc". Default: asc.
+        """
+        context = pecan.request.context
+        policy.enforce(context, 'service:detail',
+                       action='service:detail')
+        # NOTE(lucasagomes): /detail should only work agaist collections
+        parent = pecan.request.path.split('/')[:-1][-1]
+        if parent != "services":
+            raise exception.HTTPNotFound
+        expand = True
+        resource_url = '/'.join(['services', 'detail'])
+
+        return self._get_services_collection(
+            marker, limit, sort_key, sort_dir, expand, resource_url)
+
+    @wsme_pecan.wsexpose(Service, wtypes.text)
+    def get_one(self, service):
+        """Retrieve information about the given service.
+
+        :param service: ID or name of the service.
+        """
+        if self.from_services:
+            raise exception.OperationNotPermitted
+
+        context = pecan.request.context
+        rpc_service = api_utils.get_resource('Service', service)
+        policy.enforce(context, 'service:get', rpc_service,
+                       action='service:get')
+
+        return Service.convert_with_links(rpc_service)

watcher/api/controllers/v1/strategy.py

@@ -110,7 +110,7 @@ class Strategy(base.APIBase):
     """The name of the goal this audit refers to"""
 
     parameters_spec = {wtypes.text: types.jsontype}
-    """ Parameters spec dict"""
+    """Parameters spec dict"""
 
     def __init__(self, **kwargs):
         super(Strategy, self).__init__()

watcher/api/controllers/v1/utils.py

@@ -15,11 +15,13 @@
 
 import jsonpatch
 from oslo_config import cfg
+from oslo_utils import reflection
 from oslo_utils import uuidutils
 import pecan
 import wsme
 
 from watcher._i18n import _
+from watcher.common import utils
 from watcher import objects
 
 CONF = cfg.CONF
@@ -80,17 +82,27 @@ def as_filters_dict(**filters):
     return filters_dict
 
 
-def get_resource(resource, resource_ident):
-    """Get the resource from the uuid or logical name.
+def get_resource(resource, resource_id, eager=False):
+    """Get the resource from the uuid, id or logical name.
 
     :param resource: the resource type.
-    :param resource_ident: the UUID or logical name of the resource.
+    :param resource_id: the UUID, ID or logical name of the resource.
 
     :returns: The resource.
     """
     resource = getattr(objects, resource)
 
-    if uuidutils.is_uuid_like(resource_ident):
-        return resource.get_by_uuid(pecan.request.context, resource_ident)
+    _get = None
+    if utils.is_int_like(resource_id):
+        resource_id = int(resource_id)
+        _get = resource.get
+    elif uuidutils.is_uuid_like(resource_id):
+        _get = resource.get_by_uuid
+    else:
+        _get = resource.get_by_name
 
-    return resource.get_by_name(pecan.request.context, resource_ident)
+    method_signature = reflection.get_signature(_get)
+    if 'eager' in method_signature.parameters:
+        return _get(pecan.request.context, resource_id, eager=eager)
+
+    return _get(pecan.request.context, resource_id)

watcher/api/hooks.py

@@ -114,6 +114,6 @@ class NoExceptionTracebackHook(hooks.PecanHook):
             faultstring = faultstring.split(traceback_marker, 1)[0]
             # Remove trailing newlines and spaces if any.
             json_body['faultstring'] = faultstring.rstrip()
-            # Replace the whole json. Cannot change original one beacause it's
+            # Replace the whole json. Cannot change original one because it's
             # generated on the fly.
             state.response.json = json_body

watcher/api/middleware/auth_token.py

@@ -33,7 +33,7 @@ class AuthTokenMiddleware(auth_token.AuthProtocol):
     for public routes in the API.
 
     """
-    def __init__(self, app, conf, public_api_routes=[]):
+    def __init__(self, app, conf, public_api_routes=()):
         route_pattern_tpl = '%s(\.json|\.xml)?$'
 
         try:

watcher/applier/action_plan/default.py

@@ -20,9 +20,7 @@ from oslo_log import log
 
 from watcher.applier.action_plan import base
 from watcher.applier import default
-from watcher.applier.messaging import event_types
-from watcher.common.messaging.events import event
-from watcher.objects import action_plan as ap_objects
+from watcher import objects
 
 LOG = log.getLogger(__name__)
 
@@ -34,32 +32,20 @@ class DefaultActionPlanHandler(base.BaseActionPlanHandler):
         self.service = service
         self.action_plan_uuid = action_plan_uuid
 
-    def notify(self, uuid, event_type, state):
-        action_plan = ap_objects.ActionPlan.get_by_uuid(self.ctx, uuid)
+    def update_action_plan(self, uuid, state):
+        action_plan = objects.ActionPlan.get_by_uuid(self.ctx, uuid)
         action_plan.state = state
         action_plan.save()
-        ev = event.Event()
-        ev.type = event_type
-        ev.data = {}
-        payload = {'action_plan__uuid': uuid,
-                   'action_plan_state': state}
-        self.service.publish_status_event(ev.type.name, payload)
 
     def execute(self):
         try:
-            # update state
-            self.notify(self.action_plan_uuid,
-                        event_types.EventTypes.LAUNCH_ACTION_PLAN,
-                        ap_objects.State.ONGOING)
+            self.update_action_plan(self.action_plan_uuid,
+                                    objects.action_plan.State.ONGOING)
             applier = default.DefaultApplier(self.ctx, self.service)
             applier.execute(self.action_plan_uuid)
-            state = ap_objects.State.SUCCEEDED
+            state = objects.action_plan.State.SUCCEEDED
         except Exception as e:
             LOG.exception(e)
-            state = ap_objects.State.FAILED
-
+            state = objects.action_plan.State.FAILED
         finally:
-            # update state
-            self.notify(self.action_plan_uuid,
-                        event_types.EventTypes.LAUNCH_ACTION_PLAN,
-                        state)
+            self.update_action_plan(self.action_plan_uuid, state)

watcher/applier/manager.py

@@ -20,6 +20,7 @@
 from oslo_config import cfg
 
 from watcher.applier.messaging import trigger
+from watcher.common import service_manager
 
 CONF = cfg.CONF
 
@@ -36,13 +37,6 @@ APPLIER_MANAGER_OPTS = [
                help='The topic name used for'
                     'control events, this topic '
                     'used for rpc call '),
-    cfg.StrOpt('status_topic',
-               default='watcher.applier.status',
-               help='The topic name used for '
-                    'status events, this topic '
-                    'is used so as to notify'
-                    'the others components '
-                    'of the system'),
     cfg.StrOpt('publisher_id',
                default='watcher.applier.api',
                help='The identifier used by watcher '
@@ -60,17 +54,32 @@ CONF.register_group(opt_group)
 CONF.register_opts(APPLIER_MANAGER_OPTS, opt_group)
 
 
-class ApplierManager(object):
+class ApplierManager(service_manager.ServiceManager):
 
-    API_VERSION = '1.0'
+    @property
+    def service_name(self):
+        return 'watcher-applier'
 
-    conductor_endpoints = [trigger.TriggerActionPlan]
-    status_endpoints = []
-    notification_endpoints = []
-    notification_topics = []
+    @property
+    def api_version(self):
+        return '1.0'
 
-    def __init__(self):
-        self.publisher_id = CONF.watcher_applier.publisher_id
-        self.conductor_topic = CONF.watcher_applier.conductor_topic
-        self.status_topic = CONF.watcher_applier.status_topic
-        self.api_version = self.API_VERSION
+    @property
+    def publisher_id(self):
+        return CONF.watcher_applier.publisher_id
+
+    @property
+    def conductor_topic(self):
+        return CONF.watcher_applier.conductor_topic
+
+    @property
+    def notification_topics(self):
+        return []
+
+    @property
+    def conductor_endpoints(self):
+        return [trigger.TriggerActionPlan]
+
+    @property
+    def notification_endpoints(self):
+        return []

watcher/applier/rpcapi.py

@@ -15,12 +15,13 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 from oslo_config import cfg
 
 from watcher.applier import manager
 from watcher.common import exception
 from watcher.common import service
+from watcher.common import service_manager
 from watcher.common import utils
 
 
@@ -39,21 +40,35 @@ class ApplierAPI(service.Service):
             raise exception.InvalidUuidOrName(name=action_plan_uuid)
 
         return self.conductor_client.call(
-            context.to_dict(), 'launch_action_plan',
-            action_plan_uuid=action_plan_uuid)
+            context, 'launch_action_plan', action_plan_uuid=action_plan_uuid)
 
 
-class ApplierAPIManager(object):
+class ApplierAPIManager(service_manager.ServiceManager):
 
-    API_VERSION = '1.0'
+    @property
+    def service_name(self):
+        return None
 
-    conductor_endpoints = []
-    status_endpoints = []
-    notification_endpoints = []
-    notification_topics = []
+    @property
+    def api_version(self):
+        return '1.0'
 
-    def __init__(self):
-        self.publisher_id = CONF.watcher_applier.publisher_id
-        self.conductor_topic = CONF.watcher_applier.conductor_topic
-        self.status_topic = CONF.watcher_applier.status_topic
-        self.api_version = self.API_VERSION
+    @property
+    def publisher_id(self):
+        return CONF.watcher_applier.publisher_id
+
+    @property
+    def conductor_topic(self):
+        return CONF.watcher_applier.conductor_topic
+
+    @property
+    def notification_topics(self):
+        return []
+
+    @property
+    def conductor_endpoints(self):
+        return []
+
+    @property
+    def notification_endpoints(self):
+        return []

watcher/applier/workflow_engine/base.py

@@ -21,10 +21,8 @@ import abc
 import six
 
 from watcher.applier.actions import factory
-from watcher.applier.messaging import event_types
 from watcher.common import clients
 from watcher.common.loader import loadable
-from watcher.common.messaging.events import event
 from watcher import objects
 
 
@@ -77,12 +75,7 @@ class BaseWorkFlowEngine(loadable.Loadable):
         db_action = objects.Action.get_by_uuid(self.context, action.uuid)
         db_action.state = state
         db_action.save()
-        ev = event.Event()
-        ev.type = event_types.EventTypes.LAUNCH_ACTION
-        ev.data = {}
-        payload = {'action_uuid': action.uuid,
-                   'action_state': state}
-        self.applier_manager.publish_status_event(ev.type.name, payload)
+        # NOTE(v-francoise): Implement notifications for action
 
     @abc.abstractmethod
     def execute(self, actions):

watcher/applier/workflow_engine/default.py

@@ -23,7 +23,7 @@ from taskflow import task
 from watcher._i18n import _LE, _LW, _LC
 from watcher.applier.workflow_engine import base
 from watcher.common import exception
-from watcher.objects import action as obj_action
+from watcher import objects
 
 LOG = log.getLogger(__name__)
 
@@ -107,14 +107,12 @@ class TaskFlowActionContainer(task.Task):
 
     def pre_execute(self):
         try:
-            self.engine.notify(self._db_action,
-                               obj_action.State.ONGOING)
+            self.engine.notify(self._db_action, objects.action.State.ONGOING)
             LOG.debug("Pre-condition action: %s", self.name)
             self.action.pre_condition()
         except Exception as e:
             LOG.exception(e)
-            self.engine.notify(self._db_action,
-                               obj_action.State.FAILED)
+            self.engine.notify(self._db_action, objects.action.State.FAILED)
             raise
 
     def execute(self, *args, **kwargs):
@@ -122,15 +120,13 @@ class TaskFlowActionContainer(task.Task):
             LOG.debug("Running action: %s", self.name)
 
             self.action.execute()
-            self.engine.notify(self._db_action,
-                               obj_action.State.SUCCEEDED)
+            self.engine.notify(self._db_action, objects.action.State.SUCCEEDED)
         except Exception as e:
             LOG.exception(e)
             LOG.error(_LE('The workflow engine has failed '
                           'to execute the action: %s'), self.name)
 
-            self.engine.notify(self._db_action,
-                               obj_action.State.FAILED)
+            self.engine.notify(self._db_action, objects.action.State.FAILED)
             raise
 
     def post_execute(self):
@@ -139,8 +135,7 @@ class TaskFlowActionContainer(task.Task):
             self.action.post_condition()
         except Exception as e:
             LOG.exception(e)
-            self.engine.notify(self._db_action,
-                               obj_action.State.FAILED)
+            self.engine.notify(self._db_action, objects.action.State.FAILED)
             raise
 
     def revert(self, *args, **kwargs):

watcher/cmd/api.py

@@ -24,19 +24,19 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service
+from watcher import conf
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
 
     host, port = cfg.CONF.api.host, cfg.CONF.api.port
     protocol = "http" if not CONF.api.enable_ssl_api else "https"
     # Build and start the WSGI app
-    server = service.WSGIService(
-        'watcher-api', CONF.api.enable_ssl_api)
+    server = service.WSGIService('watcher-api', CONF.api.enable_ssl_api)
 
     if host == '127.0.0.1':
         LOG.info(_LI('serving on 127.0.0.1:%(port)s, '

watcher/cmd/applier.py

@@ -20,19 +20,19 @@
 import os
 import sys
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.applier import manager
 from watcher.common import service as watcher_service
+from watcher import conf
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    watcher_service.prepare_service(sys.argv)
+    watcher_service.prepare_service(sys.argv, CONF)
 
     LOG.info(_LI('Starting Watcher Applier service in PID %s'), os.getpid())
 

watcher/cmd/dbmanage.py

@@ -24,10 +24,11 @@ import sys
 from oslo_config import cfg
 
 from watcher.common import service
+from watcher import conf
 from watcher.db import migration
 from watcher.db import purge
 
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 class DBCommand(object):
@@ -152,5 +153,5 @@ def main():
     if not set(sys.argv).intersection(valid_commands):
         sys.argv.append('upgrade')
 
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
     CONF.command.func()

watcher/cmd/decisionengine.py

@@ -20,22 +20,22 @@
 import os
 import sys
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as watcher_service
+from watcher import conf
 from watcher.decision_engine import gmr
 from watcher.decision_engine import manager
 from watcher.decision_engine import scheduling
 from watcher.decision_engine import sync
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    watcher_service.prepare_service(sys.argv)
+    watcher_service.prepare_service(sys.argv, CONF)
     gmr.register_gmr_plugins()
 
     LOG.info(_LI('Starting Watcher Decision Engine service in PID %s'),

watcher/cmd/sync.py

@@ -24,15 +24,17 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as service
+from watcher import conf
 from watcher.decision_engine import sync
 
 LOG = logging.getLogger(__name__)
+CONF = conf.CONF
 
 
 def main():
     LOG.info(_LI('Watcher sync started.'))
 
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
     syncer = sync.Syncer()
     syncer.sync()
 

watcher/common/ceilometer_helper.py

@@ -144,17 +144,17 @@ class CeilometerHelper(object):
         :return:
         """
 
+        end_time = datetime.datetime.utcnow()
         start_time = (datetime.datetime.utcnow() -
                       datetime.timedelta(seconds=int(period)))
         query = self.build_query(
-            resource_id=resource_id, start_time=start_time)
+            resource_id=resource_id, start_time=start_time, end_time=end_time)
         statistic = self.query_retry(f=self.ceilometer.statistics.list,
                                      meter_name=meter_name,
                                      q=query,
                                      period=period,
                                      aggregates=[
-                                         {'func': aggregate}],
-                                     groupby=['resource_id'])
+                                         {'func': aggregate}])
 
         item_value = None
         if statistic:

watcher/common/exception.py

@@ -34,14 +34,14 @@ from watcher._i18n import _, _LE
 
 LOG = logging.getLogger(__name__)
 
-exc_log_opts = [
+EXC_LOG_OPTS = [
     cfg.BoolOpt('fatal_exception_format_errors',
                 default=False,
                 help='Make exception message format errors fatal.'),
 ]
 
 CONF = cfg.CONF
-CONF.register_opts(exc_log_opts)
+CONF.register_opts(EXC_LOG_OPTS)
 
 
 def wrap_keystone_exception(func):
@@ -119,6 +119,10 @@ class WatcherException(Exception):
             return six.text_type(self)
 
 
+class UnsupportedError(WatcherException):
+    msg_fmt = _("Not supported")
+
+
 class NotAuthorized(WatcherException):
     msg_fmt = _("Not authorized")
     code = 403
@@ -168,6 +172,14 @@ class InvalidStrategy(Invalid):
     msg_fmt = _("Strategy %(strategy)s is invalid")
 
 
+class InvalidAudit(Invalid):
+    msg_fmt = _("Audit %(audit)s is invalid")
+
+
+class EagerlyLoadedAuditRequired(InvalidAudit):
+    msg_fmt = _("Audit %(audit)s was not eagerly loaded")
+
+
 class InvalidUUID(Invalid):
     msg_fmt = _("Expected a uuid but received %(uuid)s")
 
@@ -362,6 +374,19 @@ class NoSuchMetricForHost(WatcherException):
     msg_fmt = _("No %(metric)s metric for %(host)s found.")
 
 
+class ServiceAlreadyExists(Conflict):
+    msg_fmt = _("A service with name %(name)s is already working on %(host)s.")
+
+
+class ServiceNotFound(ResourceNotFound):
+    msg_fmt = _("The service %(service)s cannot be found.")
+
+
+class WildcardCharacterIsUsed(WatcherException):
+    msg_fmt = _("You shouldn't use any other IDs of %(resource)s if you use "
+                "wildcard character.")
+
+
 # Model
 
 class InstanceNotFound(WatcherException):
@@ -386,3 +411,8 @@ class NotSoftDeletedStateError(WatcherException):
 
 class NegativeLimitError(WatcherException):
     msg_fmt = _("Limit should be positive")
+
+
+class NotificationPayloadError(WatcherException):
+    _msg_fmt = _("Payload not populated when trying to send notification "
+                 "\"%(class_name)s\"")

watcher/common/messaging/events/event.py

@@ -1,54 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-class Event(object):
-    """Generic event to use with EventDispatcher"""
-
-    def __init__(self, event_type=None, data=None, request_id=None):
-        """Default constructor
-
-        :param event_type: the type of the event
-        :param data: a dictionary which contains data
-        :param request_id: a string which represent the uuid of the request
-        """
-        self._type = event_type
-        self._data = data
-        self._request_id = request_id
-
-    @property
-    def type(self):
-        return self._type
-
-    @type.setter
-    def type(self, type):
-        self._type = type
-
-    @property
-    def data(self):
-        return self._data
-
-    @data.setter
-    def data(self, data):
-        self._data = data
-
-    @property
-    def request_id(self):
-        return self._request_id
-
-    @request_id.setter
-    def request_id(self, id):
-        self._request_id = id

watcher/common/messaging/events/event_dispatcher.py

@@ -1,78 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from oslo_log import log
-
-from watcher.decision_engine.messaging import events as messaging_events
-
-LOG = log.getLogger(__name__)
-
-
-class EventDispatcher(object):
-    """Generic event dispatcher which listen and dispatch events"""
-
-    def __init__(self):
-        self._events = dict()
-
-    def __del__(self):
-        self._events = None
-
-    def has_listener(self, event_type, listener):
-        """Return true if listener is register to event_type """
-        # Check for event type and for the listener
-        if event_type in self._events.keys():
-            return listener in self._events[event_type]
-        else:
-            return False
-
-    def dispatch_event(self, event):
-        LOG.debug("dispatch evt : %s" % str(event.type))
-        """
-        Dispatch an instance of Event class
-        """
-        if messaging_events.Events.ALL in self._events.keys():
-            listeners = self._events[messaging_events.Events.ALL]
-            for listener in listeners:
-                listener(event)
-
-        # Dispatch the event to all the associated listeners
-        if event.type in self._events.keys():
-            listeners = self._events[event.type]
-            for listener in listeners:
-                listener(event)
-
-    def add_event_listener(self, event_type, listener):
-        """Add an event listener for an event type"""
-        # Add listener to the event type
-        if not self.has_listener(event_type, listener):
-            listeners = self._events.get(event_type, [])
-            listeners.append(listener)
-            self._events[event_type] = listeners
-
-    def remove_event_listener(self, event_type, listener):
-        """Remove event listener. """
-        # Remove the listener from the event type
-        if self.has_listener(event_type, listener):
-            listeners = self._events[event_type]
-
-            if len(listeners) == 1:
-                # Only this listener remains so remove the key
-                del self._events[event_type]
-
-            else:
-                # Update listeners chain
-                listeners.remove(listener)
-                self._events[event_type] = listeners

watcher/common/messaging/messaging_handler.py

@@ -1,120 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import socket
-import threading
-
-import eventlet
-from oslo_config import cfg
-from oslo_log import log
-import oslo_messaging as om
-
-from watcher.common import rpc
-from watcher._i18n import _LE, _LW
-
-# NOTE:
-# Ubuntu 14.04 forces librabbitmq when kombu is used
-# Unfortunately it forces a version that has a crash
-# bug.  Calling eventlet.monkey_patch() tells kombu
-# to use libamqp instead.
-eventlet.monkey_patch()
-
-LOG = log.getLogger(__name__)
-CONF = cfg.CONF
-
-
-class MessagingHandler(threading.Thread):
-
-    def __init__(self, publisher_id, topic_name, endpoints, version,
-                 serializer=None):
-        super(MessagingHandler, self).__init__()
-        self.publisher_id = publisher_id
-        self.topic_name = topic_name
-        self.__endpoints = []
-        self.__serializer = serializer
-        self.__version = version
-
-        self.__server = None
-        self.__notifier = None
-        self.__transport = None
-        self.add_endpoints(endpoints)
-
-    def add_endpoints(self, endpoints):
-        self.__endpoints.extend(endpoints)
-
-    def remove_endpoint(self, endpoint):
-        if endpoint in self.__endpoints:
-            self.__endpoints.remove(endpoint)
-
-    @property
-    def endpoints(self):
-        return self.__endpoints
-
-    @property
-    def transport(self):
-        return self.__transport
-
-    def build_notifier(self):
-        serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
-        return om.Notifier(
-            self.__transport,
-            publisher_id=self.publisher_id,
-            topic=self.topic_name,
-            serializer=serializer
-        )
-
-    def build_server(self, target):
-        return om.get_rpc_server(self.__transport, target,
-                                 self.__endpoints,
-                                 executor='eventlet',
-                                 serializer=self.__serializer)
-
-    def _configure(self):
-        try:
-            self.__transport = om.get_transport(CONF)
-            self.__notifier = self.build_notifier()
-            if len(self.__endpoints):
-                target = om.Target(
-                    topic=self.topic_name,
-                    # For compatibility, we can override it with 'host' opt
-                    server=CONF.host or socket.getfqdn(),
-                    version=self.__version,
-                )
-                self.__server = self.build_server(target)
-            else:
-                LOG.warning(
-                    _LW("No endpoint defined; can only publish events"))
-        except Exception as e:
-            LOG.exception(e)
-            LOG.error(_LE("Messaging configuration error"))
-
-    def run(self):
-        LOG.debug("configure MessagingHandler for %s" % self.topic_name)
-        self._configure()
-        if len(self.__endpoints) > 0:
-            LOG.debug("Starting up server")
-            self.__server.start()
-
-    def stop(self):
-        LOG.debug('Stopped server')
-        self.__server.stop()
-
-    def publish_event(self, event_type, payload, request_id=None):
-        self.__notifier.info(
-            {'version_api': self.__version,
-             'request_id': request_id},
-            {'event_id': event_type}, payload
-        )

watcher/common/messaging/notification_handler.py

@@ -1,47 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import eventlet
-import oslo_messaging as messaging
-
-from watcher.common.messaging.utils import observable
-
-
-eventlet.monkey_patch()
-
-
-class NotificationHandler(observable.Observable):
-    def __init__(self, publisher_id):
-        super(NotificationHandler, self).__init__()
-        self.publisher_id = publisher_id
-
-    def info(self, ctx, publisher_id, event_type, payload, metadata):
-        if publisher_id == self.publisher_id:
-            self.set_changed()
-            self.notify(ctx, publisher_id, event_type, metadata, payload)
-            return messaging.NotificationResult.HANDLED
-
-    def warn(self, ctx, publisher_id, event_type, payload, metadata):
-        if publisher_id == self.publisher_id:
-            self.set_changed()
-            self.notify(ctx, publisher_id, event_type, metadata, payload)
-            return messaging.NotificationResult.HANDLED
-
-    def error(self, ctx, publisher_id, event_type, payload, metadata):
-        if publisher_id == self.publisher_id:
-            self.set_changed()
-            self.notify(ctx, publisher_id, event_type, metadata, payload)
-            return messaging.NotificationResult.HANDLED

watcher/common/nova_helper.py

@@ -63,16 +63,17 @@ class NovaHelper(object):
             LOG.exception(exc)
             raise exception.ComputeNodeNotFound(name=node_hostname)
 
+    def get_aggregate_list(self):
+        return self.nova.aggregates.list()
+
+    def get_aggregate_detail(self, aggregate_id):
+        return self.nova.aggregates.get(aggregate_id)
+
+    def get_availability_zone_list(self):
+        return self.nova.availability_zones.list(detailed=True)
+
     def find_instance(self, instance_id):
-        search_opts = {'all_tenants': True}
-        instances = self.nova.servers.list(detailed=True,
-                                           search_opts=search_opts)
-        instance = None
-        for _instance in instances:
-            if _instance.id == instance_id:
-                instance = _instance
-                break
-        return instance
+        return self.nova.servers.get(instance_id)
 
     def wait_for_volume_status(self, volume, status, timeout=60,
                                poll_interval=1):
@@ -663,7 +664,7 @@ class NovaHelper(object):
             cache[fid] = flavor
         attr_defaults = [('name', 'unknown-id-%s' % fid),
                          ('vcpus', 0), ('ram', 0), ('disk', 0),
-                         ('ephemeral', 0)]
+                         ('ephemeral', 0), ('extra_specs', {})]
         for attr, default in attr_defaults:
             if not flavor:
                 instance.flavor[attr] = default

watcher/common/rpc.py

@@ -72,8 +72,12 @@ def init(conf):
         aliases=TRANSPORT_ALIASES)
 
     serializer = RequestContextSerializer(JsonPayloadSerializer())
-    NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT,
-                                  serializer=serializer)
+    if not conf.notification_level:
+        NOTIFIER = messaging.Notifier(
+            NOTIFICATION_TRANSPORT, serializer=serializer, driver='noop')
+    else:
+        NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT,
+                                      serializer=serializer)
 
 
 def initialized():
@@ -121,7 +125,7 @@ class RequestContextSerializer(messaging.Serializer):
         return self._base.deserialize_entity(context, entity)
 
     def serialize_context(self, context):
-        return context
+        return context.to_dict()
 
     def deserialize_context(self, context):
         return watcher_context.RequestContext.from_dict(context)
@@ -146,8 +150,6 @@ def get_server(target, endpoints, serializer=None):
                                     serializer=serializer)
 
 
-def get_notifier(service=None, host=None, publisher_id=None):
+def get_notifier(publisher_id):
     assert NOTIFIER is not None
-    if not publisher_id:
-        publisher_id = "%s.%s" % (service, host or CONF.host)
     return NOTIFIER.prepare(publisher_id=publisher_id)

watcher/common/service.py

@@ -14,9 +14,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import logging
+import datetime
 import socket
 
+import eventlet
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import _options
@@ -27,30 +28,36 @@ from oslo_reports import opts as gmr_opts
 from oslo_service import service
 from oslo_service import wsgi
 
-from watcher._i18n import _, _LI
+from watcher._i18n import _
 from watcher.api import app
 from watcher.common import config
-from watcher.common.messaging.events import event_dispatcher as dispatcher
-from watcher.common.messaging import messaging_handler
+from watcher.common import context
 from watcher.common import rpc
+from watcher.common import scheduling
+from watcher.conf import plugins as plugins_conf
+from watcher import objects
 from watcher.objects import base
-from watcher import opts
+from watcher.objects import fields as wfields
 from watcher import version
 
-service_opts = [
-    cfg.IntOpt('periodic_interval',
-               default=60,
-               help=_('Seconds between running periodic tasks.')),
-    cfg.StrOpt('host',
-               default=socket.getfqdn(),
-               help=_('Name of this node. This can be an opaque identifier.  '
-                      'It is not necessarily a hostname, FQDN, or IP address. '
-                      'However, the node name must be valid within '
-                      'an AMQP key, and if using ZeroMQ, a valid '
-                      'hostname, FQDN, or IP address.')),
-]
+# NOTE:
+# Ubuntu 14.04 forces librabbitmq when kombu is used
+# Unfortunately it forces a version that has a crash
+# bug.  Calling eventlet.monkey_patch() tells kombu
+# to use libamqp instead.
+eventlet.monkey_patch()
+
+NOTIFICATION_OPTS = [
+    cfg.StrOpt('notification_level',
+               choices=[''] + list(wfields.NotificationPriority.ALL),
+               default=wfields.NotificationPriority.INFO,
+               help=_('Specifies the minimum level for which to send '
+                      'notifications. If not set, no notifications will '
+                      'be sent. The default is for this option to be at the '
+                      '`INFO` level.'))
+]
+cfg.CONF.register_opts(NOTIFICATION_OPTS)
 
-cfg.CONF.register_opts(service_opts)
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
@@ -68,21 +75,21 @@ Singleton = service.Singleton
 class WSGIService(service.ServiceBase):
     """Provides ability to launch Watcher API from wsgi app."""
 
-    def __init__(self, name, use_ssl=False):
+    def __init__(self, service_name, use_ssl=False):
         """Initialize, but do not start the WSGI server.
 
-        :param name: The name of the WSGI server given to the loader.
+        :param service_name: The service name of the WSGI server.
         :param use_ssl: Wraps the socket in an SSL context if True.
         """
-        self.name = name
+        self.service_name = service_name
         self.app = app.VersionSelectorApplication()
         self.workers = (CONF.api.workers or
                         processutils.get_worker_count())
-        self.server = wsgi.Server(CONF, name, self.app,
+        self.server = wsgi.Server(CONF, self.service_name, self.app,
                                   host=CONF.api.host,
                                   port=CONF.api.port,
                                   use_ssl=use_ssl,
-                                  logger_name=name)
+                                  logger_name=self.service_name)
 
     def start(self):
         """Start serving this service using loaded configuration"""
@@ -101,7 +108,53 @@ class WSGIService(service.ServiceBase):
         self.server.reset()
 
 
-class Service(service.ServiceBase, dispatcher.EventDispatcher):
+class ServiceHeartbeat(scheduling.BackgroundSchedulerService):
+
+    def __init__(self, gconfig=None, service_name=None, **kwargs):
+        gconfig = None or {}
+        super(ServiceHeartbeat, self).__init__(gconfig, **kwargs)
+        self.service_name = service_name
+        self.context = context.make_context()
+
+    def send_beat(self):
+        host = CONF.host
+        watcher_list = objects.Service.list(
+            self.context, filters={'name': self.service_name,
+                                   'host': host})
+        if watcher_list:
+            watcher_service = watcher_list[0]
+            watcher_service.last_seen_up = datetime.datetime.utcnow()
+            watcher_service.save()
+        else:
+            watcher_service = objects.Service(self.context)
+            watcher_service.name = self.service_name
+            watcher_service.host = host
+            watcher_service.create()
+
+    def add_heartbeat_job(self):
+        self.add_job(self.send_beat, 'interval', seconds=60,
+                     next_run_time=datetime.datetime.now())
+
+    def start(self):
+        """Start service."""
+        self.add_heartbeat_job()
+        super(ServiceHeartbeat, self).start()
+
+    def stop(self):
+        """Stop service."""
+        self.shutdown()
+
+    def wait(self):
+        """Wait for service to complete."""
+
+    def reset(self):
+        """Reset service.
+
+        Called in case service running in daemon mode receives SIGHUP.
+        """
+
+
+class Service(service.ServiceBase):
 
     API_VERSION = '1.0'
 
@@ -110,18 +163,14 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
         self.manager = manager_class()
 
         self.publisher_id = self.manager.publisher_id
-        self.api_version = self.manager.API_VERSION
+        self.api_version = self.manager.api_version
 
         self.conductor_topic = self.manager.conductor_topic
-        self.status_topic = self.manager.status_topic
         self.notification_topics = self.manager.notification_topics
 
         self.conductor_endpoints = [
             ep(self) for ep in self.manager.conductor_endpoints
         ]
-        self.status_endpoints = [
-            ep(self.publisher_id) for ep in self.manager.status_endpoints
-        ]
         self.notification_endpoints = self.manager.notification_endpoints
 
         self.serializer = rpc.RequestContextSerializer(
@@ -130,22 +179,23 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
         self._transport = None
         self._notification_transport = None
         self._conductor_client = None
-        self._status_client = None
 
         self.conductor_topic_handler = None
-        self.status_topic_handler = None
         self.notification_handler = None
 
+        self.heartbeat = None
+
         if self.conductor_topic and self.conductor_endpoints:
             self.conductor_topic_handler = self.build_topic_handler(
                 self.conductor_topic, self.conductor_endpoints)
-        if self.status_topic and self.status_endpoints:
-            self.status_topic_handler = self.build_topic_handler(
-                self.status_topic, self.status_endpoints)
         if self.notification_topics and self.notification_endpoints:
             self.notification_handler = self.build_notification_handler(
                 self.notification_topics, self.notification_endpoints
             )
+        self.service_name = self.manager.service_name
+        if self.service_name:
+            self.heartbeat = ServiceHeartbeat(
+                service_name=self.manager.service_name)
 
     @property
     def transport(self):
@@ -174,25 +224,17 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
     def conductor_client(self, c):
         self.conductor_client = c
 
-    @property
-    def status_client(self):
-        if self._status_client is None:
-            target = om.Target(
-                topic=self.status_topic,
-                version=self.API_VERSION,
-            )
-            self._status_client = om.RPCClient(
-                self.transport, target, serializer=self.serializer)
-        return self._status_client
-
-    @status_client.setter
-    def status_client(self, c):
-        self.status_client = c
-
     def build_topic_handler(self, topic_name, endpoints=()):
-        return messaging_handler.MessagingHandler(
-            self.publisher_id, topic_name, [self.manager] + list(endpoints),
-            self.api_version, self.serializer)
+        serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
+        target = om.Target(
+            topic=topic_name,
+            # For compatibility, we can override it with 'host' opt
+            server=CONF.host or socket.gethostname(),
+            version=self.api_version,
+        )
+        return om.get_rpc_server(
+            self.transport, target, endpoints,
+            executor='eventlet', serializer=serializer)
 
     def build_notification_handler(self, topic_names, endpoints=()):
         serializer = rpc.RequestContextSerializer(rpc.JsonPayloadSerializer())
@@ -207,20 +249,20 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
                   CONF.transport_url, CONF.rpc_backend)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.start()
-        if self.status_topic_handler:
-            self.status_topic_handler.start()
         if self.notification_handler:
             self.notification_handler.start()
+        if self.heartbeat:
+            self.heartbeat.start()
 
     def stop(self):
         LOG.debug("Disconnecting from '%s' (%s)",
                   CONF.transport_url, CONF.rpc_backend)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.stop()
-        if self.status_topic_handler:
-            self.status_topic_handler.stop()
         if self.notification_handler:
             self.notification_handler.stop()
+        if self.heartbeat:
+            self.heartbeat.stop()
 
     def reset(self):
         """Reset a service in case it received a SIGHUP."""
@@ -228,34 +270,11 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
     def wait(self):
         """Wait for service to complete."""
 
-    def publish_control(self, event, payload):
-        return self.conductor_topic_handler.publish_event(event, payload)
-
-    def publish_status_event(self, event, payload, request_id=None):
-        if self.status_topic_handler:
-            return self.status_topic_handler.publish_event(
-                event, payload, request_id)
-        else:
-            LOG.info(
-                _LI("No status notifier declared: notification '%s' not sent"),
-                event)
-
-    def get_version(self):
-        return self.api_version
-
-    def check_api_version(self, context):
+    def check_api_version(self, ctx):
         api_manager_version = self.conductor_client.call(
-            context.to_dict(), 'check_api_version',
-            api_version=self.api_version)
+            ctx, 'check_api_version', api_version=self.api_version)
         return api_manager_version
 
-    def response(self, evt, ctx, message):
-        payload = {
-            'request_id': ctx['request_id'],
-            'msg': message
-        }
-        self.publish_status_event(evt, payload)
-
 
 def launch(conf, service_, workers=1, restart_method='reload'):
     return service.launch(conf, service_, workers, restart_method)
@@ -269,7 +288,9 @@ def prepare_service(argv=(), conf=cfg.CONF):
     cfg.set_defaults(_options.log_opts,
                      default_log_levels=_DEFAULT_LOG_LEVELS)
     log.setup(conf, 'python-watcher')
-    conf.log_opt_values(LOG, logging.DEBUG)
+    conf.log_opt_values(LOG, log.DEBUG)
+    objects.register_all()
 
-    gmr.TextGuruMeditation.register_section(_('Plugins'), opts.show_plugins)
-    gmr.TextGuruMeditation.setup_autorun(version)
+    gmr.TextGuruMeditation.register_section(
+        _('Plugins'), plugins_conf.show_plugins)
+    gmr.TextGuruMeditation.setup_autorun(version, conf=conf)

watcher/common/service_manager.py

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2016 Servionica
+##
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ServiceManager(object):
+
+    @abc.abstractproperty
+    def service_name(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def api_version(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def publisher_id(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def conductor_topic(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def notification_topics(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def conductor_endpoints(self):
+        raise NotImplementedError()
+
+    @abc.abstractproperty
+    def notification_endpoints(self):
+        raise NotImplementedError()

watcher/common/utils.py

@@ -16,17 +16,19 @@
 
 """Utilities and helper functions."""
 
+import re
+
 from jsonschema import validators
 from oslo_config import cfg
 from oslo_log import log as logging
-from watcher.common import exception
-
-import re
+from oslo_utils import strutils
+from oslo_utils import timeutils
+from oslo_utils import uuidutils
 import six
-import uuid
-
 
 from watcher._i18n import _LW
+from watcher.common import exception
+
 
 UTILS_OPTS = [
     cfg.StrOpt('rootwrap_config',
@@ -66,6 +68,12 @@ class Struct(dict):
             raise AttributeError(name)
 
 
+generate_uuid = uuidutils.generate_uuid
+is_uuid_like = uuidutils.is_uuid_like
+is_int_like = strutils.is_int_like
+strtime = timeutils.strtime
+
+
 def safe_rstrip(value, chars=None):
     """Removes trailing characters from a string if that does not make it empty
 
@@ -83,31 +91,6 @@ def safe_rstrip(value, chars=None):
     return value.rstrip(chars) or value
 
 
-def generate_uuid():
-    return str(uuid.uuid4())
-
-
-def is_int_like(val):
-    """Check if a value looks like an int."""
-    try:
-        return str(int(val)) == str(val)
-    except Exception:
-        return False
-
-
-def is_uuid_like(val):
-    """Returns validation of a value as a UUID.
-
-    For our purposes, a UUID is a canonical form string:
-    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
-
-    """
-    try:
-        return str(uuid.UUID(val)) == val
-    except (TypeError, ValueError, AttributeError):
-        return False
-
-
 def is_hostname_safe(hostname):
     """Determine if the supplied hostname is RFC compliant.
 
@@ -133,10 +116,6 @@ def get_cls_import_path(cls):
     return module + '.' + cls.__name__
 
 
-def strtime(at):
-    return at.strftime("%Y-%m-%dT%H:%M:%S.%f")
-
-
 # Default value feedback extension as jsonschema doesn't support it
 def extend_with_default(validator_class):
     validate_properties = validator_class.VALIDATORS["properties"]
@@ -173,3 +152,5 @@ def extend_with_strict_schema(validator_class):
 
 StrictDefaultValidatingDraft4Validator = extend_with_default(
     extend_with_strict_schema(validators.Draft4Validator))
+
+Draft4Validator = validators.Draft4Validator

watcher/conf/__init__.py

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
+# Copyright (c) 2016 b<>com
 #
-# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,12 +15,11 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
-import enum
+from oslo_config import cfg
 
+from watcher.conf import service
 
-class Events(enum.Enum):
-    ALL = '*',
-    ACTION_PLAN = "action_plan"
-    TRIGGER_AUDIT = "trigger_audit"
+CONF = cfg.CONF
+
+service.register_opts(CONF)

watcher/conf/_opts.py

@@ -0,0 +1,56 @@
+# -*- encoding: utf-8 -*-
+# Copyright 2014
+# The Cloudscaling Group, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneauth1 import loading as ka_loading
+
+from watcher.api import acl as api_acl
+from watcher.api import app as api_app
+from watcher.applier import manager as applier_manager
+from watcher.common import clients
+from watcher.common import exception
+from watcher.common import paths
+from watcher.db.sqlalchemy import models
+from watcher.decision_engine.audit import continuous
+from watcher.decision_engine import manager as decision_engine_manager
+from watcher.decision_engine.planner import manager as planner_manager
+
+
+def list_opts():
+    """Legacy aggregation of all the watcher config options"""
+    return [
+        ('DEFAULT',
+         (api_app.API_SERVICE_OPTS +
+          api_acl.AUTH_OPTS +
+          exception.EXC_LOG_OPTS +
+          paths.PATH_OPTS)),
+        ('api', api_app.API_SERVICE_OPTS),
+        ('database', models.SQL_OPTS),
+        ('watcher_decision_engine',
+         (decision_engine_manager.WATCHER_DECISION_ENGINE_OPTS +
+          continuous.WATCHER_CONTINUOUS_OPTS)),
+        ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS),
+        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS),
+        ('nova_client', clients.NOVA_CLIENT_OPTS),
+        ('glance_client', clients.GLANCE_CLIENT_OPTS),
+        ('cinder_client', clients.CINDER_CLIENT_OPTS),
+        ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS),
+        ('neutron_client', clients.NEUTRON_CLIENT_OPTS),
+        ('watcher_clients_auth',
+         (ka_loading.get_auth_common_conf_options() +
+          ka_loading.get_auth_plugin_conf_options('password') +
+          ka_loading.get_session_conf_options()))
+    ]

watcher/conf/opts.py

@@ -0,0 +1,95 @@
+# Copyright 2016 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This is the single point of entry to generate the sample configuration
+file for Watcher. It collects all the necessary info from the other modules
+in this package. It is assumed that:
+
+* every other module in this package has a 'list_opts' function which
+  return a dict where
+  * the keys are strings which are the group names
+  * the value of each key is a list of config options for that group
+* the watcher.conf package doesn't have further packages with config options
+* this module is only used in the context of sample file generation
+"""
+
+import collections
+import importlib
+import os
+import pkgutil
+
+LIST_OPTS_FUNC_NAME = "list_opts"
+
+
+def _tupleize(dct):
+    """Take the dict of options and convert to the 2-tuple format."""
+    return [(key, val) for key, val in dct.items()]
+
+
+def list_opts():
+    """Grouped list of all the Watcher-specific configuration options
+
+    :return: A list of ``(group, [opt_1, opt_2])`` tuple pairs, where ``group``
+             is either a group name as a string or an OptGroup object.
+    """
+    opts = collections.defaultdict(list)
+    module_names = _list_module_names()
+    imported_modules = _import_modules(module_names)
+    _append_config_options(imported_modules, opts)
+    return _tupleize(opts)
+
+
+def _list_module_names():
+    module_names = []
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    for __, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
+        if modname == "opts" or ispkg:
+            continue
+        else:
+            module_names.append(modname)
+    return module_names
+
+
+def _import_modules(module_names):
+    imported_modules = []
+    for modname in module_names:
+        mod = importlib.import_module("watcher.conf." + modname)
+        if not hasattr(mod, LIST_OPTS_FUNC_NAME):
+            msg = "The module 'watcher.conf.%s' should have a '%s' "\
+                  "function which returns the config options." % \
+                  (modname, LIST_OPTS_FUNC_NAME)
+            raise Exception(msg)
+        else:
+            imported_modules.append(mod)
+    return imported_modules
+
+
+def _process_old_opts(configs):
+    """Convert old-style 2-tuple configs to dicts."""
+    if isinstance(configs, tuple):
+        configs = [configs]
+    return {label: options for label, options in configs}
+
+
+def _append_config_options(imported_modules, config_options):
+    for mod in imported_modules:
+        configs = mod.list_opts()
+        # TODO(markus_z): Remove this compatibility shim once all list_opts()
+        # functions have been updated to return dicts.
+        if not isinstance(configs, dict):
+            configs = _process_old_opts(configs)
+        for key, val in configs.items():
+            config_options[key].extend(val)

watcher/conf/plugins.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2014
-# The Cloudscaling Group, Inc.
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,18 +16,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keystoneauth1 import loading as ka_loading
 import prettytable as ptable
 
-import watcher.api.app
 from watcher.applier.loading import default as applier_loader
-from watcher.applier import manager as applier_manager
-from watcher.common import clients
 from watcher.common import utils
 from watcher.decision_engine.loading import default as decision_engine_loader
-from watcher.decision_engine import manager as decision_engine_manger
-from watcher.decision_engine.planner import manager as planner_manager
-
 
 PLUGIN_LOADERS = (
     applier_loader.DefaultActionLoader,
@@ -40,29 +34,7 @@ PLUGIN_LOADERS = (
 
 
 def list_opts():
-    watcher_opts = [
-        ('api', watcher.api.app.API_SERVICE_OPTS),
-        ('watcher_decision_engine',
-         decision_engine_manger.WATCHER_DECISION_ENGINE_OPTS),
-        ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS),
-        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS),
-        ('nova_client', clients.NOVA_CLIENT_OPTS),
-        ('glance_client', clients.GLANCE_CLIENT_OPTS),
-        ('cinder_client', clients.CINDER_CLIENT_OPTS),
-        ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS),
-        ('neutron_client', clients.NEUTRON_CLIENT_OPTS),
-        ('watcher_clients_auth',
-         (ka_loading.get_auth_common_conf_options() +
-          ka_loading.get_auth_plugin_conf_options('password') +
-          ka_loading.get_session_conf_options()))
-    ]
-
-    watcher_opts += list_plugin_opts()
-
-    return watcher_opts
-
-
-def list_plugin_opts():
+    """Load config options for all Watcher plugins"""
     plugins_opts = []
     for plugin_loader_cls in PLUGIN_LOADERS:
         plugin_loader = plugin_loader_cls()

watcher/conf/service.py

@@ -0,0 +1,49 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+
+from oslo_config import cfg
+
+from watcher._i18n import _
+
+SERVICE_OPTS = [
+    cfg.IntOpt('periodic_interval',
+               default=60,
+               help=_('Seconds between running periodic tasks.')),
+    cfg.StrOpt('host',
+               default=socket.gethostname(),
+               help=_('Name of this node. This can be an opaque identifier. '
+                      'It is not necessarily a hostname, FQDN, or IP address. '
+                      'However, the node name must be valid within '
+                      'an AMQP key, and if using ZeroMQ, a valid '
+                      'hostname, FQDN, or IP address.')),
+    cfg.IntOpt('service_down_time',
+               default=90,
+               help=_('Maximum time since last check-in for up service.'))
+]
+
+
+def register_opts(conf):
+    conf.register_opts(SERVICE_OPTS)
+
+
+def list_opts():
+    return [
+        ('DEFAULT', SERVICE_OPTS),
+    ]

watcher/db/api.py

@@ -36,7 +36,7 @@ class BaseConnection(object):
 
     @abc.abstractmethod
     def get_goal_list(self, context, filters=None, limit=None,
-                      marker=None, sort_key=None, sort_dir=None):
+                      marker=None, sort_key=None, sort_dir=None, eager=False):
         """Get specific columns for matching goals.
 
         Return a list of the specified columns for all goals that
@@ -50,6 +50,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -72,31 +73,34 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def get_goal_by_id(self, context, goal_id):
+    def get_goal_by_id(self, context, goal_id, eager=False):
         """Return a goal given its ID.
 
         :param context: The security context
         :param goal_id: The ID of a goal
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A goal
         :raises: :py:class:`~.GoalNotFound`
         """
 
     @abc.abstractmethod
-    def get_goal_by_uuid(self, context, goal_uuid):
+    def get_goal_by_uuid(self, context, goal_uuid, eager=False):
         """Return a goal given its UUID.
 
         :param context: The security context
         :param goal_uuid: The UUID of a goal
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A goal
         :raises: :py:class:`~.GoalNotFound`
         """
 
     @abc.abstractmethod
-    def get_goal_by_name(self, context, goal_name):
+    def get_goal_by_name(self, context, goal_name, eager=False):
         """Return a goal given its name.
 
         :param context: The security context
         :param goal_name: The name of a goal
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A goal
         :raises: :py:class:`~.GoalNotFound`
         """
@@ -129,9 +133,17 @@ class BaseConnection(object):
         :raises: :py:class:`~.Invalid`
         """
 
+    def soft_delete_goal(self, goal_id):
+        """Soft delete a goal.
+
+        :param goal_id: The id or uuid of a goal.
+        :raises: :py:class:`~.GoalNotFound`
+        """
+
     @abc.abstractmethod
     def get_strategy_list(self, context, filters=None, limit=None,
-                          marker=None, sort_key=None, sort_dir=None):
+                          marker=None, sort_key=None, sort_dir=None,
+                          eager=True):
         """Get specific columns for matching strategies.
 
         Return a list of the specified columns for all strategies that
@@ -146,6 +158,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: Direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -170,31 +183,34 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def get_strategy_by_id(self, context, strategy_id):
+    def get_strategy_by_id(self, context, strategy_id, eager=False):
         """Return a strategy given its ID.
 
         :param context: The security context
         :param strategy_id: The ID of a strategy
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A strategy
         :raises: :py:class:`~.StrategyNotFound`
         """
 
     @abc.abstractmethod
-    def get_strategy_by_uuid(self, context, strategy_uuid):
+    def get_strategy_by_uuid(self, context, strategy_uuid, eager=False):
         """Return a strategy given its UUID.
 
         :param context: The security context
         :param strategy_uuid: The UUID of a strategy
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A strategy
         :raises: :py:class:`~.StrategyNotFound`
         """
 
     @abc.abstractmethod
-    def get_strategy_by_name(self, context, strategy_name):
+    def get_strategy_by_name(self, context, strategy_name, eager=False):
         """Return a strategy given its name.
 
         :param context: The security context
         :param strategy_name: The name of a strategy
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A strategy
         :raises: :py:class:`~.StrategyNotFound`
         """
@@ -217,10 +233,17 @@ class BaseConnection(object):
         :raises: :py:class:`~.Invalid`
         """
 
+    def soft_delete_strategy(self, strategy_id):
+        """Soft delete a strategy.
+
+        :param strategy_id: The id or uuid of a strategy.
+        :raises: :py:class:`~.StrategyNotFound`
+        """
+
     @abc.abstractmethod
     def get_audit_template_list(self, context, filters=None,
                                 limit=None, marker=None, sort_key=None,
-                                sort_dir=None):
+                                sort_dir=None, eager=False):
         """Get specific columns for matching audit templates.
 
         Return a list of the specified columns for all audit templates that
@@ -234,6 +257,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -250,46 +274,50 @@ class BaseConnection(object):
                          'uuid': utils.generate_uuid(),
                          'name': 'example',
                          'description': 'free text description'
-                         'host_aggregate': 'nova aggregate name or id'
                          'goal': 'DUMMY'
-                         'extra': {'automatic': True}
                         }
         :returns: An audit template.
         :raises: :py:class:`~.AuditTemplateAlreadyExists`
         """
 
     @abc.abstractmethod
-    def get_audit_template_by_id(self, context, audit_template_id):
+    def get_audit_template_by_id(self, context, audit_template_id,
+                                 eager=False):
         """Return an audit template.
 
         :param context: The security context
         :param audit_template_id: The id of an audit template.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An audit template.
         :raises: :py:class:`~.AuditTemplateNotFound`
         """
 
     @abc.abstractmethod
-    def get_audit_template_by_uuid(self, context, audit_template_uuid):
+    def get_audit_template_by_uuid(self, context, audit_template_uuid,
+                                   eager=False):
         """Return an audit template.
 
         :param context: The security context
         :param audit_template_uuid: The uuid of an audit template.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An audit template.
         :raises: :py:class:`~.AuditTemplateNotFound`
         """
 
-    def get_audit_template_by_name(self, context, audit_template_name):
+    def get_audit_template_by_name(self, context, audit_template_name,
+                                   eager=False):
         """Return an audit template.
 
         :param context: The security context
         :param audit_template_name: The name of an audit template.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An audit template.
         :raises: :py:class:`~.AuditTemplateNotFound`
         """
 
     @abc.abstractmethod
     def destroy_audit_template(self, audit_template_id):
-        """Destroy an audit_template.
+        """Destroy an audit template.
 
         :param audit_template_id: The id or uuid of an audit template.
         :raises: :py:class:`~.AuditTemplateNotFound`
@@ -307,7 +335,7 @@ class BaseConnection(object):
 
     @abc.abstractmethod
     def soft_delete_audit_template(self, audit_template_id):
-        """Soft delete an audit_template.
+        """Soft delete an audit template.
 
         :param audit_template_id: The id or uuid of an audit template.
         :raises: :py:class:`~.AuditTemplateNotFound`
@@ -315,7 +343,7 @@ class BaseConnection(object):
 
     @abc.abstractmethod
     def get_audit_list(self, context, filters=None, limit=None,
-                       marker=None, sort_key=None, sort_dir=None):
+                       marker=None, sort_key=None, sort_dir=None, eager=False):
         """Get specific columns for matching audits.
 
         Return a list of the specified columns for all audits that match the
@@ -329,6 +357,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -345,28 +374,29 @@ class BaseConnection(object):
                         {
                          'uuid': utils.generate_uuid(),
                          'type': 'ONESHOT',
-                         'deadline': None
                         }
         :returns: An audit.
         :raises: :py:class:`~.AuditAlreadyExists`
         """
 
     @abc.abstractmethod
-    def get_audit_by_id(self, context, audit_id):
+    def get_audit_by_id(self, context, audit_id, eager=False):
         """Return an audit.
 
         :param context: The security context
         :param audit_id: The id of an audit.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An audit.
         :raises: :py:class:`~.AuditNotFound`
         """
 
     @abc.abstractmethod
-    def get_audit_by_uuid(self, context, audit_uuid):
+    def get_audit_by_uuid(self, context, audit_uuid, eager=False):
         """Return an audit.
 
         :param context: The security context
         :param audit_uuid: The uuid of an audit.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An audit.
         :raises: :py:class:`~.AuditNotFound`
         """
@@ -393,13 +423,13 @@ class BaseConnection(object):
         """Soft delete an audit and all associated action plans.
 
         :param audit_id: The id or uuid of an audit.
-        :returns: An audit.
         :raises: :py:class:`~.AuditNotFound`
         """
 
     @abc.abstractmethod
     def get_action_list(self, context, filters=None, limit=None,
-                        marker=None, sort_key=None, sort_dir=None):
+                        marker=None, sort_key=None, sort_dir=None,
+                        eager=False):
         """Get specific columns for matching actions.
 
         Return a list of the specified columns for all actions that match the
@@ -413,6 +443,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -437,21 +468,23 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def get_action_by_id(self, context, action_id):
+    def get_action_by_id(self, context, action_id, eager=False):
         """Return a action.
 
         :param context: The security context
         :param action_id: The id of a action.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A action.
         :raises: :py:class:`~.ActionNotFound`
         """
 
     @abc.abstractmethod
-    def get_action_by_uuid(self, context, action_uuid):
+    def get_action_by_uuid(self, context, action_uuid, eager=False):
         """Return a action.
 
         :param context: The security context
         :param action_uuid: The uuid of a action.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A action.
         :raises: :py:class:`~.ActionNotFound`
         """
@@ -476,10 +509,17 @@ class BaseConnection(object):
         :raises: :py:class:`~.Invalid`
         """
 
+    def soft_delete_action(self, action_id):
+        """Soft delete an action.
+
+        :param action_id: The id or uuid of an action.
+        :raises: :py:class:`~.ActionNotFound`
+        """
+
     @abc.abstractmethod
     def get_action_plan_list(
             self, context, filters=None, limit=None,
-            marker=None, sort_key=None, sort_dir=None):
+            marker=None, sort_key=None, sort_dir=None, eager=False):
         """Get specific columns for matching action plans.
 
         Return a list of the specified columns for all action plans that
@@ -493,6 +533,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -507,21 +548,23 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def get_action_plan_by_id(self, context, action_plan_id):
+    def get_action_plan_by_id(self, context, action_plan_id, eager=False):
         """Return an action plan.
 
         :param context: The security context
         :param action_plan_id: The id of an action plan.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An action plan.
         :raises: :py:class:`~.ActionPlanNotFound`
         """
 
     @abc.abstractmethod
-    def get_action_plan_by_uuid(self, context, action_plan__uuid):
+    def get_action_plan_by_uuid(self, context, action_plan__uuid, eager=False):
         """Return a action plan.
 
         :param context: The security context
         :param action_plan__uuid: The uuid of an action plan.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An action plan.
         :raises: :py:class:`~.ActionPlanNotFound`
         """
@@ -546,9 +589,17 @@ class BaseConnection(object):
         :raises: :py:class:`~.Invalid`
         """
 
+    def soft_delete_action_plan(self, action_plan_id):
+        """Soft delete an action plan.
+
+        :param action_plan_id: The id or uuid of an action plan.
+        :raises: :py:class:`~.ActionPlanNotFound`
+        """
+
     @abc.abstractmethod
     def get_efficacy_indicator_list(self, context, filters=None, limit=None,
-                                    marker=None, sort_key=None, sort_dir=None):
+                                    marker=None, sort_key=None, sort_dir=None,
+                                    eager=False):
         """Get specific columns for matching efficacy indicators.
 
         Return a list of the specified columns for all efficacy indicators that
@@ -565,6 +616,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: Direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -589,31 +641,37 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def get_efficacy_indicator_by_id(self, context, efficacy_indicator_id):
+    def get_efficacy_indicator_by_id(self, context, efficacy_indicator_id,
+                                     eager=False):
         """Return an efficacy indicator given its ID.
 
         :param context: The security context
         :param efficacy_indicator_id: The ID of an efficacy indicator
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An efficacy indicator
         :raises: :py:class:`~.EfficacyIndicatorNotFound`
         """
 
     @abc.abstractmethod
-    def get_efficacy_indicator_by_uuid(self, context, efficacy_indicator_uuid):
+    def get_efficacy_indicator_by_uuid(self, context, efficacy_indicator_uuid,
+                                       eager=False):
         """Return an efficacy indicator given its UUID.
 
         :param context: The security context
         :param efficacy_indicator_uuid: The UUID of an efficacy indicator
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An efficacy indicator
         :raises: :py:class:`~.EfficacyIndicatorNotFound`
         """
 
     @abc.abstractmethod
-    def get_efficacy_indicator_by_name(self, context, efficacy_indicator_name):
+    def get_efficacy_indicator_by_name(self, context, efficacy_indicator_name,
+                                       eager=False):
         """Return an efficacy indicator given its name.
 
         :param context: The security context
         :param efficacy_indicator_name: The name of an efficacy indicator
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: An efficacy indicator
         :raises: :py:class:`~.EfficacyIndicatorNotFound`
         """
@@ -627,7 +685,7 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def update_efficacy_indicator(self, efficacy_indicator_uuid, values):
+    def update_efficacy_indicator(self, efficacy_indicator_id, values):
         """Update properties of an efficacy indicator.
 
         :param efficacy_indicator_uuid: The UUID of an efficacy indicator
@@ -639,7 +697,7 @@ class BaseConnection(object):
     @abc.abstractmethod
     def get_scoring_engine_list(
             self, context, columns=None, filters=None, limit=None,
-            marker=None, sort_key=None, sort_dir=None):
+            marker=None, sort_key=None, sort_dir=None, eager=False):
         """Get specific columns for matching scoring engines.
 
         Return a list of the specified columns for all scoring engines that
@@ -655,6 +713,7 @@ class BaseConnection(object):
         :param sort_key: Attribute by which results should be sorted.
         :param sort_dir: direction in which results should be sorted.
                          (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A list of tuples of the specified columns.
         """
 
@@ -669,31 +728,37 @@ class BaseConnection(object):
         """
 
     @abc.abstractmethod
-    def get_scoring_engine_by_id(self, context, scoring_engine_id):
+    def get_scoring_engine_by_id(self, context, scoring_engine_id,
+                                 eager=False):
         """Return a scoring engine by its id.
 
         :param context: The security context
         :param scoring_engine_id: The id of a scoring engine.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A scoring engine.
         :raises: :py:class:`~.ScoringEngineNotFound`
         """
 
     @abc.abstractmethod
-    def get_scoring_engine_by_uuid(self, context, scoring_engine_uuid):
+    def get_scoring_engine_by_uuid(self, context, scoring_engine_uuid,
+                                   eager=False):
         """Return a scoring engine by its uuid.
 
         :param context: The security context
         :param scoring_engine_uuid: The uuid of a scoring engine.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A scoring engine.
         :raises: :py:class:`~.ScoringEngineNotFound`
         """
 
     @abc.abstractmethod
-    def get_scoring_engine_by_name(self, context, scoring_engine_name):
+    def get_scoring_engine_by_name(self, context, scoring_engine_name,
+                                   eager=False):
         """Return a scoring engine by its name.
 
         :param context: The security context
         :param scoring_engine_name: The name of a scoring engine.
+        :param eager: If True, also loads One-to-X data (Default: False)
         :returns: A scoring engine.
         :raises: :py:class:`~.ScoringEngineNotFound`
         """
@@ -715,3 +780,92 @@ class BaseConnection(object):
         :raises: :py:class:`~.ScoringEngineNotFound`
         :raises: :py:class:`~.Invalid`
         """
+
+    @abc.abstractmethod
+    def get_service_list(self, context, filters=None, limit=None, marker=None,
+                         sort_key=None, sort_dir=None, eager=False):
+        """Get specific columns for matching services.
+
+        Return a list of the specified columns for all services that
+        match the specified filters.
+
+        :param context: The security context
+        :param filters: Filters to apply. Defaults to None.
+
+        :param limit: Maximum number of services to return.
+        :param marker: The last item of the previous page; we return the next
+                       result set.
+        :param sort_key: Attribute by which results should be sorted.
+        :param sort_dir: Direction in which results should be sorted.
+                         (asc, desc)
+        :param eager: If True, also loads One-to-X data (Default: False)
+        :returns: A list of tuples of the specified columns.
+        """
+
+    @abc.abstractmethod
+    def create_service(self, values):
+        """Create a new service.
+
+        :param values: A dict containing items used to identify
+                       and track the service. For example:
+
+                       ::
+
+                        {
+                         'id': 1,
+                         'name': 'watcher-api',
+                         'status': 'ACTIVE',
+                         'host': 'controller'
+                        }
+        :returns: A service
+        :raises: :py:class:`~.ServiceAlreadyExists`
+        """
+
+    @abc.abstractmethod
+    def get_service_by_id(self, context, service_id, eager=False):
+        """Return a service given its ID.
+
+        :param context: The security context
+        :param service_id: The ID of a service
+        :param eager: If True, also loads One-to-X data (Default: False)
+        :returns: A service
+        :raises: :py:class:`~.ServiceNotFound`
+        """
+
+    @abc.abstractmethod
+    def get_service_by_name(self, context, service_name, eager=False):
+        """Return a service given its name.
+
+        :param context: The security context
+        :param service_name: The name of a service
+        :param eager: If True, also loads One-to-X data (Default: False)
+        :returns: A service
+        :raises: :py:class:`~.ServiceNotFound`
+        """
+
+    @abc.abstractmethod
+    def destroy_service(self, service_id):
+        """Destroy a service.
+
+        :param service_id: The ID of a service
+        :raises: :py:class:`~.ServiceNotFound`
+        """
+
+    @abc.abstractmethod
+    def update_service(self, service_id, values):
+        """Update properties of a service.
+
+        :param service_id: The ID of a service
+        :returns: A service
+        :raises: :py:class:`~.ServiceyNotFound`
+        :raises: :py:class:`~.Invalid`
+        """
+
+    @abc.abstractmethod
+    def soft_delete_service(self, service_id):
+        """Soft delete a service.
+
+        :param service_id: The id of a service.
+        :returns: A service.
+        :raises: :py:class:`~.ServiceNotFound`
+        """

watcher/db/sqlalchemy/api.py

@@ -24,17 +24,17 @@ from oslo_config import cfg
 from oslo_db import exception as db_exc
 from oslo_db.sqlalchemy import session as db_session
 from oslo_db.sqlalchemy import utils as db_utils
+from oslo_utils import timeutils
+from sqlalchemy.inspection import inspect
 from sqlalchemy.orm import exc
+from sqlalchemy.orm import joinedload
 
 from watcher._i18n import _
 from watcher.common import exception
 from watcher.common import utils
 from watcher.db import api
 from watcher.db.sqlalchemy import models
-from watcher.objects import action as action_objects
-from watcher.objects import action_plan as ap_objects
-from watcher.objects import audit as audit_objects
-from watcher.objects import utils as objutils
+from watcher import objects
 
 CONF = cfg.CONF
 
@@ -68,7 +68,6 @@ def model_query(model, *args, **kwargs):
 
     :param session: if present, the session to use
     """
-
     session = kwargs.get('session') or get_session()
     query = session.query(model, *args)
     return query
@@ -133,8 +132,9 @@ class Connection(api.BaseConnection):
     def __add_simple_filter(self, query, model, fieldname, value, operator_):
         field = getattr(model, fieldname)
 
-        if field.type.python_type is datetime.datetime:
-            value = objutils.datetime_or_str_or_none(value)
+        if field.type.python_type is datetime.datetime and value:
+            if not isinstance(value, datetime.datetime):
+                value = timeutils.parse_isotime(value)
 
         return query.filter(self.valid_operators[operator_](field, value))
 
@@ -233,8 +233,32 @@ class Connection(api.BaseConnection):
 
         return query
 
-    def _get(self, context, model, fieldname, value):
+    @staticmethod
+    def _get_relationships(model):
+        return inspect(model).relationships
+
+    @staticmethod
+    def _set_eager_options(model, query):
+        relationships = inspect(model).relationships
+        for relationship in relationships:
+            if not relationship.uselist:
+                # We have a One-to-X relationship
+                query = query.options(joinedload(relationship.key))
+        return query
+
+    def _create(self, model, values):
+        obj = model()
+        cleaned_values = {k: v for k, v in values.items()
+                          if k not in self._get_relationships(model)}
+        obj.update(cleaned_values)
+        obj.save()
+        return obj
+
+    def _get(self, context, model, fieldname, value, eager):
         query = model_query(model)
+        if eager:
+            query = self._set_eager_options(model, query)
+
         query = query.filter(getattr(model, fieldname) == value)
         if not context.show_deleted:
             query = query.filter(model.deleted_at.is_(None))
@@ -246,7 +270,8 @@ class Connection(api.BaseConnection):
 
         return obj
 
-    def _update(self, model, id_, values):
+    @staticmethod
+    def _update(model, id_, values):
         session = get_session()
         with session.begin():
             query = model_query(model, session=session)
@@ -259,7 +284,8 @@ class Connection(api.BaseConnection):
             ref.update(values)
         return ref
 
-    def _soft_delete(self, model, id_):
+    @staticmethod
+    def _soft_delete(model, id_):
         session = get_session()
         with session.begin():
             query = model_query(model, session=session)
@@ -271,7 +297,8 @@ class Connection(api.BaseConnection):
 
             query.soft_delete()
 
-    def _destroy(self, model, id_):
+    @staticmethod
+    def _destroy(model, id_):
         session = get_session()
         with session.begin():
             query = model_query(model, session=session)
@@ -309,8 +336,7 @@ class Connection(api.BaseConnection):
         if filters is None:
             filters = {}
 
-        plain_fields = ['uuid', 'name', 'host_aggregate',
-                        'goal_id', 'strategy_id']
+        plain_fields = ['uuid', 'name', 'goal_id', 'strategy_id']
         join_fieldmap = JoinMap(
             goal_uuid=NaturalJoinFilter(
                 join_fieldname="uuid", join_model=models.Goal),
@@ -399,10 +425,11 @@ class Connection(api.BaseConnection):
 
     # ### GOALS ### #
 
-    def get_goal_list(self, context, filters=None, limit=None,
-                      marker=None, sort_key=None, sort_dir=None):
-
+    def get_goal_list(self, context, filters=None, limit=None, marker=None,
+                      sort_key=None, sort_dir=None, eager=False):
         query = model_query(models.Goal)
+        if eager:
+            query = self._set_eager_options(models.Goal, query)
         query = self._add_goals_filters(query, filters)
         if not context.show_deleted:
             query = query.filter_by(deleted_at=None)
@@ -414,30 +441,30 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        goal = models.Goal()
-        goal.update(values)
-
         try:
-            goal.save()
+            goal = self._create(models.Goal, values)
         except db_exc.DBDuplicateEntry:
             raise exception.GoalAlreadyExists(uuid=values['uuid'])
         return goal
 
-    def _get_goal(self, context, fieldname, value):
+    def _get_goal(self, context, fieldname, value, eager):
         try:
             return self._get(context, model=models.Goal,
-                             fieldname=fieldname, value=value)
+                             fieldname=fieldname, value=value, eager=eager)
         except exception.ResourceNotFound:
             raise exception.GoalNotFound(goal=value)
 
-    def get_goal_by_id(self, context, goal_id):
-        return self._get_goal(context, fieldname="id", value=goal_id)
+    def get_goal_by_id(self, context, goal_id, eager=False):
+        return self._get_goal(
+            context, fieldname="id", value=goal_id, eager=eager)
 
-    def get_goal_by_uuid(self, context, goal_uuid):
-        return self._get_goal(context, fieldname="uuid", value=goal_uuid)
+    def get_goal_by_uuid(self, context, goal_uuid, eager=False):
+        return self._get_goal(
+            context, fieldname="uuid", value=goal_uuid, eager=eager)
 
-    def get_goal_by_name(self, context, goal_name):
-        return self._get_goal(context, fieldname="name", value=goal_name)
+    def get_goal_by_name(self, context, goal_name, eager=False):
+        return self._get_goal(
+            context, fieldname="name", value=goal_name, eager=eager)
 
     def destroy_goal(self, goal_id):
         try:
@@ -464,9 +491,11 @@ class Connection(api.BaseConnection):
     # ### STRATEGIES ### #
 
     def get_strategy_list(self, context, filters=None, limit=None,
-                          marker=None, sort_key=None, sort_dir=None):
-
+                          marker=None, sort_key=None, sort_dir=None,
+                          eager=True):
         query = model_query(models.Strategy)
+        if eager:
+            query = self._set_eager_options(models.Strategy, query)
         query = self._add_strategies_filters(query, filters)
         if not context.show_deleted:
             query = query.filter_by(deleted_at=None)
@@ -478,32 +507,30 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        strategy = models.Strategy()
-        strategy.update(values)
-
         try:
-            strategy.save()
+            strategy = self._create(models.Strategy, values)
         except db_exc.DBDuplicateEntry:
             raise exception.StrategyAlreadyExists(uuid=values['uuid'])
         return strategy
 
-    def _get_strategy(self, context, fieldname, value):
+    def _get_strategy(self, context, fieldname, value, eager):
         try:
             return self._get(context, model=models.Strategy,
-                             fieldname=fieldname, value=value)
+                             fieldname=fieldname, value=value, eager=eager)
         except exception.ResourceNotFound:
             raise exception.StrategyNotFound(strategy=value)
 
-    def get_strategy_by_id(self, context, strategy_id):
-        return self._get_strategy(context, fieldname="id", value=strategy_id)
-
-    def get_strategy_by_uuid(self, context, strategy_uuid):
+    def get_strategy_by_id(self, context, strategy_id, eager=False):
         return self._get_strategy(
-            context, fieldname="uuid", value=strategy_uuid)
+            context, fieldname="id", value=strategy_id, eager=eager)
 
-    def get_strategy_by_name(self, context, strategy_name):
+    def get_strategy_by_uuid(self, context, strategy_uuid, eager=False):
         return self._get_strategy(
-            context, fieldname="name", value=strategy_name)
+            context, fieldname="uuid", value=strategy_uuid, eager=eager)
+
+    def get_strategy_by_name(self, context, strategy_name, eager=False):
+        return self._get_strategy(
+            context, fieldname="name", value=strategy_name, eager=eager)
 
     def destroy_strategy(self, strategy_id):
         try:
@@ -530,9 +557,12 @@ class Connection(api.BaseConnection):
     # ### AUDIT TEMPLATES ### #
 
     def get_audit_template_list(self, context, filters=None, limit=None,
-                                marker=None, sort_key=None, sort_dir=None):
+                                marker=None, sort_key=None, sort_dir=None,
+                                eager=False):
 
         query = model_query(models.AuditTemplate)
+        if eager:
+            query = self._set_eager_options(models.AuditTemplate, query)
         query = self._add_audit_templates_filters(query, filters)
         if not context.show_deleted:
             query = query.filter_by(deleted_at=None)
@@ -552,34 +582,34 @@ class Connection(api.BaseConnection):
             raise exception.AuditTemplateAlreadyExists(
                 audit_template=values['name'])
 
-        audit_template = models.AuditTemplate()
-        audit_template.update(values)
-
         try:
-            audit_template.save()
+            audit_template = self._create(models.AuditTemplate, values)
         except db_exc.DBDuplicateEntry:
             raise exception.AuditTemplateAlreadyExists(
                 audit_template=values['name'])
         return audit_template
 
-    def _get_audit_template(self, context, fieldname, value):
+    def _get_audit_template(self, context, fieldname, value, eager):
         try:
             return self._get(context, model=models.AuditTemplate,
-                             fieldname=fieldname, value=value)
+                             fieldname=fieldname, value=value, eager=eager)
         except exception.ResourceNotFound:
             raise exception.AuditTemplateNotFound(audit_template=value)
 
-    def get_audit_template_by_id(self, context, audit_template_id):
+    def get_audit_template_by_id(self, context, audit_template_id,
+                                 eager=False):
         return self._get_audit_template(
-            context, fieldname="id", value=audit_template_id)
+            context, fieldname="id", value=audit_template_id, eager=eager)
 
-    def get_audit_template_by_uuid(self, context, audit_template_uuid):
+    def get_audit_template_by_uuid(self, context, audit_template_uuid,
+                                   eager=False):
         return self._get_audit_template(
-            context, fieldname="uuid", value=audit_template_uuid)
+            context, fieldname="uuid", value=audit_template_uuid, eager=eager)
 
-    def get_audit_template_by_name(self, context, audit_template_name):
+    def get_audit_template_by_name(self, context, audit_template_name,
+                                   eager=False):
         return self._get_audit_template(
-            context, fieldname="name", value=audit_template_name)
+            context, fieldname="name", value=audit_template_name, eager=eager)
 
     def destroy_audit_template(self, audit_template_id):
         try:
@@ -610,12 +640,14 @@ class Connection(api.BaseConnection):
     # ### AUDITS ### #
 
     def get_audit_list(self, context, filters=None, limit=None, marker=None,
-                       sort_key=None, sort_dir=None):
+                       sort_key=None, sort_dir=None, eager=False):
         query = model_query(models.Audit)
+        if eager:
+            query = self._set_eager_options(models.Audit, query)
         query = self._add_audits_filters(query, filters)
         if not context.show_deleted:
             query = query.filter(
-                ~(models.Audit.state == audit_objects.State.DELETED))
+                ~(models.Audit.state == objects.audit.State.DELETED))
 
         return _paginate_query(models.Audit, limit, marker,
                                sort_key, sort_dir, query)
@@ -626,41 +658,28 @@ class Connection(api.BaseConnection):
             values['uuid'] = utils.generate_uuid()
 
         if values.get('state') is None:
-            values['state'] = audit_objects.State.PENDING
-
-        audit = models.Audit()
-        audit.update(values)
+            values['state'] = objects.audit.State.PENDING
 
         try:
-            audit.save()
+            audit = self._create(models.Audit, values)
         except db_exc.DBDuplicateEntry:
             raise exception.AuditAlreadyExists(uuid=values['uuid'])
         return audit
 
-    def get_audit_by_id(self, context, audit_id):
-        query = model_query(models.Audit)
-        query = query.filter_by(id=audit_id)
+    def _get_audit(self, context, fieldname, value, eager):
         try:
-            audit = query.one()
-            if not context.show_deleted:
-                if audit.state == audit_objects.State.DELETED:
-                    raise exception.AuditNotFound(audit=audit_id)
-            return audit
-        except exc.NoResultFound:
-            raise exception.AuditNotFound(audit=audit_id)
+            return self._get(context, model=models.Audit,
+                             fieldname=fieldname, value=value, eager=eager)
+        except exception.ResourceNotFound:
+            raise exception.AuditNotFound(audit=value)
 
-    def get_audit_by_uuid(self, context, audit_uuid):
-        query = model_query(models.Audit)
-        query = query.filter_by(uuid=audit_uuid)
+    def get_audit_by_id(self, context, audit_id, eager=False):
+        return self._get_audit(
+            context, fieldname="id", value=audit_id, eager=eager)
 
-        try:
-            audit = query.one()
-            if not context.show_deleted:
-                if audit.state == audit_objects.State.DELETED:
-                    raise exception.AuditNotFound(audit=audit_uuid)
-            return audit
-        except exc.NoResultFound:
-            raise exception.AuditNotFound(audit=audit_uuid)
+    def get_audit_by_uuid(self, context, audit_uuid, eager=False):
+        return self._get_audit(
+            context, fieldname="uuid", value=audit_uuid, eager=eager)
 
     def destroy_audit(self, audit_id):
         def is_audit_referenced(session, audit_id):
@@ -705,12 +724,14 @@ class Connection(api.BaseConnection):
     # ### ACTIONS ### #
 
     def get_action_list(self, context, filters=None, limit=None, marker=None,
-                        sort_key=None, sort_dir=None):
+                        sort_key=None, sort_dir=None, eager=False):
         query = model_query(models.Action)
+        if eager:
+            query = self._set_eager_options(models.Action, query)
         query = self._add_actions_filters(query, filters)
         if not context.show_deleted:
             query = query.filter(
-                ~(models.Action.state == action_objects.State.DELETED))
+                ~(models.Action.state == objects.action.State.DELETED))
         return _paginate_query(models.Action, limit, marker,
                                sort_key, sort_dir, query)
 
@@ -719,39 +740,26 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        action = models.Action()
-        action.update(values)
         try:
-            action.save()
+            action = self._create(models.Action, values)
         except db_exc.DBDuplicateEntry:
             raise exception.ActionAlreadyExists(uuid=values['uuid'])
         return action
 
-    def get_action_by_id(self, context, action_id):
-        query = model_query(models.Action)
-        query = query.filter_by(id=action_id)
+    def _get_action(self, context, fieldname, value, eager):
         try:
-            action = query.one()
-            if not context.show_deleted:
-                if action.state == action_objects.State.DELETED:
-                    raise exception.ActionNotFound(
-                        action=action_id)
-            return action
-        except exc.NoResultFound:
-            raise exception.ActionNotFound(action=action_id)
+            return self._get(context, model=models.Action,
+                             fieldname=fieldname, value=value, eager=eager)
+        except exception.ResourceNotFound:
+            raise exception.ActionNotFound(action=value)
 
-    def get_action_by_uuid(self, context, action_uuid):
-        query = model_query(models.Action)
-        query = query.filter_by(uuid=action_uuid)
-        try:
-            action = query.one()
-            if not context.show_deleted:
-                if action.state == action_objects.State.DELETED:
-                    raise exception.ActionNotFound(
-                        action=action_uuid)
-            return action
-        except exc.NoResultFound:
-            raise exception.ActionNotFound(action=action_uuid)
+    def get_action_by_id(self, context, action_id, eager=False):
+        return self._get_action(
+            context, fieldname="id", value=action_id, eager=eager)
+
+    def get_action_by_uuid(self, context, action_uuid, eager=False):
+        return self._get_action(
+            context, fieldname="uuid", value=action_uuid, eager=eager)
 
     def destroy_action(self, action_id):
         session = get_session()
@@ -766,12 +774,12 @@ class Connection(api.BaseConnection):
         # NOTE(dtantsur): this can lead to very strange errors
         if 'uuid' in values:
             raise exception.Invalid(
-                message=_("Cannot overwrite UUID for an existing "
-                          "Action."))
+                message=_("Cannot overwrite UUID for an existing Action."))
 
         return self._do_update_action(action_id, values)
 
-    def _do_update_action(self, action_id, values):
+    @staticmethod
+    def _do_update_action(action_id, values):
         session = get_session()
         with session.begin():
             query = model_query(models.Action, session=session)
@@ -800,13 +808,16 @@ class Connection(api.BaseConnection):
     # ### ACTION PLANS ### #
 
     def get_action_plan_list(
-            self, context, filters=None, limit=None,
-            marker=None, sort_key=None, sort_dir=None):
+            self, context, filters=None, limit=None, marker=None,
+            sort_key=None, sort_dir=None, eager=False):
         query = model_query(models.ActionPlan)
+        if eager:
+            query = self._set_eager_options(models.ActionPlan, query)
         query = self._add_action_plans_filters(query, filters)
         if not context.show_deleted:
             query = query.filter(
-                ~(models.ActionPlan.state == ap_objects.State.DELETED))
+                ~(models.ActionPlan.state ==
+                  objects.action_plan.State.DELETED))
 
         return _paginate_query(models.ActionPlan, limit, marker,
                                sort_key, sort_dir, query)
@@ -816,41 +827,26 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        action_plan = models.ActionPlan()
-        action_plan.update(values)
-
         try:
-            action_plan.save()
+            action_plan = self._create(models.ActionPlan, values)
         except db_exc.DBDuplicateEntry:
             raise exception.ActionPlanAlreadyExists(uuid=values['uuid'])
         return action_plan
 
-    def get_action_plan_by_id(self, context, action_plan_id):
-        query = model_query(models.ActionPlan)
-        query = query.filter_by(id=action_plan_id)
+    def _get_action_plan(self, context, fieldname, value, eager):
         try:
-            action_plan = query.one()
-            if not context.show_deleted:
-                if action_plan.state == ap_objects.State.DELETED:
-                    raise exception.ActionPlanNotFound(
-                        action_plan=action_plan_id)
-            return action_plan
-        except exc.NoResultFound:
-            raise exception.ActionPlanNotFound(action_plan=action_plan_id)
+            return self._get(context, model=models.ActionPlan,
+                             fieldname=fieldname, value=value, eager=eager)
+        except exception.ResourceNotFound:
+            raise exception.ActionPlanNotFound(action_plan=value)
 
-    def get_action_plan_by_uuid(self, context, action_plan__uuid):
-        query = model_query(models.ActionPlan)
-        query = query.filter_by(uuid=action_plan__uuid)
+    def get_action_plan_by_id(self, context, action_plan_id, eager=False):
+        return self._get_action_plan(
+            context, fieldname="id", value=action_plan_id, eager=eager)
 
-        try:
-            action_plan = query.one()
-            if not context.show_deleted:
-                if action_plan.state == ap_objects.State.DELETED:
-                    raise exception.ActionPlanNotFound(
-                        action_plan=action_plan__uuid)
-            return action_plan
-        except exc.NoResultFound:
-            raise exception.ActionPlanNotFound(action_plan=action_plan__uuid)
+    def get_action_plan_by_uuid(self, context, action_plan_uuid, eager=False):
+        return self._get_action_plan(
+            context, fieldname="uuid", value=action_plan_uuid, eager=eager)
 
     def destroy_action_plan(self, action_plan_id):
         def is_action_plan_referenced(session, action_plan_id):
@@ -884,7 +880,8 @@ class Connection(api.BaseConnection):
 
         return self._do_update_action_plan(action_plan_id, values)
 
-    def _do_update_action_plan(self, action_plan_id, values):
+    @staticmethod
+    def _do_update_action_plan(action_plan_id, values):
         session = get_session()
         with session.begin():
             query = model_query(models.ActionPlan, session=session)
@@ -913,9 +910,12 @@ class Connection(api.BaseConnection):
     # ### EFFICACY INDICATORS ### #
 
     def get_efficacy_indicator_list(self, context, filters=None, limit=None,
-                                    marker=None, sort_key=None, sort_dir=None):
+                                    marker=None, sort_key=None, sort_dir=None,
+                                    eager=False):
 
         query = model_query(models.EfficacyIndicator)
+        if eager:
+            query = self._set_eager_options(models.EfficacyIndicator, query)
         query = self._add_efficacy_indicators_filters(query, filters)
         if not context.show_deleted:
             query = query.filter_by(deleted_at=None)
@@ -927,33 +927,36 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        efficacy_indicator = models.EfficacyIndicator()
-        efficacy_indicator.update(values)
-
         try:
-            efficacy_indicator.save()
+            efficacy_indicator = self._create(models.EfficacyIndicator, values)
         except db_exc.DBDuplicateEntry:
             raise exception.EfficacyIndicatorAlreadyExists(uuid=values['uuid'])
         return efficacy_indicator
 
-    def _get_efficacy_indicator(self, context, fieldname, value):
+    def _get_efficacy_indicator(self, context, fieldname, value, eager):
         try:
             return self._get(context, model=models.EfficacyIndicator,
-                             fieldname=fieldname, value=value)
+                             fieldname=fieldname, value=value, eager=eager)
         except exception.ResourceNotFound:
             raise exception.EfficacyIndicatorNotFound(efficacy_indicator=value)
 
-    def get_efficacy_indicator_by_id(self, context, efficacy_indicator_id):
+    def get_efficacy_indicator_by_id(self, context, efficacy_indicator_id,
+                                     eager=False):
         return self._get_efficacy_indicator(
-            context, fieldname="id", value=efficacy_indicator_id)
+            context, fieldname="id",
+            value=efficacy_indicator_id, eager=eager)
 
-    def get_efficacy_indicator_by_uuid(self, context, efficacy_indicator_uuid):
+    def get_efficacy_indicator_by_uuid(self, context, efficacy_indicator_uuid,
+                                       eager=False):
         return self._get_efficacy_indicator(
-            context, fieldname="uuid", value=efficacy_indicator_uuid)
+            context, fieldname="uuid",
+            value=efficacy_indicator_uuid, eager=eager)
 
-    def get_efficacy_indicator_by_name(self, context, efficacy_indicator_name):
+    def get_efficacy_indicator_by_name(self, context, efficacy_indicator_name,
+                                       eager=False):
         return self._get_efficacy_indicator(
-            context, fieldname="name", value=efficacy_indicator_name)
+            context, fieldname="name",
+            value=efficacy_indicator_name, eager=eager)
 
     def update_efficacy_indicator(self, efficacy_indicator_id, values):
         if 'uuid' in values:
@@ -996,9 +999,11 @@ class Connection(api.BaseConnection):
             plain_fields=plain_fields)
 
     def get_scoring_engine_list(
-        self, context, columns=None, filters=None, limit=None,
-            marker=None, sort_key=None, sort_dir=None):
+            self, context, columns=None, filters=None, limit=None,
+            marker=None, sort_key=None, sort_dir=None, eager=False):
         query = model_query(models.ScoringEngine)
+        if eager:
+            query = self._set_eager_options(models.ScoringEngine, query)
         query = self._add_scoring_engine_filters(query, filters)
         if not context.show_deleted:
             query = query.filter_by(deleted_at=None)
@@ -1011,33 +1016,33 @@ class Connection(api.BaseConnection):
         if not values.get('uuid'):
             values['uuid'] = utils.generate_uuid()
 
-        scoring_engine = models.ScoringEngine()
-        scoring_engine.update(values)
-
         try:
-            scoring_engine.save()
+            scoring_engine = self._create(models.ScoringEngine, values)
         except db_exc.DBDuplicateEntry:
             raise exception.ScoringEngineAlreadyExists(uuid=values['uuid'])
         return scoring_engine
 
-    def _get_scoring_engine(self, context, fieldname, value):
+    def _get_scoring_engine(self, context, fieldname, value, eager):
         try:
             return self._get(context, model=models.ScoringEngine,
-                             fieldname=fieldname, value=value)
+                             fieldname=fieldname, value=value, eager=eager)
         except exception.ResourceNotFound:
             raise exception.ScoringEngineNotFound(scoring_engine=value)
 
-    def get_scoring_engine_by_id(self, context, scoring_engine_id):
+    def get_scoring_engine_by_id(self, context, scoring_engine_id,
+                                 eager=False):
         return self._get_scoring_engine(
-            context, fieldname="id", value=scoring_engine_id)
+            context, fieldname="id", value=scoring_engine_id, eager=eager)
 
-    def get_scoring_engine_by_uuid(self, context, scoring_engine_uuid):
+    def get_scoring_engine_by_uuid(self, context, scoring_engine_uuid,
+                                   eager=False):
         return self._get_scoring_engine(
-            context, fieldname="uuid", value=scoring_engine_uuid)
+            context, fieldname="uuid", value=scoring_engine_uuid, eager=eager)
 
-    def get_scoring_engine_by_name(self, context, scoring_engine_name):
+    def get_scoring_engine_by_name(self, context, scoring_engine_name,
+                                   eager=False):
         return self._get_scoring_engine(
-            context, fieldname="name", value=scoring_engine_name)
+            context, fieldname="name", value=scoring_engine_name, eager=eager)
 
     def destroy_scoring_engine(self, scoring_engine_id):
         try:
@@ -1047,9 +1052,9 @@ class Connection(api.BaseConnection):
                 scoring_engine=scoring_engine_id)
 
     def update_scoring_engine(self, scoring_engine_id, values):
-        if 'id' in values:
+        if 'uuid' in values:
             raise exception.Invalid(
-                message=_("Cannot overwrite ID for an existing "
+                message=_("Cannot overwrite UUID for an existing "
                           "Scoring Engine."))
 
         try:
@@ -1065,3 +1070,67 @@ class Connection(api.BaseConnection):
         except exception.ResourceNotFound:
             raise exception.ScoringEngineNotFound(
                 scoring_engine=scoring_engine_id)
+
+    # ### SERVICES ### #
+
+    def _add_services_filters(self, query, filters):
+        if not filters:
+            filters = {}
+
+        plain_fields = ['id', 'name', 'host']
+
+        return self._add_filters(
+            query=query, model=models.Service, filters=filters,
+            plain_fields=plain_fields)
+
+    def get_service_list(self, context, filters=None, limit=None, marker=None,
+                         sort_key=None, sort_dir=None, eager=False):
+        query = model_query(models.Service)
+        if eager:
+            query = self._set_eager_options(models.Service, query)
+        query = self._add_services_filters(query, filters)
+        if not context.show_deleted:
+            query = query.filter_by(deleted_at=None)
+        return _paginate_query(models.Service, limit, marker,
+                               sort_key, sort_dir, query)
+
+    def create_service(self, values):
+        try:
+            service = self._create(models.Service, values)
+        except db_exc.DBDuplicateEntry:
+            raise exception.ServiceAlreadyExists(name=values['name'],
+                                                 host=values['host'])
+        return service
+
+    def _get_service(self, context, fieldname, value, eager):
+        try:
+            return self._get(context, model=models.Service,
+                             fieldname=fieldname, value=value, eager=eager)
+        except exception.ResourceNotFound:
+            raise exception.ServiceNotFound(service=value)
+
+    def get_service_by_id(self, context, service_id, eager=False):
+        return self._get_service(
+            context, fieldname="id", value=service_id, eager=eager)
+
+    def get_service_by_name(self, context, service_name, eager=False):
+        return self._get_service(
+            context, fieldname="name", value=service_name, eager=eager)
+
+    def destroy_service(self, service_id):
+        try:
+            return self._destroy(models.Service, service_id)
+        except exception.ResourceNotFound:
+            raise exception.ServiceNotFound(service=service_id)
+
+    def update_service(self, service_id, values):
+        try:
+            return self._update(models.Service, service_id, values)
+        except exception.ResourceNotFound:
+            raise exception.ServiceNotFound(service=service_id)
+
+    def soft_delete_service(self, service_id):
+        try:
+            self._soft_delete(models.Service, service_id)
+        except exception.ResourceNotFound:
+            raise exception.ServiceNotFound(service=service_id)

watcher/db/sqlalchemy/models.py

@@ -27,14 +27,15 @@ from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import ForeignKey
 from sqlalchemy import Integer
 from sqlalchemy import Numeric
-from sqlalchemy import schema
+from sqlalchemy import orm
 from sqlalchemy import String
 from sqlalchemy import Text
 from sqlalchemy.types import TypeDecorator, TEXT
+from sqlalchemy import UniqueConstraint
 
 from watcher.common import paths
 
-sql_opts = [
+SQL_OPTS = [
     cfg.StrOpt('mysql_engine',
                default='InnoDB',
                help='MySQL engine to use.')
@@ -43,7 +44,7 @@ sql_opts = [
 _DEFAULT_SQL_CONNECTION = 'sqlite:///{0}'.format(
     paths.state_path_def('watcher.sqlite'))
 
-cfg.CONF.register_opts(sql_opts, 'database')
+cfg.CONF.register_opts(SQL_OPTS, 'database')
 db_options.set_defaults(cfg.CONF, _DEFAULT_SQL_CONNECTION, 'watcher.sqlite')
 
 
@@ -57,6 +58,7 @@ def table_args():
 
 class JsonEncodedType(TypeDecorator):
     """Abstract base type serialized as json-encoded string in db."""
+
     type = None
     impl = TEXT
 
@@ -81,11 +83,13 @@ class JsonEncodedType(TypeDecorator):
 
 class JSONEncodedDict(JsonEncodedType):
     """Represents dict serialized as json-encoded string in db."""
+
     type = dict
 
 
 class JSONEncodedList(JsonEncodedType):
     """Represents list serialized as json-encoded string in db."""
+
     type = list
 
 
@@ -111,35 +115,39 @@ class WatcherBase(models.SoftDeleteMixin,
 Base = declarative_base(cls=WatcherBase)
 
 
+class Goal(Base):
+    """Represents a goal."""
+
+    __tablename__ = 'goals'
+    __table_args__ = (
+        UniqueConstraint('uuid', name='uniq_goals0uuid'),
+        UniqueConstraint('name', 'deleted', name='uniq_goals0name'),
+        table_args(),
+    )
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    uuid = Column(String(36))
+    name = Column(String(63), nullable=False)
+    display_name = Column(String(63), nullable=False)
+    efficacy_specification = Column(JSONEncodedList, nullable=False)
+
+
 class Strategy(Base):
     """Represents a strategy."""
 
     __tablename__ = 'strategies'
     __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_strategies0uuid'),
+        UniqueConstraint('uuid', name='uniq_strategies0uuid'),
+        UniqueConstraint('name', 'deleted', name='uniq_strategies0name'),
         table_args()
     )
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
     uuid = Column(String(36))
     name = Column(String(63), nullable=False)
     display_name = Column(String(63), nullable=False)
     goal_id = Column(Integer, ForeignKey('goals.id'), nullable=False)
     parameters_spec = Column(JSONEncodedDict, nullable=True)
 
-
-class Goal(Base):
-    """Represents a goal."""
-
-    __tablename__ = 'goals'
-    __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_goals0uuid'),
-        table_args(),
-    )
-    id = Column(Integer, primary_key=True)
-    uuid = Column(String(36))
-    name = Column(String(63), nullable=False)
-    display_name = Column(String(63), nullable=False)
-    efficacy_specification = Column(JSONEncodedList, nullable=False)
+    goal = orm.relationship(Goal, foreign_keys=goal_id, lazy=None)
 
 
 class AuditTemplate(Base):
@@ -147,18 +155,20 @@ class AuditTemplate(Base):
 
     __tablename__ = 'audit_templates'
     __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_audit_templates0uuid'),
+        UniqueConstraint('uuid', name='uniq_audit_templates0uuid'),
+        UniqueConstraint('name', 'deleted', name='uniq_audit_templates0name'),
         table_args()
     )
     id = Column(Integer, primary_key=True)
     uuid = Column(String(36))
     name = Column(String(63), nullable=True)
     description = Column(String(255), nullable=True)
-    host_aggregate = Column(Integer, nullable=True)
     goal_id = Column(Integer, ForeignKey('goals.id'), nullable=False)
     strategy_id = Column(Integer, ForeignKey('strategies.id'), nullable=True)
-    extra = Column(JSONEncodedDict)
-    version = Column(String(15), nullable=True)
+    scope = Column(JSONEncodedList)
+
+    goal = orm.relationship(Goal, foreign_keys=goal_id, lazy=None)
+    strategy = orm.relationship(Strategy, foreign_keys=strategy_id, lazy=None)
 
 
 class Audit(Base):
@@ -166,19 +176,41 @@ class Audit(Base):
 
     __tablename__ = 'audits'
     __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_audits0uuid'),
+        UniqueConstraint('uuid', name='uniq_audits0uuid'),
         table_args()
     )
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
     uuid = Column(String(36))
     audit_type = Column(String(20))
     state = Column(String(20), nullable=True)
-    deadline = Column(DateTime, nullable=True)
     parameters = Column(JSONEncodedDict, nullable=True)
     interval = Column(Integer, nullable=True)
-    host_aggregate = Column(Integer, nullable=True)
     goal_id = Column(Integer, ForeignKey('goals.id'), nullable=False)
     strategy_id = Column(Integer, ForeignKey('strategies.id'), nullable=True)
+    scope = Column(JSONEncodedList, nullable=True)
+
+    goal = orm.relationship(Goal, foreign_keys=goal_id, lazy=None)
+    strategy = orm.relationship(Strategy, foreign_keys=strategy_id, lazy=None)
+
+
+class ActionPlan(Base):
+    """Represents an action plan."""
+
+    __tablename__ = 'action_plans'
+    __table_args__ = (
+        UniqueConstraint('uuid', name='uniq_action_plans0uuid'),
+        table_args()
+    )
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    uuid = Column(String(36))
+    first_action_id = Column(Integer)
+    audit_id = Column(Integer, ForeignKey('audits.id'), nullable=False)
+    strategy_id = Column(Integer, ForeignKey('strategies.id'), nullable=False)
+    state = Column(String(20), nullable=True)
+    global_efficacy = Column(JSONEncodedDict, nullable=True)
+
+    audit = orm.relationship(Audit, foreign_keys=audit_id, lazy=None)
+    strategy = orm.relationship(Strategy, foreign_keys=strategy_id, lazy=None)
 
 
 class Action(Base):
@@ -186,10 +218,10 @@ class Action(Base):
 
     __tablename__ = 'actions'
     __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_actions0uuid'),
+        UniqueConstraint('uuid', name='uniq_actions0uuid'),
         table_args()
     )
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
     uuid = Column(String(36), nullable=False)
     action_plan_id = Column(Integer, ForeignKey('action_plans.id'),
                             nullable=False)
@@ -199,22 +231,8 @@ class Action(Base):
     state = Column(String(20), nullable=True)
     next = Column(String(36), nullable=True)
 
-
-class ActionPlan(Base):
-    """Represents an action plan."""
-
-    __tablename__ = 'action_plans'
-    __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_action_plans0uuid'),
-        table_args()
-    )
-    id = Column(Integer, primary_key=True)
-    uuid = Column(String(36))
-    first_action_id = Column(Integer)
-    audit_id = Column(Integer, ForeignKey('audits.id'), nullable=False)
-    strategy_id = Column(Integer, ForeignKey('strategies.id'), nullable=False)
-    state = Column(String(20), nullable=True)
-    global_efficacy = Column(JSONEncodedDict, nullable=True)
+    action_plan = orm.relationship(
+        ActionPlan, foreign_keys=action_plan_id, lazy=None)
 
 
 class EfficacyIndicator(Base):
@@ -222,10 +240,10 @@ class EfficacyIndicator(Base):
 
     __tablename__ = 'efficacy_indicators'
     __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_efficacy_indicators0uuid'),
+        UniqueConstraint('uuid', name='uniq_efficacy_indicators0uuid'),
         table_args()
     )
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
     uuid = Column(String(36))
     name = Column(String(63))
     description = Column(String(255), nullable=True)
@@ -234,16 +252,20 @@ class EfficacyIndicator(Base):
     action_plan_id = Column(Integer, ForeignKey('action_plans.id'),
                             nullable=False)
 
+    action_plan = orm.relationship(
+        ActionPlan, foreign_keys=action_plan_id, lazy=None)
+
 
 class ScoringEngine(Base):
     """Represents a scoring engine."""
 
     __tablename__ = 'scoring_engines'
     __table_args__ = (
-        schema.UniqueConstraint('uuid', name='uniq_scoring_engines0uuid'),
+        UniqueConstraint('uuid', name='uniq_scoring_engines0uuid'),
+        UniqueConstraint('name', 'deleted', name='uniq_scoring_engines0name'),
         table_args()
     )
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
     uuid = Column(String(36), nullable=False)
     name = Column(String(63), nullable=False)
     description = Column(String(255), nullable=True)
@@ -251,3 +273,18 @@ class ScoringEngine(Base):
     # The format might vary between different models (e.g. be JSON, XML or
     # even some custom format), the blob type should cover all scenarios.
     metainfo = Column(Text, nullable=True)
+
+
+class Service(Base):
+    """Represents a service entity"""
+
+    __tablename__ = 'services'
+    __table_args__ = (
+        UniqueConstraint('host', 'name', 'deleted',
+                         name="uniq_services0host0name0deleted"),
+        table_args()
+    )
+    id = Column(Integer, primary_key=True)
+    name = Column(String(255), nullable=False)
+    host = Column(String(255), nullable=False)
+    last_seen_up = Column(DateTime, nullable=True)

watcher/decision_engine/audit/base.py

@@ -22,17 +22,18 @@ import six
 
 from oslo_log import log
 
-from watcher.common.messaging.events import event as watcher_event
-from watcher.decision_engine.messaging import events as de_events
 from watcher.decision_engine.planner import manager as planner_manager
 from watcher.decision_engine.strategy.context import default as default_context
-from watcher.objects import audit as audit_objects
+from watcher import notifications
+from watcher import objects
+from watcher.objects import fields
 
 LOG = log.getLogger(__name__)
 
 
 @six.add_metaclass(abc.ABCMeta)
 class BaseAuditHandler(object):
+
     @abc.abstractmethod
     def execute(self, audit_uuid, request_context):
         raise NotImplementedError()
@@ -72,32 +73,40 @@ class AuditHandler(BaseAuditHandler):
     def strategy_context(self):
         return self._strategy_context
 
-    def notify(self, audit_uuid, event_type, status):
-        event = watcher_event.Event()
-        event.type = event_type
-        event.data = {}
-        payload = {'audit_uuid': audit_uuid,
-                   'audit_status': status}
-        self.messaging.publish_status_event(event.type.name, payload)
+    def do_schedule(self, request_context, audit, solution):
+        try:
+            notifications.audit.send_action_notification(
+                request_context, audit,
+                action=fields.NotificationAction.PLANNER,
+                phase=fields.NotificationPhase.START)
+            self.planner.schedule(request_context, audit.id, solution)
+            notifications.audit.send_action_notification(
+                request_context, audit,
+                action=fields.NotificationAction.PLANNER,
+                phase=fields.NotificationPhase.END)
+        except Exception:
+            notifications.audit.send_action_notification(
+                request_context, audit,
+                action=fields.NotificationAction.PLANNER,
+                priority=fields.NotificationPriority.ERROR,
+                phase=fields.NotificationPhase.ERROR)
+            raise
 
-    def update_audit_state(self, request_context, audit, state):
+    @staticmethod
+    def update_audit_state(audit, state):
         LOG.debug("Update audit state: %s", state)
         audit.state = state
         audit.save()
-        self.notify(audit.uuid, de_events.Events.TRIGGER_AUDIT, state)
 
     def pre_execute(self, audit, request_context):
         LOG.debug("Trigger audit %s", audit.uuid)
         # change state of the audit to ONGOING
-        self.update_audit_state(request_context, audit,
-                                audit_objects.State.ONGOING)
+        self.update_audit_state(audit, objects.audit.State.ONGOING)
 
     def post_execute(self, audit, solution, request_context):
-        self.planner.schedule(request_context, audit.id, solution)
-
+        self.do_schedule(request_context, audit, solution)
         # change state of the audit to SUCCEEDED
-        self.update_audit_state(request_context, audit,
-                                audit_objects.State.SUCCEEDED)
+        self.update_audit_state(audit, objects.audit.State.SUCCEEDED)
 
     def execute(self, audit, request_context):
         try:
@@ -106,5 +115,4 @@ class AuditHandler(BaseAuditHandler):
             self.post_execute(audit, solution, request_context)
         except Exception as e:
             LOG.exception(e)
-            self.update_audit_state(request_context, audit,
-                                    audit_objects.State.FAILED)
+            self.update_audit_state(audit, objects.audit.State.FAILED)

watcher/decision_engine/audit/continuous.py

@@ -25,8 +25,7 @@ from oslo_config import cfg
 
 from watcher.common import context
 from watcher.decision_engine.audit import base
-from watcher.objects import action_plan as action_objects
-from watcher.objects import audit as audit_objects
+from watcher import objects
 
 CONF = cfg.CONF
 
@@ -56,11 +55,11 @@ class ContinuousAuditHandler(base.AuditHandler):
         return self._scheduler
 
     def _is_audit_inactive(self, audit):
-        audit = audit_objects.Audit.get_by_uuid(self.context_show_deleted,
-                                                audit.uuid)
-        if audit.state in (audit_objects.State.CANCELLED,
-                           audit_objects.State.DELETED,
-                           audit_objects.State.FAILED):
+        audit = objects.Audit.get_by_uuid(
+            self.context_show_deleted, audit.uuid)
+        if audit.state in (objects.audit.State.CANCELLED,
+                           objects.audit.State.DELETED,
+                           objects.audit.State.FAILED):
             # if audit isn't in active states, audit's job must be removed to
             # prevent using of inactive audit in future.
             job_to_delete = [job for job in self.jobs
@@ -77,14 +76,13 @@ class ContinuousAuditHandler(base.AuditHandler):
         solution = self.strategy_context.execute_strategy(
             audit, request_context)
 
-        if audit.audit_type == audit_objects.AuditType.CONTINUOUS.value:
+        if audit.audit_type == objects.audit.AuditType.CONTINUOUS.value:
             a_plan_filters = {'audit_uuid': audit.uuid,
-                              'state': action_objects.State.RECOMMENDED}
-            action_plans = action_objects.ActionPlan.list(
-                request_context,
-                filters=a_plan_filters)
+                              'state': objects.action_plan.State.RECOMMENDED}
+            action_plans = objects.ActionPlan.list(
+                request_context, filters=a_plan_filters)
             for plan in action_plans:
-                plan.state = action_objects.State.CANCELLED
+                plan.state = objects.action_plan.State.CANCELLED
                 plan.save()
         return solution
 
@@ -93,18 +91,18 @@ class ContinuousAuditHandler(base.AuditHandler):
             self.execute(audit, request_context)
 
     def post_execute(self, audit, solution, request_context):
-        self.planner.schedule(request_context, audit.id, solution)
+        self.do_schedule(request_context, audit, solution)
 
     def launch_audits_periodically(self):
         audit_context = context.RequestContext(is_admin=True)
         audit_filters = {
-            'audit_type': audit_objects.AuditType.CONTINUOUS.value,
-            'state__in': (audit_objects.State.PENDING,
-                          audit_objects.State.ONGOING,
-                          audit_objects.State.SUCCEEDED)
+            'audit_type': objects.audit.AuditType.CONTINUOUS.value,
+            'state__in': (objects.audit.State.PENDING,
+                          objects.audit.State.ONGOING,
+                          objects.audit.State.SUCCEEDED)
         }
-        audits = audit_objects.Audit.list(audit_context,
-                                          filters=audit_filters)
+        audits = objects.Audit.list(
+            audit_context, filters=audit_filters, eager=True)
         scheduler_job_args = [job.args for job in self.scheduler.get_jobs()
                               if job.name == 'execute_audit']
         for audit in audits:

watcher/decision_engine/cluster/history/base.py

@@ -59,7 +59,7 @@ in any appropriate storage system (InfluxDB, OpenTSDB, MongoDB,...).
 import abc
 import six
 
-""" Work in progress Helper to query metrics """
+"""Work in progress Helper to query metrics"""
 
 
 @six.add_metaclass(abc.ABCMeta)

watcher/decision_engine/goal/efficacy/specs.py

@@ -37,9 +37,9 @@ class ServerConsolidation(base.EfficacySpecification):
             indicators.InstanceMigrationsCount(),
         ]
 
-    def get_global_efficacy_indicator(self, indicators_map):
+    def get_global_efficacy_indicator(self, indicators_map=None):
         value = 0
-        if indicators_map.instance_migrations_count > 0:
+        if indicators_map and indicators_map.instance_migrations_count > 0:
             value = (float(indicators_map.released_compute_nodes_count) /
                      float(indicators_map.instance_migrations_count)) * 100
 

watcher/decision_engine/manager.py

@@ -38,6 +38,7 @@ See :doc:`../architecture` for more details on this component.
 
 from oslo_config import cfg
 
+from watcher.common import service_manager
 from watcher.decision_engine.messaging import audit_endpoint
 from watcher.decision_engine.model.collector import manager
 
@@ -50,13 +51,6 @@ WATCHER_DECISION_ENGINE_OPTS = [
                help='The topic name used for '
                     'control events, this topic '
                     'used for RPC calls'),
-    cfg.StrOpt('status_topic',
-               default='watcher.decision.status',
-               help='The topic name used for '
-                    'status events; this topic '
-                    'is used so as to notify'
-                    'the others components '
-                    'of the system'),
     cfg.ListOpt('notification_topics',
                 default=['versioned_notifications', 'watcher_notifications'],
                 help='The topic names from which notification events '
@@ -78,23 +72,36 @@ CONF.register_group(decision_engine_opt_group)
 CONF.register_opts(WATCHER_DECISION_ENGINE_OPTS, decision_engine_opt_group)
 
 
-class DecisionEngineManager(object):
+class DecisionEngineManager(service_manager.ServiceManager):
 
-    API_VERSION = '1.0'
+    @property
+    def service_name(self):
+        return 'watcher-decision-engine'
 
-    def __init__(self):
-        self.api_version = self.API_VERSION
+    @property
+    def api_version(self):
+        return '1.0'
 
-        self.publisher_id = CONF.watcher_decision_engine.publisher_id
-        self.conductor_topic = CONF.watcher_decision_engine.conductor_topic
-        self.status_topic = CONF.watcher_decision_engine.status_topic
-        self.notification_topics = (
-            CONF.watcher_decision_engine.notification_topics)
+    @property
+    def publisher_id(self):
+        return CONF.watcher_decision_engine.publisher_id
 
-        self.conductor_endpoints = [audit_endpoint.AuditEndpoint]
+    @property
+    def conductor_topic(self):
+        return CONF.watcher_decision_engine.conductor_topic
 
-        self.status_endpoints = []
+    @property
+    def notification_topics(self):
+        return CONF.watcher_decision_engine.notification_topics
 
-        self.collector_manager = manager.CollectorManager()
-        self.notification_endpoints = (
-            self.collector_manager.get_notification_endpoints())
+    @property
+    def conductor_endpoints(self):
+        return [audit_endpoint.AuditEndpoint]
+
+    @property
+    def notification_endpoints(self):
+        return self.collector_manager.get_notification_endpoints()
+
+    @property
+    def collector_manager(self):
+        return manager.CollectorManager()

watcher/decision_engine/messaging/audit_endpoint.py

@@ -23,7 +23,7 @@ from oslo_log import log
 
 from watcher.decision_engine.audit import continuous as continuous_handler
 from watcher.decision_engine.audit import oneshot as oneshot_handler
-from watcher.objects import audit as audit_objects
+from watcher import objects
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
@@ -49,7 +49,7 @@ class AuditEndpoint(object):
         return self._messaging
 
     def do_trigger_audit(self, context, audit_uuid):
-        audit = audit_objects.Audit.get_by_uuid(context, audit_uuid)
+        audit = objects.Audit.get_by_uuid(context, audit_uuid, eager=True)
         self._oneshot_handler.execute(audit, context)
 
     def trigger_audit(self, context, audit_uuid):

watcher/decision_engine/model/base.py

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
+# Copyright (c) 2016 b<>com
 #
-# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+# Authors: Vincent FRANCOISE <Vincent.FRANCOISE@b-com.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,11 +15,22 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
-import enum
+"""
+This component is in charge of executing the
+:ref:`Action Plan <action_plan_definition>` built by the
+:ref:`Watcher Decision Engine <watcher_decision_engine_definition>`.
+
+See: :doc:`../architecture` for more details on this component.
+"""
+
+import abc
+import six
 
 
-class EventTypes(enum.Enum):
-    LAUNCH_ACTION_PLAN = "launch_action_plan"
-    LAUNCH_ACTION = "launch_action"
+@six.add_metaclass(abc.ABCMeta)
+class Model(object):
+
+    @abc.abstractmethod
+    def to_string(self):
+        raise NotImplementedError()

watcher/decision_engine/model/collector/base.py

@@ -19,8 +19,8 @@
 #
 
 """
-A :ref:`Cluster Data Model <cluster_data_model_definition>` is a logical
-representation of the current state and topology of the :ref:`Cluster
+A :ref:`Cluster Data Model <cluster_data_model_definition>` (or CDM) is a
+logical representation of the current state and topology of the :ref:`Cluster
 <cluster_definition>` :ref:`Managed resources <managed_resource_definition>`.
 
 It is represented as a set of :ref:`Managed resources
@@ -31,9 +31,8 @@ to know the current relationships between the different :ref:`resources
 during an :ref:`Audit <audit_definition>` and enables the :ref:`Strategy
 <strategy_definition>` to request information such as:
 
-- What compute nodes are in a given :ref:`Availability Zone
-  <availability_zone_definition>` or a given :ref:`Host Aggregate
-  <host_aggregates_definition>`?
+- What compute nodes are in a given :ref:`Audit Scope
+  <audit_scope_definition>`?
 - What :ref:`Instances <instance_definition>` are hosted on a given compute
   node?
 - What is the current load of a compute node?
@@ -59,8 +58,11 @@ to know:
 In the Watcher project, we aim at providing a some generic and basic
 :ref:`Cluster Data Model <cluster_data_model_definition>` for each :ref:`Goal
 <goal_definition>`, usable in the associated :ref:`Strategies
-<strategy_definition>` through a plugin-based mechanism that are directly
-accessible from the strategies classes in order to:
+<strategy_definition>` through a plugin-based mechanism which are called
+cluster data model collectors (or CDMCs). These CDMCs are responsible for
+loading and keeping up-to-date their associated CDM by listening to events and
+also periodically rebuilding themselves from the ground up. They are also
+directly accessible from the strategies classes. These CDMs are used to:
 
 - simplify the development of a new :ref:`Strategy <strategy_definition>` for a
   given :ref:`Goal <goal_definition>` when there already are some existing

watcher/decision_engine/model/collector/nova.py

@@ -28,16 +28,10 @@ LOG = log.getLogger(__name__)
 
 
 class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
-    """nova
+    """Nova cluster data model collector
 
-    *Description*
-
-    This Nova cluster data model collector creates an in-memory representation
-    of the resources exposed by the compute service.
-
-    *Spec URL*
-
-    <None>
+       The Nova cluster data model collector creates an in-memory
+       representation of the resources exposed by the compute service.
     """
 
     def __init__(self, config, osc=None):

watcher/decision_engine/model/mapping.py

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import threading
-
+from oslo_concurrency import lockutils
 from oslo_log import log
 
 from watcher._i18n import _LW
@@ -28,7 +27,6 @@ class Mapping(object):
         self.model = model
         self.compute_node_mapping = {}
         self.instance_mapping = {}
-        self.lock = threading.Lock()
 
     def map(self, node, instance):
         """Select the node where the instance is launched
@@ -36,9 +34,7 @@ class Mapping(object):
         :param node: the node
         :param instance: the virtual machine or instance
         """
-        try:
-            self.lock.acquire()
-
+        with lockutils.lock(__name__):
             # init first
             if node.uuid not in self.compute_node_mapping.keys():
                 self.compute_node_mapping[node.uuid] = set()
@@ -49,9 +45,6 @@ class Mapping(object):
             # map instance => node
             self.instance_mapping[instance.uuid] = node.uuid
 
-        finally:
-            self.lock.release()
-
     def unmap(self, node, instance):
         """Remove the instance from the node
 
@@ -65,8 +58,7 @@ class Mapping(object):
 
         :rtype : object
         """
-        try:
-            self.lock.acquire()
+        with lockutils.lock(__name__):
             if str(node_uuid) in self.compute_node_mapping:
                 self.compute_node_mapping[str(node_uuid)].remove(
                     str(instance_uuid))
@@ -77,8 +69,6 @@ class Mapping(object):
                     _LW("Trying to delete the instance %(instance)s but it "
                         "was not found on node %(node)s") %
                     {'instance': instance_uuid, 'node': node_uuid})
-        finally:
-            self.lock.release()
 
     def get_mapping(self):
         return self.compute_node_mapping

watcher/decision_engine/model/model_root.py

@@ -22,11 +22,13 @@ import six
 from watcher._i18n import _
 from watcher.common import exception
 from watcher.common import utils
+from watcher.decision_engine.model import base
 from watcher.decision_engine.model import element
 from watcher.decision_engine.model import mapping
 
 
-class ModelRoot(object):
+class ModelRoot(base.Model):
+
     def __init__(self, stale=False):
         self._nodes = utils.Struct()
         self._instances = utils.Struct()
@@ -223,3 +225,53 @@ class ModelRoot(object):
                 root.append(self._build_instance_element(instance))
 
         return etree.tostring(root, pretty_print=True).decode('utf-8')
+
+    @classmethod
+    def from_xml(cls, data):
+        model = cls()
+        root = etree.fromstring(data)
+
+        mem = element.Resource(element.ResourceType.memory)
+        num_cores = element.Resource(element.ResourceType.cpu_cores)
+        disk = element.Resource(element.ResourceType.disk)
+        disk_capacity = element.Resource(element.ResourceType.disk_capacity)
+        model.create_resource(mem)
+        model.create_resource(num_cores)
+        model.create_resource(disk)
+        model.create_resource(disk_capacity)
+
+        for cn in root.findall('.//ComputeNode'):
+            node = element.ComputeNode(cn.get('id'))
+            node.uuid = cn.get('uuid')
+            node.hostname = cn.get('hostname')
+            # set capacity
+            mem.set_capacity(node, int(cn.get(str(mem.name))))
+            disk.set_capacity(node, int(cn.get(str(disk.name))))
+            disk_capacity.set_capacity(
+                node, int(cn.get(str(disk_capacity.name))))
+            num_cores.set_capacity(node, int(cn.get(str(num_cores.name))))
+            node.state = cn.get('state')
+            node.status = cn.get('status')
+
+            model.add_node(node)
+
+        for inst in root.findall('.//Instance'):
+            instance = element.Instance()
+            instance.uuid = inst.get('uuid')
+            instance.state = inst.get('state')
+
+            mem.set_capacity(instance, int(inst.get(str(mem.name))))
+            disk.set_capacity(instance, int(inst.get(str(disk.name))))
+            disk_capacity.set_capacity(
+                instance, int(inst.get(str(disk_capacity.name))))
+            num_cores.set_capacity(
+                instance, int(inst.get(str(num_cores.name))))
+
+            parent = inst.getparent()
+            if parent.tag == 'ComputeNode':
+                node = model.get_node_by_uuid(parent.get('uuid'))
+                model.map_instance(instance, node)
+            else:
+                model.add_instance(instance)
+
+        return model

watcher/decision_engine/model/notification/base.py

@@ -19,8 +19,6 @@
 import abc
 import six
 
-from watcher.common import rpc
-
 
 @six.add_metaclass(abc.ABCMeta)
 class NotificationEndpoint(object):
@@ -38,10 +36,3 @@ class NotificationEndpoint(object):
     @property
     def cluster_data_model(self):
         return self.collector.cluster_data_model
-
-    @property
-    def notifier(self):
-        if self._notifier is None:
-            self._notifier = rpc.get_notifier('decision-engine')
-
-        return self._notifier

watcher/decision_engine/planner/default.py

@@ -128,7 +128,7 @@ class DefaultPlanner(base.BasePlanner):
         }
 
         new_action_plan = objects.ActionPlan(context, **action_plan_dict)
-        new_action_plan.create(context)
+        new_action_plan.create()
 
         return new_action_plan
 
@@ -145,7 +145,7 @@ class DefaultPlanner(base.BasePlanner):
             }
             new_efficacy_indicator = objects.EfficacyIndicator(
                 context, **efficacy_indicator_dict)
-            new_efficacy_indicator.create(context)
+            new_efficacy_indicator.create()
 
             efficacy_indicators.append(new_efficacy_indicator)
         return efficacy_indicators
@@ -156,7 +156,7 @@ class DefaultPlanner(base.BasePlanner):
                       _action.get("action_type"))
 
             new_action = objects.Action(context, **_action)
-            new_action.create(context)
+            new_action.create()
             new_action.save()
 
             if parent_action:

watcher/decision_engine/rpcapi.py

@@ -20,8 +20,8 @@
 from oslo_config import cfg
 
 from watcher.common import exception
-from watcher.common.messaging import notification_handler
 from watcher.common import service
+from watcher.common import service_manager
 from watcher.common import utils
 from watcher.decision_engine import manager
 
@@ -43,20 +43,35 @@ class DecisionEngineAPI(service.Service):
             raise exception.InvalidUuidOrName(name=audit_uuid)
 
         return self.conductor_client.call(
-            context.to_dict(), 'trigger_audit', audit_uuid=audit_uuid)
+            context, 'trigger_audit', audit_uuid=audit_uuid)
 
 
-class DecisionEngineAPIManager(object):
+class DecisionEngineAPIManager(service_manager.ServiceManager):
 
-    API_VERSION = '1.0'
+    @property
+    def service_name(self):
+        return None
 
-    conductor_endpoints = []
-    status_endpoints = [notification_handler.NotificationHandler]
-    notification_endpoints = []
-    notification_topics = []
+    @property
+    def api_version(self):
+        return '1.0'
 
-    def __init__(self):
-        self.publisher_id = CONF.watcher_decision_engine.publisher_id
-        self.conductor_topic = CONF.watcher_decision_engine.conductor_topic
-        self.status_topic = CONF.watcher_decision_engine.status_topic
-        self.api_version = self.API_VERSION
+    @property
+    def publisher_id(self):
+        return CONF.watcher_decision_engine.publisher_id
+
+    @property
+    def conductor_topic(self):
+        return CONF.watcher_decision_engine.conductor_topic
+
+    @property
+    def notification_topics(self):
+        return []
+
+    @property
+    def conductor_endpoints(self):
+        return []
+
+    @property
+    def notification_endpoints(self):
+        return []