Implemented base + moved plugins & service conf

In this changeset, I implemented the main logic although this is
mainly a shameful copy/paste of Nova's blueprint
https://blueprints.launchpad.net/nova/+spec/centralize-config-options

Partially Implements: blueprint centralise-config-opts

Change-Id: Ib645ad5da5c706336bb6ac37e85b027d05665c32

watcher/conf/__init__.py

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+from watcher.conf import service
+
+CONF = cfg.CONF
+
+service.register_opts(CONF)

watcher/conf/_opts.py

@@ -0,0 +1,56 @@
+# -*- encoding: utf-8 -*-
+# Copyright 2014
+# The Cloudscaling Group, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneauth1 import loading as ka_loading
+
+from watcher.api import acl as api_acl
+from watcher.api import app as api_app
+from watcher.applier import manager as applier_manager
+from watcher.common import clients
+from watcher.common import exception
+from watcher.common import paths
+from watcher.db.sqlalchemy import models
+from watcher.decision_engine.audit import continuous
+from watcher.decision_engine import manager as decision_engine_manager
+from watcher.decision_engine.planner import manager as planner_manager
+
+
+def list_opts():
+    """Legacy aggregation of all the watcher config options"""
+    return [
+        ('DEFAULT',
+         (api_app.API_SERVICE_OPTS +
+          api_acl.AUTH_OPTS +
+          exception.EXC_LOG_OPTS +
+          paths.PATH_OPTS)),
+        ('api', api_app.API_SERVICE_OPTS),
+        ('database', models.SQL_OPTS),
+        ('watcher_decision_engine',
+         (decision_engine_manager.WATCHER_DECISION_ENGINE_OPTS +
+          continuous.WATCHER_CONTINUOUS_OPTS)),
+        ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS),
+        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS),
+        ('nova_client', clients.NOVA_CLIENT_OPTS),
+        ('glance_client', clients.GLANCE_CLIENT_OPTS),
+        ('cinder_client', clients.CINDER_CLIENT_OPTS),
+        ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS),
+        ('neutron_client', clients.NEUTRON_CLIENT_OPTS),
+        ('watcher_clients_auth',
+         (ka_loading.get_auth_common_conf_options() +
+          ka_loading.get_auth_plugin_conf_options('password') +
+          ka_loading.get_session_conf_options()))
+    ]

watcher/conf/opts.py

@@ -0,0 +1,95 @@
+# Copyright 2016 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This is the single point of entry to generate the sample configuration
+file for Watcher. It collects all the necessary info from the other modules
+in this package. It is assumed that:
+
+* every other module in this package has a 'list_opts' function which
+  return a dict where
+  * the keys are strings which are the group names
+  * the value of each key is a list of config options for that group
+* the watcher.conf package doesn't have further packages with config options
+* this module is only used in the context of sample file generation
+"""
+
+import collections
+import importlib
+import os
+import pkgutil
+
+LIST_OPTS_FUNC_NAME = "list_opts"
+
+
+def _tupleize(dct):
+    """Take the dict of options and convert to the 2-tuple format."""
+    return [(key, val) for key, val in dct.items()]
+
+
+def list_opts():
+    """Grouped list of all the Watcher-specific configuration options
+
+    :return: A list of ``(group, [opt_1, opt_2])`` tuple pairs, where ``group``
+             is either a group name as a string or an OptGroup object.
+    """
+    opts = collections.defaultdict(list)
+    module_names = _list_module_names()
+    imported_modules = _import_modules(module_names)
+    _append_config_options(imported_modules, opts)
+    return _tupleize(opts)
+
+
+def _list_module_names():
+    module_names = []
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    for __, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
+        if modname == "opts" or ispkg:
+            continue
+        else:
+            module_names.append(modname)
+    return module_names
+
+
+def _import_modules(module_names):
+    imported_modules = []
+    for modname in module_names:
+        mod = importlib.import_module("watcher.conf." + modname)
+        if not hasattr(mod, LIST_OPTS_FUNC_NAME):
+            msg = "The module 'watcher.conf.%s' should have a '%s' "\
+                  "function which returns the config options." % \
+                  (modname, LIST_OPTS_FUNC_NAME)
+            raise Exception(msg)
+        else:
+            imported_modules.append(mod)
+    return imported_modules
+
+
+def _process_old_opts(configs):
+    """Convert old-style 2-tuple configs to dicts."""
+    if isinstance(configs, tuple):
+        configs = [configs]
+    return {label: options for label, options in configs}
+
+
+def _append_config_options(imported_modules, config_options):
+    for mod in imported_modules:
+        configs = mod.list_opts()
+        # TODO(markus_z): Remove this compatibility shim once all list_opts()
+        # functions have been updated to return dicts.
+        if not isinstance(configs, dict):
+            configs = _process_old_opts(configs)
+        for key, val in configs.items():
+            config_options[key].extend(val)

watcher/conf/plugins.py

@@ -0,0 +1,72 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import prettytable as ptable
+
+from watcher.applier.loading import default as applier_loader
+from watcher.common import utils
+from watcher.decision_engine.loading import default as decision_engine_loader
+
+PLUGIN_LOADERS = (
+    applier_loader.DefaultActionLoader,
+    decision_engine_loader.DefaultPlannerLoader,
+    decision_engine_loader.DefaultScoringLoader,
+    decision_engine_loader.DefaultScoringContainerLoader,
+    decision_engine_loader.DefaultStrategyLoader,
+    decision_engine_loader.ClusterDataModelCollectorLoader,
+    applier_loader.DefaultWorkFlowEngineLoader,
+)
+
+
+def list_opts():
+    """Load config options for all Watcher plugins"""
+    plugins_opts = []
+    for plugin_loader_cls in PLUGIN_LOADERS:
+        plugin_loader = plugin_loader_cls()
+        plugins_map = plugin_loader.list_available()
+
+        for plugin_name, plugin_cls in plugins_map.items():
+            plugin_opts = plugin_cls.get_config_opts()
+            if plugin_opts:
+                plugins_opts.append(
+                    (plugin_loader.get_entry_name(plugin_name), plugin_opts))
+
+    return plugins_opts
+
+
+def _show_plugins_ascii_table(rows):
+    headers = ["Namespace", "Plugin name", "Import path"]
+    table = ptable.PrettyTable(field_names=headers)
+    for row in rows:
+        table.add_row(row)
+    return table.get_string()
+
+
+def show_plugins():
+    rows = []
+    for plugin_loader_cls in PLUGIN_LOADERS:
+        plugin_loader = plugin_loader_cls()
+        plugins_map = plugin_loader.list_available()
+
+        rows += [
+            (plugin_loader.get_entry_name(plugin_name),
+             plugin_name,
+             utils.get_cls_import_path(plugin_cls))
+            for plugin_name, plugin_cls in plugins_map.items()]
+
+    return _show_plugins_ascii_table(rows)

watcher/conf/service.py

@@ -0,0 +1,49 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+
+from oslo_config import cfg
+
+from watcher._i18n import _
+
+SERVICE_OPTS = [
+    cfg.IntOpt('periodic_interval',
+               default=60,
+               help=_('Seconds between running periodic tasks.')),
+    cfg.StrOpt('host',
+               default=socket.gethostname(),
+               help=_('Name of this node. This can be an opaque identifier. '
+                      'It is not necessarily a hostname, FQDN, or IP address. '
+                      'However, the node name must be valid within '
+                      'an AMQP key, and if using ZeroMQ, a valid '
+                      'hostname, FQDN, or IP address.')),
+    cfg.IntOpt('service_down_time',
+               default=90,
+               help=_('Maximum time since last check-in for up service.'))
+]
+
+
+def register_opts(conf):
+    conf.register_opts(SERVICE_OPTS)
+
+
+def list_opts():
+    return [
+        ('DEFAULT', SERVICE_OPTS),
+    ]