Implemented base + moved plugins & service conf

In this changeset, I implemented the main logic although this is
mainly a shameful copy/paste of Nova's blueprint
https://blueprints.launchpad.net/nova/+spec/centralize-config-options

Partially Implements: blueprint centralise-config-opts

Change-Id: Ib645ad5da5c706336bb6ac37e85b027d05665c32

setup.cfg

@@ -32,7 +32,7 @@ setup-hooks =
 
 [entry_points]
 oslo.config.opts =
-    watcher = watcher.opts:list_opts
+    watcher = watcher.conf.opts:list_opts
 
 console_scripts =
     watcher-api = watcher.cmd.api:main

watcher/cmd/api.py

@@ -24,13 +24,14 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service
+from watcher import conf
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
 
     host, port = cfg.CONF.api.host, cfg.CONF.api.port
     protocol = "http" if not CONF.api.enable_ssl_api else "https"

watcher/cmd/applier.py

@@ -20,19 +20,19 @@
 import os
 import sys
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.applier import manager
 from watcher.common import service as watcher_service
+from watcher import conf
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    watcher_service.prepare_service(sys.argv)
+    watcher_service.prepare_service(sys.argv, CONF)
 
     LOG.info(_LI('Starting Watcher Applier service in PID %s'), os.getpid())
 

watcher/cmd/dbmanage.py

@@ -24,10 +24,11 @@ import sys
 from oslo_config import cfg
 
 from watcher.common import service
+from watcher import conf
 from watcher.db import migration
 from watcher.db import purge
 
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 class DBCommand(object):
@@ -152,5 +153,5 @@ def main():
     if not set(sys.argv).intersection(valid_commands):
         sys.argv.append('upgrade')
 
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
     CONF.command.func()

watcher/cmd/decisionengine.py

@@ -20,22 +20,22 @@
 import os
 import sys
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as watcher_service
+from watcher import conf
 from watcher.decision_engine import gmr
 from watcher.decision_engine import manager
 from watcher.decision_engine import scheduling
 from watcher.decision_engine import sync
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 def main():
-    watcher_service.prepare_service(sys.argv)
+    watcher_service.prepare_service(sys.argv, CONF)
     gmr.register_gmr_plugins()
 
     LOG.info(_LI('Starting Watcher Decision Engine service in PID %s'),

watcher/cmd/sync.py

@@ -24,15 +24,17 @@ from oslo_log import log as logging
 
 from watcher._i18n import _LI
 from watcher.common import service as service
+from watcher import conf
 from watcher.decision_engine import sync
 
 LOG = logging.getLogger(__name__)
+CONF = conf.CONF
 
 
 def main():
     LOG.info(_LI('Watcher sync started.'))
 
-    service.prepare_service(sys.argv)
+    service.prepare_service(sys.argv, CONF)
     syncer = sync.Syncer()
     syncer.sync()
 

watcher/common/exception.py

@@ -34,14 +34,14 @@ from watcher._i18n import _, _LE
 
 LOG = logging.getLogger(__name__)
 
-exc_log_opts = [
+EXC_LOG_OPTS = [
     cfg.BoolOpt('fatal_exception_format_errors',
                 default=False,
                 help='Make exception message format errors fatal.'),
 ]
 
 CONF = cfg.CONF
-CONF.register_opts(exc_log_opts)
+CONF.register_opts(EXC_LOG_OPTS)
 
 
 def wrap_keystone_exception(func):

watcher/common/service.py

@@ -34,10 +34,10 @@ from watcher.common import config
 from watcher.common import context
 from watcher.common import rpc
 from watcher.common import scheduling
+from watcher.conf import plugins as plugins_conf
 from watcher import objects
 from watcher.objects import base
 from watcher.objects import fields as wfields
-from watcher import opts
 from watcher import version
 
 # NOTE:
@@ -47,24 +47,6 @@ from watcher import version
 # to use libamqp instead.
 eventlet.monkey_patch()
 
-service_opts = [
-    cfg.IntOpt('periodic_interval',
-               default=60,
-               help=_('Seconds between running periodic tasks.')),
-    cfg.StrOpt('host',
-               default=socket.getfqdn(),
-               help=_('Name of this node. This can be an opaque identifier.  '
-                      'It is not necessarily a hostname, FQDN, or IP address. '
-                      'However, the node name must be valid within '
-                      'an AMQP key, and if using ZeroMQ, a valid '
-                      'hostname, FQDN, or IP address.')),
-    cfg.IntOpt('service_down_time',
-               default=90,
-               help=_('Maximum time since last check-in for up service.'))
-]
-
-cfg.CONF.register_opts(service_opts)
-
 NOTIFICATION_OPTS = [
     cfg.StrOpt('notification_level',
                choices=[''] + list(wfields.NotificationPriority.ALL),
@@ -247,7 +229,7 @@ class Service(service.ServiceBase):
         target = om.Target(
             topic=topic_name,
             # For compatibility, we can override it with 'host' opt
-            server=CONF.host or socket.getfqdn(),
+            server=CONF.host or socket.gethostname(),
             version=self.api_version,
         )
         return om.get_rpc_server(
@@ -309,5 +291,6 @@ def prepare_service(argv=(), conf=cfg.CONF):
     conf.log_opt_values(LOG, log.DEBUG)
     objects.register_all()
 
-    gmr.TextGuruMeditation.register_section(_('Plugins'), opts.show_plugins)
+    gmr.TextGuruMeditation.register_section(
+        _('Plugins'), plugins_conf.show_plugins)
     gmr.TextGuruMeditation.setup_autorun(version, conf=conf)

watcher/conf/__init__.py

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+from watcher.conf import service
+
+CONF = cfg.CONF
+
+service.register_opts(CONF)

watcher/conf/_opts.py

@@ -0,0 +1,56 @@
+# -*- encoding: utf-8 -*-
+# Copyright 2014
+# The Cloudscaling Group, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneauth1 import loading as ka_loading
+
+from watcher.api import acl as api_acl
+from watcher.api import app as api_app
+from watcher.applier import manager as applier_manager
+from watcher.common import clients
+from watcher.common import exception
+from watcher.common import paths
+from watcher.db.sqlalchemy import models
+from watcher.decision_engine.audit import continuous
+from watcher.decision_engine import manager as decision_engine_manager
+from watcher.decision_engine.planner import manager as planner_manager
+
+
+def list_opts():
+    """Legacy aggregation of all the watcher config options"""
+    return [
+        ('DEFAULT',
+         (api_app.API_SERVICE_OPTS +
+          api_acl.AUTH_OPTS +
+          exception.EXC_LOG_OPTS +
+          paths.PATH_OPTS)),
+        ('api', api_app.API_SERVICE_OPTS),
+        ('database', models.SQL_OPTS),
+        ('watcher_decision_engine',
+         (decision_engine_manager.WATCHER_DECISION_ENGINE_OPTS +
+          continuous.WATCHER_CONTINUOUS_OPTS)),
+        ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS),
+        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS),
+        ('nova_client', clients.NOVA_CLIENT_OPTS),
+        ('glance_client', clients.GLANCE_CLIENT_OPTS),
+        ('cinder_client', clients.CINDER_CLIENT_OPTS),
+        ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS),
+        ('neutron_client', clients.NEUTRON_CLIENT_OPTS),
+        ('watcher_clients_auth',
+         (ka_loading.get_auth_common_conf_options() +
+          ka_loading.get_auth_plugin_conf_options('password') +
+          ka_loading.get_session_conf_options()))
+    ]

watcher/conf/opts.py

@@ -0,0 +1,95 @@
+# Copyright 2016 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This is the single point of entry to generate the sample configuration
+file for Watcher. It collects all the necessary info from the other modules
+in this package. It is assumed that:
+
+* every other module in this package has a 'list_opts' function which
+  return a dict where
+  * the keys are strings which are the group names
+  * the value of each key is a list of config options for that group
+* the watcher.conf package doesn't have further packages with config options
+* this module is only used in the context of sample file generation
+"""
+
+import collections
+import importlib
+import os
+import pkgutil
+
+LIST_OPTS_FUNC_NAME = "list_opts"
+
+
+def _tupleize(dct):
+    """Take the dict of options and convert to the 2-tuple format."""
+    return [(key, val) for key, val in dct.items()]
+
+
+def list_opts():
+    """Grouped list of all the Watcher-specific configuration options
+
+    :return: A list of ``(group, [opt_1, opt_2])`` tuple pairs, where ``group``
+             is either a group name as a string or an OptGroup object.
+    """
+    opts = collections.defaultdict(list)
+    module_names = _list_module_names()
+    imported_modules = _import_modules(module_names)
+    _append_config_options(imported_modules, opts)
+    return _tupleize(opts)
+
+
+def _list_module_names():
+    module_names = []
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    for __, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
+        if modname == "opts" or ispkg:
+            continue
+        else:
+            module_names.append(modname)
+    return module_names
+
+
+def _import_modules(module_names):
+    imported_modules = []
+    for modname in module_names:
+        mod = importlib.import_module("watcher.conf." + modname)
+        if not hasattr(mod, LIST_OPTS_FUNC_NAME):
+            msg = "The module 'watcher.conf.%s' should have a '%s' "\
+                  "function which returns the config options." % \
+                  (modname, LIST_OPTS_FUNC_NAME)
+            raise Exception(msg)
+        else:
+            imported_modules.append(mod)
+    return imported_modules
+
+
+def _process_old_opts(configs):
+    """Convert old-style 2-tuple configs to dicts."""
+    if isinstance(configs, tuple):
+        configs = [configs]
+    return {label: options for label, options in configs}
+
+
+def _append_config_options(imported_modules, config_options):
+    for mod in imported_modules:
+        configs = mod.list_opts()
+        # TODO(markus_z): Remove this compatibility shim once all list_opts()
+        # functions have been updated to return dicts.
+        if not isinstance(configs, dict):
+            configs = _process_old_opts(configs)
+        for key, val in configs.items():
+            config_options[key].extend(val)

watcher/conf/plugins.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2014
+# Copyright (c) 2016 b<>com
-# The Cloudscaling Group, Inc.
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,18 +16,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keystoneauth1 import loading as ka_loading
 import prettytable as ptable
 
-import watcher.api.app
 from watcher.applier.loading import default as applier_loader
-from watcher.applier import manager as applier_manager
-from watcher.common import clients
 from watcher.common import utils
 from watcher.decision_engine.loading import default as decision_engine_loader
-from watcher.decision_engine import manager as decision_engine_manger
-from watcher.decision_engine.planner import manager as planner_manager
-
 
 PLUGIN_LOADERS = (
     applier_loader.DefaultActionLoader,
@@ -40,29 +34,7 @@ PLUGIN_LOADERS = (
 
 
 def list_opts():
-    watcher_opts = [
+    """Load config options for all Watcher plugins"""
-        ('api', watcher.api.app.API_SERVICE_OPTS),
-        ('watcher_decision_engine',
-         decision_engine_manger.WATCHER_DECISION_ENGINE_OPTS),
-        ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS),
-        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS),
-        ('nova_client', clients.NOVA_CLIENT_OPTS),
-        ('glance_client', clients.GLANCE_CLIENT_OPTS),
-        ('cinder_client', clients.CINDER_CLIENT_OPTS),
-        ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS),
-        ('neutron_client', clients.NEUTRON_CLIENT_OPTS),
-        ('watcher_clients_auth',
-         (ka_loading.get_auth_common_conf_options() +
-          ka_loading.get_auth_plugin_conf_options('password') +
-          ka_loading.get_session_conf_options()))
-    ]
-
-    watcher_opts += list_plugin_opts()
-
-    return watcher_opts
-
-
-def list_plugin_opts():
     plugins_opts = []
     for plugin_loader_cls in PLUGIN_LOADERS:
         plugin_loader = plugin_loader_cls()

watcher/conf/service.py

@@ -0,0 +1,49 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+
+from oslo_config import cfg
+
+from watcher._i18n import _
+
+SERVICE_OPTS = [
+    cfg.IntOpt('periodic_interval',
+               default=60,
+               help=_('Seconds between running periodic tasks.')),
+    cfg.StrOpt('host',
+               default=socket.gethostname(),
+               help=_('Name of this node. This can be an opaque identifier. '
+                      'It is not necessarily a hostname, FQDN, or IP address. '
+                      'However, the node name must be valid within '
+                      'an AMQP key, and if using ZeroMQ, a valid '
+                      'hostname, FQDN, or IP address.')),
+    cfg.IntOpt('service_down_time',
+               default=90,
+               help=_('Maximum time since last check-in for up service.'))
+]
+
+
+def register_opts(conf):
+    conf.register_opts(SERVICE_OPTS)
+
+
+def list_opts():
+    return [
+        ('DEFAULT', SERVICE_OPTS),
+    ]

watcher/db/sqlalchemy/models.py

@@ -35,7 +35,7 @@ from sqlalchemy import UniqueConstraint
 
 from watcher.common import paths
 
-sql_opts = [
+SQL_OPTS = [
     cfg.StrOpt('mysql_engine',
                default='InnoDB',
                help='MySQL engine to use.')
@@ -44,7 +44,7 @@ sql_opts = [
 _DEFAULT_SQL_CONNECTION = 'sqlite:///{0}'.format(
     paths.state_path_def('watcher.sqlite'))
 
-cfg.CONF.register_opts(sql_opts, 'database')
+cfg.CONF.register_opts(SQL_OPTS, 'database')
 db_options.set_defaults(cfg.CONF, _DEFAULT_SQL_CONNECTION, 'watcher.sqlite')
 
 

watcher/tests/cmd/test_db_manage.py

@@ -54,7 +54,7 @@ class TestDBManageRunApp(base.TestCase):
         dbmanage.main()
         self.assertEqual(1, m_func.call_count)
         m_prepare_service.assert_called_once_with(
-            ["watcher-db-manage", self.expected])
+            ["watcher-db-manage", self.expected], cfg.CONF)
 
 
 class TestDBManageRunCommand(base.TestCase):

watcher/tests/conf/test_list_opts.py

@@ -17,7 +17,8 @@
 import mock
 from stevedore import extension
 
-from watcher import opts
+from watcher.conf import opts
+from watcher.conf import plugins
 from watcher.tests import base
 from watcher.tests.decision_engine import fake_strategies
 
@@ -26,10 +27,10 @@ class TestListOpts(base.TestCase):
     def setUp(self):
         super(TestListOpts, self).setUp()
         self.base_sections = [
-            'api', 'watcher_decision_engine', 'watcher_applier',
+            'DEFAULT', 'api', 'database', 'watcher_decision_engine',
-            'watcher_planner', 'nova_client', 'glance_client',
+            'watcher_applier', 'watcher_planner', 'nova_client',
-            'cinder_client', 'ceilometer_client', 'neutron_client',
+            'glance_client', 'cinder_client', 'ceilometer_client',
-            'watcher_clients_auth']
+            'neutron_client', 'watcher_clients_auth']
         self.opt_sections = list(dict(opts.list_opts()).keys())
 
     def test_run_list_opts(self):
@@ -135,10 +136,10 @@ class TestPlugins(base.TestCase):
 
         with mock.patch.object(extension, "ExtensionManager") as m_ext_manager:
             with mock.patch.object(
-                opts, "_show_plugins_ascii_table"
+                plugins, "_show_plugins_ascii_table"
             ) as m_show:
                 m_ext_manager.side_effect = m_list_available
-                opts.show_plugins()
+                plugins.show_plugins()
                 m_show.assert_called_once_with(
                     [('watcher_strategies.strategy_1', 'strategy_1',
                       'watcher.tests.decision_engine.'

watcher/tests/conf_fixture.py

@@ -20,7 +20,7 @@ from oslo_config import cfg
 from watcher.common import config
 
 CONF = cfg.CONF
-CONF.import_opt('host', 'watcher.common.service')
+CONF.import_opt('host', 'watcher.conf.service')
 CONF.import_opt('connection', 'oslo_db.options', group='database')
 CONF.import_opt('sqlite_synchronous', 'oslo_db.options', group='database')