Fix config loading when running the watcher-api

This regression was caused when we upgrade the version of olso.
Unfortunately this issue wasn't picked up with the unit tests as
the cmds doesn't have unit tests yet.

Error message: "watcher-api fail with NoSuchOptError: no such option:debug"

This patchset implements unit tests for:

- 'watcher-api' command
- 'watcher-db-manage' command (and sub-command)
- 'watcher-applier' command

Change-Id: I2bea8aee28dec913ebc45f2824bf474f86652642

watcher/api/app.py

@@ -49,8 +49,7 @@ CONF.register_opts(strategy_selector.WATCHER_GOALS_OPTS)
 
 def get_pecan_config():
     # Set up the pecan configuration
-    filename = api_config.__file__.replace('.pyc', '.py')
-    return pecan.configuration.conf_from_file(filename)
+    return pecan.configuration.conf_from_dict(api_config.PECAN_CONFIG)
 
 
 def setup_app(config=None):

watcher/api/config.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import unicode_literals
+
 from oslo_config import cfg
 from watcher.api import hooks
 
@@ -42,5 +44,11 @@ app = {
 # WSME Configurations
 # See https://wsme.readthedocs.org/en/latest/integrate.html#configuration
 wsme = {
-    'debug': cfg.CONF.debug,
+    'debug': cfg.CONF.get("debug") if "debug" in cfg.CONF else False,
+}
+
+PECAN_CONFIG = {
+    "server": server,
+    "app": app,
+    "wsme": wsme,
 }

watcher/applier/framework/manager_applier.py

@@ -27,9 +27,9 @@ from watcher.common.messaging.messaging_core import MessagingCore
 from watcher.common.messaging.notification_handler import NotificationHandler
 from watcher.decision_engine.framework.messaging.events import Events
 
-CONF = cfg.CONF
 
 LOG = log.getLogger(__name__)
+CONF = cfg.CONF
 
 # Register options
 APPLIER_MANAGER_OPTS = [
@@ -51,7 +51,7 @@ APPLIER_MANAGER_OPTS = [
                help='The identifier used by watcher '
                     'module on the message broker')
 ]
-CONF = cfg.CONF
+
 opt_group = cfg.OptGroup(name='watcher_applier',
                          title='Options for the Applier messaging'
                                'core')

watcher/cmd/applier.py

@@ -33,7 +33,7 @@ _LI = i18n._LI
 
 
 def main():
-    cfg.CONF(sys.argv[1:], project='watcher')
+    cfg.CONF(sys.argv[1:], project='python-watcher')
     logging.setup(CONF, 'watcher')
 
     LOG.info(_LI('Starting server in PID %s') % os.getpid())

watcher/cmd/dbmanage.py

@@ -32,34 +32,38 @@ CONF = cfg.CONF
 
 class DBCommand(object):
 
-    def upgrade(self):
+    @staticmethod
+    def upgrade():
         migration.upgrade(CONF.command.revision)
 
-    def downgrade(self):
+    @staticmethod
+    def downgrade():
         migration.downgrade(CONF.command.revision)
 
-    def revision(self):
+    @staticmethod
+    def revision():
         migration.revision(CONF.command.message, CONF.command.autogenerate)
 
-    def stamp(self):
+    @staticmethod
+    def stamp():
         migration.stamp(CONF.command.revision)
 
-    def version(self):
+    @staticmethod
+    def version():
         print(migration.version())
 
-    def create_schema(self):
+    @staticmethod
+    def create_schema():
         migration.create_schema()
 
 
 def add_command_parsers(subparsers):
-    command_object = DBCommand()
-
     parser = subparsers.add_parser(
         'upgrade',
         help="Upgrade the database schema to the latest version. "
              "Optionally, use --revision to specify an alembic revision "
              "string to upgrade to.")
-    parser.set_defaults(func=command_object.upgrade)
+    parser.set_defaults(func=DBCommand.upgrade)
     parser.add_argument('--revision', nargs='?')
 
     parser = subparsers.add_parser(
@@ -67,12 +71,12 @@ def add_command_parsers(subparsers):
         help="Downgrade the database schema to the oldest revision. "
              "While optional, one should generally use --revision to "
              "specify the alembic revision string to downgrade to.")
-    parser.set_defaults(func=command_object.downgrade)
+    parser.set_defaults(func=DBCommand.downgrade)
     parser.add_argument('--revision', nargs='?')
 
     parser = subparsers.add_parser('stamp')
     parser.add_argument('--revision', nargs='?')
-    parser.set_defaults(func=command_object.stamp)
+    parser.set_defaults(func=DBCommand.stamp)
 
     parser = subparsers.add_parser(
         'revision',
@@ -80,17 +84,17 @@ def add_command_parsers(subparsers):
              "Use --message to set the message string.")
     parser.add_argument('-m', '--message')
     parser.add_argument('--autogenerate', action='store_true')
-    parser.set_defaults(func=command_object.revision)
+    parser.set_defaults(func=DBCommand.revision)
 
     parser = subparsers.add_parser(
         'version',
         help="Print the current version information and exit.")
-    parser.set_defaults(func=command_object.version)
+    parser.set_defaults(func=DBCommand.version)
 
     parser = subparsers.add_parser(
         'create_schema',
         help="Create the database schema.")
-    parser.set_defaults(func=command_object.create_schema)
+    parser.set_defaults(func=DBCommand.create_schema)
 
 
 command_opt = cfg.SubCommandOpt('command',
@@ -98,17 +102,20 @@ command_opt = cfg.SubCommandOpt('command',
                                 help='Available commands',
                                 handler=add_command_parsers)
 
-CONF.register_cli_opt(command_opt)
+
+def register_sub_command_opts():
+    cfg.CONF.register_cli_opt(command_opt)
 
 
 def main():
+    register_sub_command_opts()
     # this is hack to work with previous usage of watcher-dbsync
     # pls change it to watcher-dbsync upgrade
     valid_commands = set([
         'upgrade', 'downgrade', 'revision',
         'version', 'stamp', 'create_schema',
     ])
-    if not set(sys.argv) & valid_commands:
+    if not set(sys.argv).intersection(valid_commands):
         sys.argv.append('upgrade')
 
     service.prepare_service(sys.argv)

watcher/cmd/decisionengine.py

@@ -28,18 +28,13 @@ from watcher.decision_engine.framework.manager_decision_engine import \
     DecisionEngineManager
 from watcher import i18n
 
-cfg.CONF.import_opt('hostname',
-                    'watcher.metrics_engine.framework.'
-                    'datasources.influxdb_collector',
-                    group='watcher_influxdb_collector')
-
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 _LI = i18n._LI
 
 
 def main():
-    cfg.CONF(sys.argv[1:], project='watcher')
+    cfg.CONF(sys.argv[1:], project='python-watcher')
     logging.setup(CONF, 'watcher')
 
     LOG.info(_LI('Starting server in PID %s') % os.getpid())

watcher/common/service.py

@@ -128,4 +128,4 @@ def prepare_service(argv=[]):
     config.parse_args(argv)
     cfg.set_defaults(_options.log_opts,
                      default_log_levels=_DEFAULT_LOG_LEVELS)
-    log.setup(cfg.CONF, 'watcher')
+    log.setup(cfg.CONF, 'python-watcher')

watcher/tests/base.py

@@ -42,7 +42,7 @@ class BaseTestCase(testscenarios.WithScenarios, base.BaseTestCase):
         self.addCleanup(cfg.CONF.reset)
 
 
-class TestCase(base.BaseTestCase):
+class TestCase(BaseTestCase):
     """Test case base class for all unit tests."""
 
     def setUp(self):

watcher/tests/cmd/test_api.py

@@ -0,0 +1,66 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from SocketServer import BaseServer
+
+
+import types
+from wsgiref import simple_server
+
+from mock import Mock
+from mock import patch
+from oslo_config import cfg
+from pecan.testing import load_test_app
+from watcher.api import config as api_config
+from watcher.cmd import api
+from watcher.tests.base import BaseTestCase
+
+
+class TestApi(BaseTestCase):
+    def setUp(self):
+        super(TestApi, self).setUp()
+
+        self.conf = cfg.CONF
+        self._parse_cli_opts = self.conf._parse_cli_opts
+
+        def _fake_parse(self, args=[]):
+            return cfg.ConfigOpts._parse_cli_opts(self, [])
+
+        _fake_parse_method = types.MethodType(_fake_parse, self.conf)
+        self.conf._parse_cli_opts = _fake_parse_method
+
+    def tearDown(self):
+        super(TestApi, self).tearDown()
+        self.conf._parse_cli_opts = self._parse_cli_opts
+
+    @patch("watcher.api.app.pecan.make_app")
+    @patch.object(BaseServer, "serve_forever", Mock())
+    @patch.object(simple_server, "make_server")
+    def test_run_api_app(self, m_make, m_make_app):
+        m_make_app.return_value = load_test_app(config=api_config.PECAN_CONFIG)
+        api.main()
+        self.assertEqual(m_make.call_count, 1)
+
+    @patch("watcher.api.app.pecan.make_app")
+    @patch.object(BaseServer, "serve_forever", Mock())
+    @patch.object(simple_server, "make_server")
+    def test_run_api_app_serve_specific_address(self, m_make, m_make_app):
+        cfg.CONF.set_default("host", "localhost", group="api")
+        m_make_app.return_value = load_test_app(config=api_config.PECAN_CONFIG)
+        api.main()
+        self.assertEqual(m_make.call_count, 1)

watcher/tests/cmd/test_applier.py

@@ -0,0 +1,51 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import types
+
+from mock import patch
+from oslo_config import cfg
+from watcher.applier.framework.manager_applier import ApplierManager
+from watcher.cmd import applier
+from watcher.tests.base import BaseTestCase
+
+
+class TestApplier(BaseTestCase):
+    def setUp(self):
+        super(TestApplier, self).setUp()
+
+        self.conf = cfg.CONF
+        self._parse_cli_opts = self.conf._parse_cli_opts
+
+        def _fake_parse(self, args=[]):
+            return cfg.ConfigOpts._parse_cli_opts(self, [])
+
+        _fake_parse_method = types.MethodType(_fake_parse, self.conf)
+        self.conf._parse_cli_opts = _fake_parse_method
+
+    def tearDown(self):
+        super(TestApplier, self).tearDown()
+        self.conf._parse_cli_opts = self._parse_cli_opts
+
+    @patch.object(ApplierManager, "connect")
+    @patch.object(ApplierManager, "join")
+    def test_run_applier_app(self, m_connect, m_join):
+        applier.main()
+        self.assertEqual(m_connect.call_count, 1)
+        self.assertEqual(m_join.call_count, 1)

watcher/tests/cmd/test_db_manage.py

@@ -0,0 +1,100 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mock import Mock
+from mock import patch
+from oslo_config import cfg
+from watcher.cmd import dbmanage
+from watcher.db import migration
+from watcher.tests.base import TestCase
+
+
+class TestDBManageRunApp(TestCase):
+
+    scenarios = (
+        ("upgrade", {"command": "upgrade", "expected": "upgrade"}),
+        ("downgrade", {"command": "downgrade", "expected": "downgrade"}),
+        ("revision", {"command": "revision", "expected": "revision"}),
+        ("stamp", {"command": "stamp", "expected": "stamp"}),
+        ("version", {"command": "version", "expected": "version"}),
+        ("create_schema", {"command": "create_schema",
+                           "expected": "create_schema"}),
+        ("no_param", {"command": None, "expected": "upgrade"}),
+    )
+
+    @patch.object(dbmanage, "register_sub_command_opts", Mock())
+    @patch("watcher.cmd.dbmanage.service.prepare_service")
+    @patch("watcher.cmd.dbmanage.sys")
+    def test_run_db_manage_app(self, m_sys, m_prepare_service):
+        # Patch command arguments
+        m_func = Mock()
+        cfg.CONF.register_opt(cfg.Opt("func"), group="command")
+        cfg.CONF.set_override("func", m_func, group="command")
+        # Only append if the command is not None
+        m_sys.argv = filter(None, ["watcher-db-manage", self.command])
+
+        dbmanage.main()
+        self.assertEqual(m_func.call_count, 1)
+        m_prepare_service.assert_called_once_with(
+            ["watcher-db-manage", self.expected]
+        )
+
+
+class TestDBManageRunCommand(TestCase):
+
+    @patch.object(migration, "upgrade")
+    def test_run_db_upgrade(self, m_upgrade):
+        cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
+        cfg.CONF.set_default("revision", "dummy", group="command")
+        dbmanage.DBCommand.upgrade()
+
+        m_upgrade.assert_called_once_with("dummy")
+
+    @patch.object(migration, "downgrade")
+    def test_run_db_downgrade(self, m_downgrade):
+        cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
+        cfg.CONF.set_default("revision", "dummy", group="command")
+        dbmanage.DBCommand.downgrade()
+
+        m_downgrade.assert_called_once_with("dummy")
+
+    @patch.object(migration, "revision")
+    def test_run_db_revision(self, m_revision):
+        cfg.CONF.register_opt(cfg.StrOpt("message"), group="command")
+        cfg.CONF.register_opt(cfg.StrOpt("autogenerate"), group="command")
+        cfg.CONF.set_default(
+            "message", "dummy_message", group="command"
+        )
+        cfg.CONF.set_default(
+            "autogenerate", "dummy_autogenerate", group="command"
+        )
+        dbmanage.DBCommand.revision()
+
+        m_revision.assert_called_once_with(
+            "dummy_message", "dummy_autogenerate"
+        )
+
+    @patch.object(migration, "stamp")
+    def test_run_db_stamp(self, m_stamp):
+        cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
+        cfg.CONF.set_default("revision", "dummy", group="command")
+        dbmanage.DBCommand.stamp()
+
+    @patch.object(migration, "version")
+    def test_run_db_version(self, m_version):
+        dbmanage.DBCommand.version()
+
+        self.assertEqual(m_version.call_count, 1)

watcher/tests/cmd/test_decision_engine.py

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import types
+
+from mock import patch
+from oslo_config import cfg
+from watcher.tests.base import TestCase
+
+from watcher.cmd import decisionengine
+from watcher.decision_engine.framework.manager_decision_engine import \
+    DecisionEngineManager
+
+
+class TestDecisionEngine(TestCase):
+
+    def setUp(self):
+        super(TestDecisionEngine, self).setUp()
+
+        self.conf = cfg.CONF
+        self._parse_cli_opts = self.conf._parse_cli_opts
+
+        def _fake_parse(self, args=[]):
+            return cfg.ConfigOpts._parse_cli_opts(self, [])
+
+        _fake_parse_method = types.MethodType(_fake_parse, self.conf)
+        self.conf._parse_cli_opts = _fake_parse_method
+
+    def tearDown(self):
+        super(TestDecisionEngine, self).tearDown()
+        self.conf._parse_cli_opts = self._parse_cli_opts
+
+    @patch.object(DecisionEngineManager, "connect")
+    @patch.object(DecisionEngineManager, "join")
+    def test_run_de_app(self, m_connect, m_join):
+        decisionengine.main()
+        self.assertEqual(m_connect.call_count, 1)
+        self.assertEqual(m_join.call_count, 1)

watcher/tests/conf_fixture.py

@@ -19,7 +19,10 @@ from oslo_config import cfg
 
 from watcher.common import config
 
-cfg.CONF.register_opt(cfg.StrOpt('host', default='localhost', help='host'))
+CONF = cfg.CONF
+CONF.import_opt('host', 'watcher.common.service')
+CONF.import_opt('connection', 'oslo_db.options', group='database')
+CONF.import_opt('sqlite_synchronous', 'oslo_db.options', group='database')
 
 
 class ConfFixture(fixtures.Fixture):