From 6a55914b05c77f7e7fd3e95ce5ce65954939d314 Mon Sep 17 00:00:00 2001 From: Jean-Emile DARTOIS Date: Thu, 19 Nov 2015 19:03:17 +0100 Subject: [PATCH] Fix config loading when running the watcher-api This regression was caused when we upgrade the version of olso. Unfortunately this issue wasn't picked up with the unit tests as the cmds doesn't have unit tests yet. Error message: "watcher-api fail with NoSuchOptError: no such option:debug" This patchset implements unit tests for: - 'watcher-api' command - 'watcher-db-manage' command (and sub-command) - 'watcher-applier' command Change-Id: I2bea8aee28dec913ebc45f2824bf474f86652642 --- watcher/api/app.py | 3 +- watcher/api/config.py | 10 +- watcher/applier/framework/manager_applier.py | 4 +- watcher/cmd/applier.py | 2 +- watcher/cmd/dbmanage.py | 39 +++++--- watcher/cmd/decisionengine.py | 7 +- watcher/common/service.py | 2 +- watcher/tests/base.py | 2 +- watcher/tests/cmd/__init__.py | 0 watcher/tests/cmd/test_api.py | 66 ++++++++++++ watcher/tests/cmd/test_applier.py | 51 ++++++++++ watcher/tests/cmd/test_db_manage.py | 100 +++++++++++++++++++ watcher/tests/cmd/test_decision_engine.py | 54 ++++++++++ watcher/tests/conf_fixture.py | 5 +- 14 files changed, 314 insertions(+), 31 deletions(-) create mode 100644 watcher/tests/cmd/__init__.py create mode 100644 watcher/tests/cmd/test_api.py create mode 100644 watcher/tests/cmd/test_applier.py create mode 100644 watcher/tests/cmd/test_db_manage.py create mode 100644 watcher/tests/cmd/test_decision_engine.py diff --git a/watcher/api/app.py b/watcher/api/app.py index 2c65ba5eb..74131a835 100644 --- a/watcher/api/app.py +++ b/watcher/api/app.py @@ -49,8 +49,7 @@ CONF.register_opts(strategy_selector.WATCHER_GOALS_OPTS) def get_pecan_config(): # Set up the pecan configuration - filename = api_config.__file__.replace('.pyc', '.py') - return pecan.configuration.conf_from_file(filename) + return pecan.configuration.conf_from_dict(api_config.PECAN_CONFIG) def setup_app(config=None): diff --git a/watcher/api/config.py b/watcher/api/config.py index d06cd6c86..d6f7a5e46 100644 --- a/watcher/api/config.py +++ b/watcher/api/config.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals + from oslo_config import cfg from watcher.api import hooks @@ -42,5 +44,11 @@ app = { # WSME Configurations # See https://wsme.readthedocs.org/en/latest/integrate.html#configuration wsme = { - 'debug': cfg.CONF.debug, + 'debug': cfg.CONF.get("debug") if "debug" in cfg.CONF else False, +} + +PECAN_CONFIG = { + "server": server, + "app": app, + "wsme": wsme, } diff --git a/watcher/applier/framework/manager_applier.py b/watcher/applier/framework/manager_applier.py index 303b08b0a..a8200e8a3 100644 --- a/watcher/applier/framework/manager_applier.py +++ b/watcher/applier/framework/manager_applier.py @@ -27,9 +27,9 @@ from watcher.common.messaging.messaging_core import MessagingCore from watcher.common.messaging.notification_handler import NotificationHandler from watcher.decision_engine.framework.messaging.events import Events -CONF = cfg.CONF LOG = log.getLogger(__name__) +CONF = cfg.CONF # Register options APPLIER_MANAGER_OPTS = [ @@ -51,7 +51,7 @@ APPLIER_MANAGER_OPTS = [ help='The identifier used by watcher ' 'module on the message broker') ] -CONF = cfg.CONF + opt_group = cfg.OptGroup(name='watcher_applier', title='Options for the Applier messaging' 'core') diff --git a/watcher/cmd/applier.py b/watcher/cmd/applier.py index ad5eca0b1..496bf009a 100644 --- a/watcher/cmd/applier.py +++ b/watcher/cmd/applier.py @@ -33,7 +33,7 @@ _LI = i18n._LI def main(): - cfg.CONF(sys.argv[1:], project='watcher') + cfg.CONF(sys.argv[1:], project='python-watcher') logging.setup(CONF, 'watcher') LOG.info(_LI('Starting server in PID %s') % os.getpid()) diff --git a/watcher/cmd/dbmanage.py b/watcher/cmd/dbmanage.py index 6c9d6a151..90667d521 100644 --- a/watcher/cmd/dbmanage.py +++ b/watcher/cmd/dbmanage.py @@ -32,34 +32,38 @@ CONF = cfg.CONF class DBCommand(object): - def upgrade(self): + @staticmethod + def upgrade(): migration.upgrade(CONF.command.revision) - def downgrade(self): + @staticmethod + def downgrade(): migration.downgrade(CONF.command.revision) - def revision(self): + @staticmethod + def revision(): migration.revision(CONF.command.message, CONF.command.autogenerate) - def stamp(self): + @staticmethod + def stamp(): migration.stamp(CONF.command.revision) - def version(self): + @staticmethod + def version(): print(migration.version()) - def create_schema(self): + @staticmethod + def create_schema(): migration.create_schema() def add_command_parsers(subparsers): - command_object = DBCommand() - parser = subparsers.add_parser( 'upgrade', help="Upgrade the database schema to the latest version. " "Optionally, use --revision to specify an alembic revision " "string to upgrade to.") - parser.set_defaults(func=command_object.upgrade) + parser.set_defaults(func=DBCommand.upgrade) parser.add_argument('--revision', nargs='?') parser = subparsers.add_parser( @@ -67,12 +71,12 @@ def add_command_parsers(subparsers): help="Downgrade the database schema to the oldest revision. " "While optional, one should generally use --revision to " "specify the alembic revision string to downgrade to.") - parser.set_defaults(func=command_object.downgrade) + parser.set_defaults(func=DBCommand.downgrade) parser.add_argument('--revision', nargs='?') parser = subparsers.add_parser('stamp') parser.add_argument('--revision', nargs='?') - parser.set_defaults(func=command_object.stamp) + parser.set_defaults(func=DBCommand.stamp) parser = subparsers.add_parser( 'revision', @@ -80,17 +84,17 @@ def add_command_parsers(subparsers): "Use --message to set the message string.") parser.add_argument('-m', '--message') parser.add_argument('--autogenerate', action='store_true') - parser.set_defaults(func=command_object.revision) + parser.set_defaults(func=DBCommand.revision) parser = subparsers.add_parser( 'version', help="Print the current version information and exit.") - parser.set_defaults(func=command_object.version) + parser.set_defaults(func=DBCommand.version) parser = subparsers.add_parser( 'create_schema', help="Create the database schema.") - parser.set_defaults(func=command_object.create_schema) + parser.set_defaults(func=DBCommand.create_schema) command_opt = cfg.SubCommandOpt('command', @@ -98,17 +102,20 @@ command_opt = cfg.SubCommandOpt('command', help='Available commands', handler=add_command_parsers) -CONF.register_cli_opt(command_opt) + +def register_sub_command_opts(): + cfg.CONF.register_cli_opt(command_opt) def main(): + register_sub_command_opts() # this is hack to work with previous usage of watcher-dbsync # pls change it to watcher-dbsync upgrade valid_commands = set([ 'upgrade', 'downgrade', 'revision', 'version', 'stamp', 'create_schema', ]) - if not set(sys.argv) & valid_commands: + if not set(sys.argv).intersection(valid_commands): sys.argv.append('upgrade') service.prepare_service(sys.argv) diff --git a/watcher/cmd/decisionengine.py b/watcher/cmd/decisionengine.py index 17b2b9d9f..dd000d205 100644 --- a/watcher/cmd/decisionengine.py +++ b/watcher/cmd/decisionengine.py @@ -28,18 +28,13 @@ from watcher.decision_engine.framework.manager_decision_engine import \ DecisionEngineManager from watcher import i18n -cfg.CONF.import_opt('hostname', - 'watcher.metrics_engine.framework.' - 'datasources.influxdb_collector', - group='watcher_influxdb_collector') - LOG = logging.getLogger(__name__) CONF = cfg.CONF _LI = i18n._LI def main(): - cfg.CONF(sys.argv[1:], project='watcher') + cfg.CONF(sys.argv[1:], project='python-watcher') logging.setup(CONF, 'watcher') LOG.info(_LI('Starting server in PID %s') % os.getpid()) diff --git a/watcher/common/service.py b/watcher/common/service.py index f2adcb37f..56100eaba 100644 --- a/watcher/common/service.py +++ b/watcher/common/service.py @@ -128,4 +128,4 @@ def prepare_service(argv=[]): config.parse_args(argv) cfg.set_defaults(_options.log_opts, default_log_levels=_DEFAULT_LOG_LEVELS) - log.setup(cfg.CONF, 'watcher') + log.setup(cfg.CONF, 'python-watcher') diff --git a/watcher/tests/base.py b/watcher/tests/base.py index f5b0d5e7f..7bbe1c201 100644 --- a/watcher/tests/base.py +++ b/watcher/tests/base.py @@ -42,7 +42,7 @@ class BaseTestCase(testscenarios.WithScenarios, base.BaseTestCase): self.addCleanup(cfg.CONF.reset) -class TestCase(base.BaseTestCase): +class TestCase(BaseTestCase): """Test case base class for all unit tests.""" def setUp(self): diff --git a/watcher/tests/cmd/__init__.py b/watcher/tests/cmd/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/cmd/test_api.py b/watcher/tests/cmd/test_api.py new file mode 100644 index 000000000..1e2bce719 --- /dev/null +++ b/watcher/tests/cmd/test_api.py @@ -0,0 +1,66 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import unicode_literals +from SocketServer import BaseServer + + +import types +from wsgiref import simple_server + +from mock import Mock +from mock import patch +from oslo_config import cfg +from pecan.testing import load_test_app +from watcher.api import config as api_config +from watcher.cmd import api +from watcher.tests.base import BaseTestCase + + +class TestApi(BaseTestCase): + def setUp(self): + super(TestApi, self).setUp() + + self.conf = cfg.CONF + self._parse_cli_opts = self.conf._parse_cli_opts + + def _fake_parse(self, args=[]): + return cfg.ConfigOpts._parse_cli_opts(self, []) + + _fake_parse_method = types.MethodType(_fake_parse, self.conf) + self.conf._parse_cli_opts = _fake_parse_method + + def tearDown(self): + super(TestApi, self).tearDown() + self.conf._parse_cli_opts = self._parse_cli_opts + + @patch("watcher.api.app.pecan.make_app") + @patch.object(BaseServer, "serve_forever", Mock()) + @patch.object(simple_server, "make_server") + def test_run_api_app(self, m_make, m_make_app): + m_make_app.return_value = load_test_app(config=api_config.PECAN_CONFIG) + api.main() + self.assertEqual(m_make.call_count, 1) + + @patch("watcher.api.app.pecan.make_app") + @patch.object(BaseServer, "serve_forever", Mock()) + @patch.object(simple_server, "make_server") + def test_run_api_app_serve_specific_address(self, m_make, m_make_app): + cfg.CONF.set_default("host", "localhost", group="api") + m_make_app.return_value = load_test_app(config=api_config.PECAN_CONFIG) + api.main() + self.assertEqual(m_make.call_count, 1) diff --git a/watcher/tests/cmd/test_applier.py b/watcher/tests/cmd/test_applier.py new file mode 100644 index 000000000..6155ae86c --- /dev/null +++ b/watcher/tests/cmd/test_applier.py @@ -0,0 +1,51 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import types + +from mock import patch +from oslo_config import cfg +from watcher.applier.framework.manager_applier import ApplierManager +from watcher.cmd import applier +from watcher.tests.base import BaseTestCase + + +class TestApplier(BaseTestCase): + def setUp(self): + super(TestApplier, self).setUp() + + self.conf = cfg.CONF + self._parse_cli_opts = self.conf._parse_cli_opts + + def _fake_parse(self, args=[]): + return cfg.ConfigOpts._parse_cli_opts(self, []) + + _fake_parse_method = types.MethodType(_fake_parse, self.conf) + self.conf._parse_cli_opts = _fake_parse_method + + def tearDown(self): + super(TestApplier, self).tearDown() + self.conf._parse_cli_opts = self._parse_cli_opts + + @patch.object(ApplierManager, "connect") + @patch.object(ApplierManager, "join") + def test_run_applier_app(self, m_connect, m_join): + applier.main() + self.assertEqual(m_connect.call_count, 1) + self.assertEqual(m_join.call_count, 1) diff --git a/watcher/tests/cmd/test_db_manage.py b/watcher/tests/cmd/test_db_manage.py new file mode 100644 index 000000000..fe26dc673 --- /dev/null +++ b/watcher/tests/cmd/test_db_manage.py @@ -0,0 +1,100 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mock import Mock +from mock import patch +from oslo_config import cfg +from watcher.cmd import dbmanage +from watcher.db import migration +from watcher.tests.base import TestCase + + +class TestDBManageRunApp(TestCase): + + scenarios = ( + ("upgrade", {"command": "upgrade", "expected": "upgrade"}), + ("downgrade", {"command": "downgrade", "expected": "downgrade"}), + ("revision", {"command": "revision", "expected": "revision"}), + ("stamp", {"command": "stamp", "expected": "stamp"}), + ("version", {"command": "version", "expected": "version"}), + ("create_schema", {"command": "create_schema", + "expected": "create_schema"}), + ("no_param", {"command": None, "expected": "upgrade"}), + ) + + @patch.object(dbmanage, "register_sub_command_opts", Mock()) + @patch("watcher.cmd.dbmanage.service.prepare_service") + @patch("watcher.cmd.dbmanage.sys") + def test_run_db_manage_app(self, m_sys, m_prepare_service): + # Patch command arguments + m_func = Mock() + cfg.CONF.register_opt(cfg.Opt("func"), group="command") + cfg.CONF.set_override("func", m_func, group="command") + # Only append if the command is not None + m_sys.argv = filter(None, ["watcher-db-manage", self.command]) + + dbmanage.main() + self.assertEqual(m_func.call_count, 1) + m_prepare_service.assert_called_once_with( + ["watcher-db-manage", self.expected] + ) + + +class TestDBManageRunCommand(TestCase): + + @patch.object(migration, "upgrade") + def test_run_db_upgrade(self, m_upgrade): + cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command") + cfg.CONF.set_default("revision", "dummy", group="command") + dbmanage.DBCommand.upgrade() + + m_upgrade.assert_called_once_with("dummy") + + @patch.object(migration, "downgrade") + def test_run_db_downgrade(self, m_downgrade): + cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command") + cfg.CONF.set_default("revision", "dummy", group="command") + dbmanage.DBCommand.downgrade() + + m_downgrade.assert_called_once_with("dummy") + + @patch.object(migration, "revision") + def test_run_db_revision(self, m_revision): + cfg.CONF.register_opt(cfg.StrOpt("message"), group="command") + cfg.CONF.register_opt(cfg.StrOpt("autogenerate"), group="command") + cfg.CONF.set_default( + "message", "dummy_message", group="command" + ) + cfg.CONF.set_default( + "autogenerate", "dummy_autogenerate", group="command" + ) + dbmanage.DBCommand.revision() + + m_revision.assert_called_once_with( + "dummy_message", "dummy_autogenerate" + ) + + @patch.object(migration, "stamp") + def test_run_db_stamp(self, m_stamp): + cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command") + cfg.CONF.set_default("revision", "dummy", group="command") + dbmanage.DBCommand.stamp() + + @patch.object(migration, "version") + def test_run_db_version(self, m_version): + dbmanage.DBCommand.version() + + self.assertEqual(m_version.call_count, 1) diff --git a/watcher/tests/cmd/test_decision_engine.py b/watcher/tests/cmd/test_decision_engine.py new file mode 100644 index 000000000..b7efb0ee0 --- /dev/null +++ b/watcher/tests/cmd/test_decision_engine.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import types + +from mock import patch +from oslo_config import cfg +from watcher.tests.base import TestCase + +from watcher.cmd import decisionengine +from watcher.decision_engine.framework.manager_decision_engine import \ + DecisionEngineManager + + +class TestDecisionEngine(TestCase): + + def setUp(self): + super(TestDecisionEngine, self).setUp() + + self.conf = cfg.CONF + self._parse_cli_opts = self.conf._parse_cli_opts + + def _fake_parse(self, args=[]): + return cfg.ConfigOpts._parse_cli_opts(self, []) + + _fake_parse_method = types.MethodType(_fake_parse, self.conf) + self.conf._parse_cli_opts = _fake_parse_method + + def tearDown(self): + super(TestDecisionEngine, self).tearDown() + self.conf._parse_cli_opts = self._parse_cli_opts + + @patch.object(DecisionEngineManager, "connect") + @patch.object(DecisionEngineManager, "join") + def test_run_de_app(self, m_connect, m_join): + decisionengine.main() + self.assertEqual(m_connect.call_count, 1) + self.assertEqual(m_join.call_count, 1) diff --git a/watcher/tests/conf_fixture.py b/watcher/tests/conf_fixture.py index 2e3130075..da50a6dac 100644 --- a/watcher/tests/conf_fixture.py +++ b/watcher/tests/conf_fixture.py @@ -19,7 +19,10 @@ from oslo_config import cfg from watcher.common import config -cfg.CONF.register_opt(cfg.StrOpt('host', default='localhost', help='host')) +CONF = cfg.CONF +CONF.import_opt('host', 'watcher.common.service') +CONF.import_opt('connection', 'oslo_db.options', group='database') +CONF.import_opt('sqlite_synchronous', 'oslo_db.options', group='database') class ConfFixture(fixtures.Fixture):